Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Validation for test csv formats #980

Merged
merged 8 commits into from
Aug 23, 2023
Merged

Validation for test csv formats #980

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
48e99a5
validation wip 1
gnarf Aug 16, 2023
0f56957
move create-example-tests to process-test-directory in lib folder
gnarf Aug 16, 2023
6947388
add check for consistent columns count in CSVs
gnarf Aug 16, 2023
4dd172a
fix column count for a few rows of tests
gnarf Aug 16, 2023
c4ee792
change wording on error slightly
gnarf Aug 21, 2023
63d262d
correct folder name in comment
gnarf Aug 23, 2023
7b671ce
special error for numeric key format
gnarf Aug 23, 2023
ae92bec
Generate .html source files with scripts automatically
gnarf Aug 23, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .eslintignore
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
tests/**/reference/**/*
build/tests/**/*
# keys are manually parsed in create-example-test.js and depend on format
# keys are manually parsed in process-test-directory.js and depend on format
tests/resources/keys.mjs
232 changes: 151 additions & 81 deletions scripts/create-example-tests.js → lib/data/process-test-directory.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
/// <reference path="../types/aria-at-csv.js" />
/// <reference path="../types/aria-at-parsed.js" />
/// <reference path="../types/aria-at-validated.js" />
/// <reference path="../types/aria-at-file.js" />
/// <reference path="../lib/util/file-record-types.js" />
/// <reference path="../../types/aria-at-csv.js" />
/// <reference path="../../types/aria-at-parsed.js" />
/// <reference path="../../types/aria-at-validated.js" />
/// <reference path="../../types/aria-at-file.js" />
/// <reference path="../util/file-record-types.js" />

'use strict';

Expand All @@ -16,28 +16,24 @@ const csv = require('csv-parser');
const fse = require('fs-extra');
const beautify = require('json-beautify');

const { validate } = require('../lib/util/error');
const { reindent } = require('../lib/util/lines');
const { Queryable } = require('../lib/util/queryable');
const { FileRecordChain } = require('../lib/util/file-record-chain');
const { validate } = require('../util/error');
const { reindent } = require('../util/lines');
const { Queryable } = require('../util/queryable');
const { FileRecordChain } = require('../util/file-record-chain');

const { parseSupport } = require('../lib/data/parse-support');
const { parseTestCSVRow } = require('../lib/data/parse-test-csv-row');
const { parseCommandCSVRow } = require('../lib/data/parse-command-csv-row');
const {
createCommandTuplesATModeTaskLookup,
} = require('../lib/data/command-tuples-at-mode-task-lookup');
const { parseSupport } = require('./parse-support');
const { parseTestCSVRow } = require('./parse-test-csv-row');
const { parseCommandCSVRow } = require('./parse-command-csv-row');
const { createCommandTuplesATModeTaskLookup } = require('./command-tuples-at-mode-task-lookup');

const {
renderHTML: renderCollectedTestHtml,
} = require('../lib/data/templates/collected-test.html');
const { createExampleScriptsTemplate } = require('../lib/data/example-scripts-template');
const { renderHTML: renderCollectedTestHtml } = require('./templates/collected-test.html');
const { createExampleScriptsTemplate } = require('./example-scripts-template');

/**
* @param {string} directory - path to directory of data to be used to generate test
* @param {object} args={}
*/
const createExampleTests = async ({ directory, args = {} }) => {
const processTestDirectory = async ({ directory, args = {} }) => {
let VERBOSE_CHECK = false;
let VALIDATE_CHECK = false;

Expand Down Expand Up @@ -79,9 +75,9 @@ const createExampleTests = async ({ directory, args = {} }) => {

const validModes = ['reading', 'interaction', 'item'];

// cwd; @param rootDirectory is dependent on this file not moving from the scripts folder
const scriptsDirectory = path.dirname(__filename);
const rootDirectory = path.join(scriptsDirectory, '..');
// cwd; @param rootDirectory is dependent on this file not moving from the `lib/data` folder
const libDataDirectory = path.dirname(__filename);
const rootDirectory = path.join(libDataDirectory, '../..');

const testsDirectory = path.join(rootDirectory, 'tests');
const testPlanDirectory = path.join(rootDirectory, directory);
Expand Down Expand Up @@ -597,29 +593,103 @@ ${rows}
}
}

function validateCSVKeys(result) {
for (const row of result) {
if (typeof row.refId !== 'string' || typeof row.value !== 'string')
log.error(
`ERROR: References CSV file processing failed: ${referencesCsvFilePath}. Ensure rows are properly formatted.`
);
// intended to be an internal helper to reduce some code duplication and make logging for csv errors simpler
async function readCSVFile(filePath, rowValidator = identity => identity) {
const rawCSV = await readCSV(testPlanRecord.find(filePath));
let index = 0;
function printError(message) {
// line number is index+2
log.warning(
`WARNING: Error parsing ${path.join(testPlanDirectory, filePath)} line ${
index + 2
}: ${message}`
);
}
try {
const firstRowKeysLength = Object.keys(rawCSV[0]).length;
for (; index < rawCSV.length; index++) {
const keysLength = Object.keys(rawCSV[index]).length;
if (keysLength != firstRowKeysLength) {
gnarf marked this conversation as resolved.
Show resolved Hide resolved
printError(
`column number mismatch, please include empty cells to match headers. Expected ${firstRowKeysLength} columns, found ${keysLength}`
);
}
if (!rowValidator(rawCSV[index])) {
printError('validator returned false result');
return;
}
}
} catch (err) {
printError(err);
return;
}
log(`Successfully parsed ${path.join(testPlanDirectory, filePath)}`);
return rawCSV;
}

function validateReferencesKeys(row) {
if (typeof row.refId !== 'string' || typeof row.value !== 'string') {
throw new Error('Row missing refId or value');
}
return row;
}

const validCommandKeys = /^(?:testId|task|mode|at|command[A-Z])$/;
const numericKeyFormat = /^_(\d+)$/;
function validateCommandsKeys(row) {
// example header:
// testId,task,mode,at,commandA,commandB,commandC,commandD,commandE,commandF
for (const key of Object.keys(row)) {
if (numericKeyFormat.test(key)) {
throw new Error(`Column found without header row, ${+key.substring(1) + 1}`);
} else if (!validCommandKeys.test(key)) {
throw new Error(`Unknown commands.csv key: ${key} - check header row?`);
}
}
if (
!(
row.testId?.length &&
row.task?.length &&
row.mode?.length &&
row.at?.length &&
row.commandA?.length
)
) {
throw new Error('Missing one of required testId, task, mode, at, commandA');
}
return row;
}

const validTestsKeys =
/^(?:testId|title|appliesTo|mode|task|setupScript|setupScriptDescription|refs|instructions|assertion(?:[1-9]|[1-2][0-9]|30))$/;
function validateTestsKeys(row) {
// example header:
// testId,title,appliesTo,mode,task,setupScript,setupScriptDescription,refs,instructions,assertion1,assertion2,assertion3,assertion4,assertion5,assertion6,assertion7
for (const key of Object.keys(row)) {
if (numericKeyFormat.test(key)) {
throw new Error(`Column found without header row, ${+key.substring(1) + 1}`);
} else if (!validTestsKeys.test(key)) {
throw new Error(`Unknown tests.csv key: ${key} - check header row?`);
}
}
if (
!(
row.testId?.length &&
row.title?.length &&
row.appliesTo?.length &&
row.mode?.length &&
row.task?.length
)
) {
throw new Error('Missing one of required testId, title, appliesTo, mode, task');
}
log(`References CSV file successfully processed: ${referencesCsvFilePath}`);
return result;
return row;
}

const [refRows, atCommands, tests] = await Promise.all([
readCSV(testPlanRecord.find('data/references.csv'))
.then(rows => rows)
.then(validateCSVKeys),
readCSV(testPlanRecord.find('data/commands.csv')).then(rows => {
log(`Commands CSV file successfully processed: ${atCommandsCsvFilePath}`);
return rows;
}),
readCSV(testPlanRecord.find('data/tests.csv')).then(rows => {
log(`Test CSV file successfully processed: ${testsCsvFilePath}`);
return rows;
}),
readCSVFile('data/references.csv', validateReferencesKeys),
readCSVFile('data/commands.csv', validateCommandsKeys),
readCSVFile('data/tests.csv', validateTestsKeys),
]);

for (const row of refRows) {
Expand Down Expand Up @@ -806,44 +876,7 @@ ${rows}
return { isSuccessfulRun: errorCount === 0, suppressedMessages };
};

function toBuffer(content) {
if (Buffer.isBuffer(content) || isArrayBufferView(content) || isArrayBuffer(content)) {
return content;
} else if (typeof content === 'string') {
return Buffer.from(content);
}
return Buffer.from(content.toString());
}

function exampleTemplateParams(name, source) {
return {
script: reindent`
<!-- Generated by create-example-tests.js -->
<script>
(function() {
function setupScript(testPageDocument) {
// ${name}
${source}
};
document.addEventListener('click', function(event) {
if (event.target.classList.contains('button-run-test-setup')) {
event.target.disabled = true;
setupScript(document);
}
});
})();
</script>
<!-- End of generated output -->`,
button: reindent`
<!-- Generated by create-example-tests.js -->
<div style="position: relative; left: 0; right: 0; height: 2rem;">
<button class="button-run-test-setup" autofocus style="height: 100%; width: 100%;"${
source ? '' : ' disabled'
}>Run Test Setup</button>
</div>
<!-- End of generated output -->`,
};
}
exports.processTestDirectory = processTestDirectory;

/**
* @param {FileRecord.Record} record
Expand Down Expand Up @@ -1300,4 +1333,41 @@ function createCollectedTestHtmlFile(test, testPlanBuildDirectory) {
};
}

exports.createExampleTests = createExampleTests;
function toBuffer(content) {
if (Buffer.isBuffer(content) || isArrayBufferView(content) || isArrayBuffer(content)) {
return content;
} else if (typeof content === 'string') {
return Buffer.from(content);
}
return Buffer.from(content.toString());
}

function exampleTemplateParams(name, source) {
return {
script: reindent`
<!-- Generated by process-test-directory.js -->
<script>
(function() {
function setupScript(testPageDocument) {
// ${name}
${source}
};
document.addEventListener('click', function(event) {
if (event.target.classList.contains('button-run-test-setup')) {
event.target.disabled = true;
setupScript(document);
}
});
})();
</script>
<!-- End of generated output -->`,
button: reindent`
<!-- Generated by process-test-directory.js -->
<div style="position: relative; left: 0; right: 0; height: 2rem;">
<button class="button-run-test-setup" autofocus style="height: 100%; width: 100%;"${
source ? '' : ' disabled'
}>Run Test Setup</button>
</div>
<!-- End of generated output -->`,
};
}
4 changes: 2 additions & 2 deletions scripts/create-all-tests.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ const path = require('path');

const fse = require('fs-extra');

const { createExampleTests } = require('./create-example-tests');
const { processTestDirectory } = require('../lib/data/process-test-directory');

const args = require('minimist')(process.argv.slice(2), {
alias: {
Expand Down Expand Up @@ -59,7 +59,7 @@ async function main() {

const filteredTests = await Promise.all(
filteredTestPlans.map(directory =>
createExampleTests({
processTestDirectory({
directory: path.join('tests', directory),
args,
}).catch(error => {
Expand Down
4 changes: 2 additions & 2 deletions scripts/create-tests.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
'use strict';
const { createExampleTests } = require('./create-example-tests');
const { processTestDirectory } = require('../lib/data/process-test-directory');

const args = require('minimist')(process.argv.slice(2), {
alias: {
Expand Down Expand Up @@ -29,4 +29,4 @@ if (args._.length !== 1) {
process.exit();
}

createExampleTests({ directory: args._[0] });
processTestDirectory({ directory: args._[0] });
4 changes: 2 additions & 2 deletions tests/alert/reference/2022-4-8_144013/alert.setFocusOnButton.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
<link href="css/alert.css" rel="stylesheet">
<script src="js/alert.js" type="text/javascript"></script>

<!-- Generated by create-example-tests.js -->
<!-- Generated by process-test-directory.js -->
<script>
(function() {
function setupScript(testPageDocument) {
Expand Down Expand Up @@ -51,7 +51,7 @@ <h2 id="ex_label">Example</h2>
<div role="separator" id="ex_end_sep" aria-labelledby="ex_end_sep ex_label" aria-label="End of"></div>
</section>

<!-- Generated by create-example-tests.js -->
<!-- Generated by process-test-directory.js -->
<div style="position: relative; left: 0; right: 0; height: 2rem;">
<button class="button-run-test-setup" autofocus style="height: 100%; width: 100%;">Run Test Setup</button>
</div>
Expand Down
4 changes: 2 additions & 2 deletions tests/banner/reference/2021-10-24_135455/banner.setFocusAfterBanner.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
<link href="css/visua11y.css" rel="stylesheet" media="screen"/>
<link href="css/common.css" rel="stylesheet" media="screen"/>

<!-- Generated by create-example-tests.js -->
<!-- Generated by process-test-directory.js -->
<script>
(function() {
function setupScript(testPageDocument) {
Expand Down Expand Up @@ -51,7 +51,7 @@ <h1 id="id1"> Banner Landmark</h1>
<p>A <code>banner</code> landmark identifies site-oriented content at the beginning of each page within a website. Site-oriented content typically includes things such as the logo or identity of the site sponsor, and site-specific search tool. A banner usually appears at the top of the page and typically spans the full width.
</p>

<!-- Generated by create-example-tests.js -->
<!-- Generated by process-test-directory.js -->
<div style="position: relative; left: 0; right: 0; height: 2rem;">
<button class="button-run-test-setup" autofocus style="height: 100%; width: 100%;">Run Test Setup</button>
</div>
Expand Down
4 changes: 2 additions & 2 deletions tests/banner/reference/2021-10-24_135455/banner.setFocusAfterBannerAndHideBottomLink.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
<link href="css/visua11y.css" rel="stylesheet" media="screen"/>
<link href="css/common.css" rel="stylesheet" media="screen"/>

<!-- Generated by create-example-tests.js -->
<!-- Generated by process-test-directory.js -->
<script>
(function() {
function setupScript(testPageDocument) {
Expand Down Expand Up @@ -52,7 +52,7 @@ <h1 id="id1"> Banner Landmark</h1>
<p>A <code>banner</code> landmark identifies site-oriented content at the beginning of each page within a website. Site-oriented content typically includes things such as the logo or identity of the site sponsor, and site-specific search tool. A banner usually appears at the top of the page and typically spans the full width.
</p>

<!-- Generated by create-example-tests.js -->
<!-- Generated by process-test-directory.js -->
<div style="position: relative; left: 0; right: 0; height: 2rem;">
<button class="button-run-test-setup" autofocus style="height: 100%; width: 100%;">Run Test Setup</button>
</div>
Expand Down
4 changes: 2 additions & 2 deletions tests/banner/reference/2021-10-24_135455/banner.setFocusBeforeBanner.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
<link href="css/visua11y.css" rel="stylesheet" media="screen"/>
<link href="css/common.css" rel="stylesheet" media="screen"/>

<!-- Generated by create-example-tests.js -->
<!-- Generated by process-test-directory.js -->
<script>
(function() {
function setupScript(testPageDocument) {
Expand Down Expand Up @@ -51,7 +51,7 @@ <h1 id="id1"> Banner Landmark</h1>
<p>A <code>banner</code> landmark identifies site-oriented content at the beginning of each page within a website. Site-oriented content typically includes things such as the logo or identity of the site sponsor, and site-specific search tool. A banner usually appears at the top of the page and typically spans the full width.
</p>

<!-- Generated by create-example-tests.js -->
<!-- Generated by process-test-directory.js -->
<div style="position: relative; left: 0; right: 0; height: 2rem;">
<button class="button-run-test-setup" autofocus style="height: 100%; width: 100%;">Run Test Setup</button>
</div>
Expand Down
Loading