422 lines
17 KiB
JavaScript
422 lines
17 KiB
JavaScript
#!/usr/bin/env node
|
|
// Anti-regression gate for the static accessibility test suite.
|
|
//
|
|
// We enforce a minimum number of `*.a11y.test.tsx` / `*.axe.test.tsx` files
|
|
// in the web app.
|
|
// Removing a11y coverage is a deliberate act and must show up in code review.
|
|
//
|
|
// Pair with `scripts/verify-test-density.mjs` — that script enforces a
|
|
// per-workspace ratio of test files to source files but does not care
|
|
// about test type. This script keeps the a11y class of tests from
|
|
// silently dropping to zero.
|
|
|
|
import fs from "node:fs";
|
|
import path from "node:path";
|
|
import { pathToFileURL } from "node:url";
|
|
import { fileURLToPath } from "node:url";
|
|
|
|
export const repoRoot = path.resolve(fileURLToPath(import.meta.url), "..", "..");
|
|
export const componentSearchRoot = path.join(repoRoot, "apps/web/app");
|
|
export const e2eSearchRoot = path.join(repoRoot, "tests/e2e");
|
|
export const MIN_COMPONENT_A11Y_TESTS = 12;
|
|
export const MIN_E2E_A11Y_SPECS = 7;
|
|
export const REQUIRED_E2E_FLOW_ASSERTIONS = {
|
|
"registration-account-dashboard": {
|
|
file: "tests/e2e/registration-account-dashboard.mobile-a11y.spec.ts",
|
|
axe: true,
|
|
mobileOverflow: true,
|
|
requiredSnippets: [
|
|
"E2E_ALLOW_REGISTRATION_CREATE",
|
|
"E2E_ALLOW_ACCOUNT_CREATE",
|
|
"E2E_ALLOW_NO_TENANT_ACCOUNT_CREATE",
|
|
"guards seeded tenant account membership before dashboard and account smoke",
|
|
"replays new-account login redirects and returns to the dashboard on mobile",
|
|
"can create a tenant account and reach its dashboard when explicitly enabled",
|
|
"can onboard a registered user with no tenant through account creation when explicitly enabled",
|
|
"can register a new user and stop at the email verification state when explicitly enabled",
|
|
"E2E_REGISTRATION_EXPECTS_EMAIL_VERIFICATION",
|
|
"zeroTenantAccountStatePattern",
|
|
"/login?redirectTo=",
|
|
'[data-ui-id="ui.account.create.form"]',
|
|
"failOnPaymentProviderNetworkRequests(",
|
|
"tenantSettingsSubscriptionPath(",
|
|
'[data-ui-id^="ui.billing.selfServiceSubscription."]',
|
|
'[data-ui-id="ui.subscription.billing.portal"]',
|
|
],
|
|
},
|
|
"plugin-settings": {
|
|
file: "tests/e2e/plugin-settings.mobile-a11y.spec.ts",
|
|
axe: true,
|
|
mobileOverflow: true,
|
|
requiredSnippets: [
|
|
"lifecycle controls submit through a non-mutating interception contract",
|
|
"lifecycle errors stay announced and keep controls retryable",
|
|
"lifecycle controls persist enable and disable state for the seeded lifecycle plugin",
|
|
"app.plugins.use users cannot submit forged lifecycle mutations without manage permission",
|
|
"settings iframe exposes sandbox and CSP contract when available",
|
|
"settings iframe persists values across reload when write slice is enabled",
|
|
// AK-UI-05: lifecycle and revoke are ~/ui controls, not hidden-input <Form>s.
|
|
// The lifecycle button carries its op; the capability review is a real dialog.
|
|
'[data-ui-id="ui.plugins.lifecycle.button"]',
|
|
'[data-ui-id="ui.plugins.capabilityReview.form"]',
|
|
'[data-ui-id="ui.plugins.capability.revoke.confirm"]',
|
|
"page.waitForURL(/\\/unauthorized\\/app\\.plugins\\.manage",
|
|
'[data-ui-id="ui.plugins.settings.frame"]',
|
|
],
|
|
},
|
|
"tenant-api-docs-logs": {
|
|
file: "tests/e2e/tenant-api-docs-logs.mobile-a11y.spec.ts",
|
|
axe: true,
|
|
mobileOverflow: true,
|
|
},
|
|
"tenant-api-keys": {
|
|
file: "tests/e2e/tenant-api-keys.mobile-a11y.spec.ts",
|
|
axe: true,
|
|
mobileOverflow: true,
|
|
requiredSnippets: [
|
|
"opens create form without mobile overflow or critical a11y violations",
|
|
"submits create contract with alias, active state, and csrf without mutating data",
|
|
"E2E_TENANT_API_KEYS_PATH",
|
|
"interceptCurrentRoutePost(",
|
|
'form:has(input[name="action"][value="create"])',
|
|
'input[name="alias"]',
|
|
'input[name="active"]',
|
|
],
|
|
},
|
|
"tenant-audit-logs": {
|
|
file: "tests/e2e/tenant-audit-logs.mobile-a11y.spec.ts",
|
|
axe: true,
|
|
mobileOverflow: true,
|
|
requiredSnippets: [
|
|
"keeps audit logs usable at",
|
|
"keeps audit event filters and table reachable on mobile",
|
|
"compactMobileViewport",
|
|
'[data-ui-id="ui.settings.logs.page"]',
|
|
'[data-ui-id="ui.settings.logs.filters"]',
|
|
'[data-ui-id="ui.settings.logs.table"]',
|
|
'[data-ui-id="ui.settings.logs.events.table"]',
|
|
],
|
|
},
|
|
"tenant-audit-export": {
|
|
file: "tests/e2e/tenant-audit-export.mobile-a11y.spec.ts",
|
|
axe: true,
|
|
mobileOverflow: true,
|
|
requiredSnippets: [
|
|
"keeps compliance export filters usable at",
|
|
"compactMobileViewport",
|
|
'[data-ui-id="ui.audit.export"]',
|
|
'[data-ui-id="ui.audit.export.form"]',
|
|
'[data-ui-id="ui.audit.export.csv"]',
|
|
'[data-ui-id="ui.audit.export.pdf"]',
|
|
],
|
|
},
|
|
"privacy-self-service": {
|
|
file: "tests/e2e/privacy-self-service.mobile-a11y.spec.ts",
|
|
axe: true,
|
|
mobileOverflow: true,
|
|
requiredSnippets: [
|
|
"renders export and deletion controls without mobile overflow or critical a11y violations",
|
|
"submits export request with intent and csrf without mutating data",
|
|
"submits deletion request with confirmation, intent and csrf without mutating data",
|
|
"interceptCurrentRoutePost(",
|
|
'[data-ui-id="ui.settings.privacy.page"]',
|
|
'[data-ui-id="ui.settings.privacy.export.form"]',
|
|
'[data-ui-id="ui.settings.privacy.delete.form"]',
|
|
],
|
|
},
|
|
"profile-security": {
|
|
file: "tests/e2e/profile-security.mobile-a11y.spec.ts",
|
|
axe: true,
|
|
mobileOverflow: true,
|
|
requiredSnippets: [
|
|
"renders profile security dialogs and the dedicated session page on mobile",
|
|
"submits password change contract with csrf without mutating data",
|
|
"submits MFA toggle contract with csrf without changing enrollment state",
|
|
"E2E_TENANT_SETTINGS_PROFILE_PATH",
|
|
"interceptCurrentRoutePost(",
|
|
'[data-ui-id="ui.settings.profile.page"]',
|
|
'[data-ui-id="ui.settings.profile.password.modal"]',
|
|
'[data-ui-id="ui.settings.profile.mfa.gate"]',
|
|
'[data-ui-id="ui.settings.security.page"]',
|
|
'[data-ui-id="ui.settings.security.sessions"]',
|
|
],
|
|
},
|
|
"tenant-members": {
|
|
file: "tests/e2e/tenant-members.mobile-a11y.spec.ts",
|
|
axe: true,
|
|
mobileOverflow: true,
|
|
requiredSnippets: [
|
|
"renders members table at",
|
|
"submits new member invite contract with csrf without mutating data",
|
|
"compactMobileViewport",
|
|
"interceptCurrentRoutePost(",
|
|
'[data-ui-id="ui.settings.members.page"]',
|
|
'[data-ui-id="ui.settings.members.table"]',
|
|
'[data-ui-id="ui.settings.members.new.form"]',
|
|
],
|
|
},
|
|
"tenant-notifications-settings": {
|
|
file: "tests/e2e/tenant-notifications-settings.mobile-a11y.spec.ts",
|
|
axe: true,
|
|
mobileOverflow: true,
|
|
requiredSnippets: [
|
|
"renders notification categories without mobile overflow or critical a11y violations",
|
|
"submits preference update contract with csrf without mutating data",
|
|
"interceptCurrentRoutePost(",
|
|
'[data-ui-id="ui.settings.notifications.page"]',
|
|
"ui.settings.notifications.category.${category}",
|
|
'[data-ui-id="ui.settings.notifications.preference.team.email"]',
|
|
],
|
|
},
|
|
"workflow-builder-run": {
|
|
file: "tests/e2e/workflow-builder-run.mobile-a11y.spec.ts",
|
|
axe: true,
|
|
mobileOverflow: true,
|
|
requiredSnippets: [
|
|
"builder stays usable on mobile",
|
|
"run dropdown stays keyboard reachable without starting execution",
|
|
"manual run contract stays usable on mobile",
|
|
"stream run waiting-block continuation stays usable on mobile",
|
|
"manual run result deep-links to execution details on mobile when explicitly enabled",
|
|
"manual run reaches expected execution status when outbox-drain readiness is required",
|
|
"E2E_ALLOW_WORKFLOW_RUN",
|
|
"requireWorkflowOutboxDrainReadiness(",
|
|
"waitForWorkflowExecutionTerminal(",
|
|
'[data-ui-id="ui.workflows.builder.canvas"]',
|
|
'[data-ui-id="ui.workflows.run.manual.form"]',
|
|
'[data-ui-id="ui.workflows.run.stream.progress"]',
|
|
'[data-ui-id="ui.workflows.run.manual.viewExecutionFlow"]',
|
|
],
|
|
},
|
|
"admin-email-sequences": {
|
|
file: "tests/e2e/admin-email-sequences.mobile-a11y.spec.ts",
|
|
axe: true,
|
|
mobileOverflow: true,
|
|
requiredSnippets: [
|
|
"renders seeded sequence list, detail, and stats surfaces on mobile",
|
|
"toggle lifecycle form posts intent without mutating the seeded sequence",
|
|
"E2E_ADMIN_EMAIL_SEQUENCES_PATH",
|
|
"seq_e2e_customer_welcome",
|
|
"lead.converted_to_customer",
|
|
"interceptCurrentRoutePost(",
|
|
'postDataIncludesField(submittedPostData.value, "intent", /^toggle$/u)',
|
|
],
|
|
},
|
|
"readiness-golden-path": {
|
|
file: "tests/e2e/readiness-golden-path.mobile-a11y.spec.ts",
|
|
axe: true,
|
|
mobileOverflow: true,
|
|
requiredSnippets: [
|
|
"navigates authenticated mobile readiness surfaces without mutations",
|
|
"failOnPaymentProviderNetworkRequests(",
|
|
'[data-ui-id^="ui.billing.selfServiceSubscription."]',
|
|
'[data-ui-id="ui.billing.selfServiceSubscription.builder"]',
|
|
'[data-ui-id="ui.billing.selfServiceSubscription.preview"]',
|
|
'[data-ui-id="ui.plugins.page"]',
|
|
],
|
|
},
|
|
"subscription-smoke": {
|
|
file: "tests/e2e/va-p24a-subscription-smoke.spec.ts",
|
|
axe: true,
|
|
mobileOverflow: false,
|
|
// VA-P24a keeps the flow non-mutating while exercising the real server-side
|
|
// preview. The Mollie checkout action remains deliberately untouched.
|
|
requiredSnippets: [
|
|
"validates plan and seats on the server without starting Mollie checkout",
|
|
'[data-ui-id="ui.billing.selfServiceSubscription.page"]',
|
|
'[data-ui-id="ui.billing.selfServiceSubscription.builder"]',
|
|
'[data-ui-id="ui.billing.selfServiceSubscription.officeSeats"]',
|
|
'[data-ui-id="ui.billing.selfServiceSubscription.fieldSeats"]',
|
|
'[data-ui-id="ui.billing.selfServiceSubscription.preview"]',
|
|
'[data-ui-id="ui.billing.selfServiceSubscription.previewDialog"]',
|
|
'postDataIncludesField(request.postData(), "intent", /^preview$/u)',
|
|
"expect(checkoutSubmissions).toEqual([])",
|
|
"failOnPaymentProviderNetworkRequests(",
|
|
],
|
|
},
|
|
"mobile-authenticated-shell": {
|
|
file: "tests/e2e/mobile-authenticated-shell.spec.ts",
|
|
axe: true,
|
|
mobileOverflow: true,
|
|
requiredSnippets: [
|
|
"traps keyboard focus inside the mobile sidebar and returns focus to the trigger",
|
|
"mobile command palette focuses search and closes with Escape",
|
|
"mobile more menu exposes profile and sign-out contracts without submitting",
|
|
"skip link moves keyboard focus to the main app region",
|
|
"signs out from the mobile more menu and protects the dashboard",
|
|
'[data-ui-id="ui.action.shell.sidebarTrigger"]',
|
|
'[data-ui-id="ui.surface.commandPalette.portal"]',
|
|
"skip-to-content",
|
|
],
|
|
},
|
|
"authenticated-mobile-locale": {
|
|
file: "tests/e2e/authenticated-locale.mobile.spec.ts",
|
|
axe: true,
|
|
mobileOverflow: true,
|
|
requiredSnippets: [
|
|
"persists an authenticated locale switch across mobile shell navigation",
|
|
'await page.reload({ waitUntil: "domcontentloaded" });',
|
|
"tenantSettingsProfilePath(dashboardPath)",
|
|
"restoreLocale(page, initialLocale)",
|
|
'[data-ui-id="ui.nav.app.dashboard"]',
|
|
'[data-ui-id="ui.nav.app.settings"]',
|
|
],
|
|
},
|
|
"public-auth": {
|
|
file: "tests/e2e/public-auth.mobile-a11y.spec.ts",
|
|
axe: true,
|
|
mobileOverflow: true,
|
|
},
|
|
"auth-locale": {
|
|
file: "tests/e2e/auth-locale.mobile-a11y.spec.ts",
|
|
axe: true,
|
|
mobileOverflow: true,
|
|
},
|
|
"mobile-public": {
|
|
file: "tests/e2e/mobile-public.spec.ts",
|
|
axe: true,
|
|
mobileOverflow: true,
|
|
requiredSnippets: [
|
|
"mobile public pages",
|
|
"compact mobile public pages",
|
|
"landing page fits a narrow mobile viewport",
|
|
"not-found page fits a narrow mobile viewport",
|
|
"feature-disabled page has no mobile overflow or critical a11y violations",
|
|
"public footer selectors open without mobile overflow",
|
|
"fits a 320px mobile viewport",
|
|
"feature-disabled page handles long labels on compact mobile",
|
|
"public footer selectors stay inside a 320px viewport",
|
|
"compactMobileViewport",
|
|
'"/privacy-policy"',
|
|
'"/terms-and-conditions"',
|
|
'"/401"',
|
|
'"/403"',
|
|
'"/500"',
|
|
],
|
|
},
|
|
"tenant-governance-settings": {
|
|
file: "tests/e2e/tenant-governance-settings.mobile-a11y.spec.ts",
|
|
axe: true,
|
|
mobileOverflow: true,
|
|
},
|
|
"tenant-roles-permissions-mobile": {
|
|
file: "tests/e2e/tenant-roles-permissions.mobile-a11y.spec.ts",
|
|
axe: true,
|
|
mobileOverflow: true,
|
|
},
|
|
};
|
|
export const REQUIRED_E2E_FLOW_SPECS = Object.fromEntries(Object.entries(REQUIRED_E2E_FLOW_ASSERTIONS).map(([flow, contract]) => [flow, contract.file]));
|
|
const ignoredDirs = new Set(["node_modules", "dist", "build", "coverage", ".react-router", "generated", "__generated__"]);
|
|
|
|
function isComponentA11yTest(name) {
|
|
return name.endsWith(".a11y.test.tsx") || name.endsWith(".axe.test.tsx");
|
|
}
|
|
|
|
function isE2eA11ySpec(name) {
|
|
return (
|
|
name.endsWith(".a11y.spec.ts") ||
|
|
name.endsWith(".a11y.spec.tsx") ||
|
|
name.endsWith(".mobile-a11y.spec.ts") ||
|
|
name.endsWith(".mobile-a11y.spec.tsx") ||
|
|
name.endsWith(".desktop-a11y.spec.ts") ||
|
|
name.endsWith(".desktop-a11y.spec.tsx")
|
|
);
|
|
}
|
|
|
|
function isE2eSpec(name) {
|
|
return name.endsWith(".spec.ts") || name.endsWith(".spec.tsx");
|
|
}
|
|
|
|
export function walk(dir, matchesFile, root = repoRoot, found = []) {
|
|
if (!fs.existsSync(dir)) return found;
|
|
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
|
|
if (ignoredDirs.has(entry.name) || entry.name.startsWith(".")) continue;
|
|
const full = path.join(dir, entry.name);
|
|
if (entry.isDirectory()) {
|
|
walk(full, matchesFile, root, found);
|
|
continue;
|
|
}
|
|
if (entry.isFile() && matchesFile(entry.name)) {
|
|
found.push(path.relative(root, full).split(path.sep).join("/"));
|
|
}
|
|
}
|
|
return found;
|
|
}
|
|
|
|
export function collectA11yCoverage(root = repoRoot) {
|
|
const componentRoot = path.join(root, "apps/web/app");
|
|
const e2eRoot = path.join(root, "tests/e2e");
|
|
const e2eSpecFiles = walk(e2eRoot, isE2eSpec, root).sort();
|
|
return {
|
|
componentFiles: walk(componentRoot, isComponentA11yTest, root).sort(),
|
|
e2eFiles: walk(e2eRoot, isE2eA11ySpec, root).sort(),
|
|
e2eSpecFiles,
|
|
e2eSpecContents: Object.fromEntries(e2eSpecFiles.map((file) => [file, fs.readFileSync(path.join(root, file), "utf8")])),
|
|
};
|
|
}
|
|
|
|
export function verifyA11yCoverage({
|
|
componentFiles,
|
|
e2eFiles,
|
|
e2eSpecFiles = e2eFiles,
|
|
e2eSpecContents = {},
|
|
minComponent = MIN_COMPONENT_A11Y_TESTS,
|
|
minE2e = MIN_E2E_A11Y_SPECS,
|
|
requiredE2eSpecs = REQUIRED_E2E_FLOW_SPECS,
|
|
requiredE2eAssertions = REQUIRED_E2E_FLOW_ASSERTIONS,
|
|
}) {
|
|
const failures = [];
|
|
if (componentFiles.length < minComponent) {
|
|
failures.push(`expected at least ${minComponent} component a11y test file(s), found ${componentFiles.length}`);
|
|
}
|
|
if (e2eFiles.length < minE2e) {
|
|
failures.push(`expected at least ${minE2e} browser a11y spec file(s), found ${e2eFiles.length}`);
|
|
}
|
|
const e2eSpecSet = new Set(e2eSpecFiles);
|
|
for (const [flow, file] of Object.entries(requiredE2eSpecs)) {
|
|
if (!e2eSpecSet.has(file)) {
|
|
failures.push(`required e2e flow "${flow}" must be covered by ${file}`);
|
|
continue;
|
|
}
|
|
const assertionContract = requiredE2eAssertions[flow];
|
|
if (!assertionContract) {
|
|
continue;
|
|
}
|
|
const content = e2eSpecContents[file] ?? "";
|
|
if (assertionContract.axe && !content.includes("expectNoCriticalA11yViolations(")) {
|
|
failures.push(`required e2e flow "${flow}" in ${file} must call expectNoCriticalA11yViolations`);
|
|
}
|
|
if (assertionContract.mobileOverflow && !content.includes("expectNoHorizontalOverflow(")) {
|
|
failures.push(`required e2e flow "${flow}" in ${file} must call expectNoHorizontalOverflow`);
|
|
}
|
|
for (const snippet of assertionContract.requiredSnippets ?? []) {
|
|
if (!content.includes(snippet)) {
|
|
failures.push(`required e2e flow "${flow}" in ${file} must include ${JSON.stringify(snippet)}`);
|
|
}
|
|
}
|
|
}
|
|
return { ok: failures.length === 0, failures };
|
|
}
|
|
|
|
export function main(root = repoRoot) {
|
|
const coverage = collectA11yCoverage(root);
|
|
for (const f of coverage.componentFiles) console.log(`component-a11y: ${f}`);
|
|
for (const f of coverage.e2eFiles) console.log(`e2e-a11y: ${f}`);
|
|
console.log(`Found ${coverage.componentFiles.length} component a11y test file(s) (min ${MIN_COMPONENT_A11Y_TESTS}).`);
|
|
console.log(`Found ${coverage.e2eFiles.length} browser a11y spec file(s) (min ${MIN_E2E_A11Y_SPECS}).`);
|
|
console.log(
|
|
`Required browser readiness/a11y specs: ${Object.entries(REQUIRED_E2E_FLOW_SPECS)
|
|
.map(([flow, file]) => `${flow}=${file}`)
|
|
.join(", ")}.`
|
|
);
|
|
const result = verifyA11yCoverage(coverage);
|
|
if (!result.ok) {
|
|
console.error(`FAIL: ${result.failures.join("; ")}.`);
|
|
process.exit(1);
|
|
}
|
|
}
|
|
|
|
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
|
|
main();
|
|
}
|