453 lines
18 KiB
JavaScript
453 lines
18 KiB
JavaScript
#!/usr/bin/env node
|
|
// Anti-regression gate for the static accessibility test suite.
|
|
//
|
|
// We enforce a minimum number of `*.a11y.test.tsx` / `*.axe.test.tsx` files
|
|
// in the web app.
|
|
// Removing a11y coverage is a deliberate act and must show up in code review.
|
|
//
|
|
// Pair with `scripts/verify-test-density.mjs` — that script enforces a
|
|
// per-workspace ratio of test files to source files but does not care
|
|
// about test type. This script keeps the a11y class of tests from
|
|
// silently dropping to zero.
|
|
|
|
import fs from "node:fs";
|
|
import path from "node:path";
|
|
import { pathToFileURL } from "node:url";
|
|
import { fileURLToPath } from "node:url";
|
|
|
|
export const repoRoot = path.resolve(fileURLToPath(import.meta.url), "..", "..");
|
|
export const componentSearchRoot = path.join(repoRoot, "apps/web/app");
|
|
export const e2eSearchRoot = path.join(repoRoot, "tests/e2e");
|
|
export const MIN_COMPONENT_A11Y_TESTS = 12;
|
|
export const MIN_E2E_A11Y_SPECS = 7;
|
|
export const REQUIRED_E2E_FLOW_ASSERTIONS = {
|
|
"registration-account-dashboard": {
|
|
file: "tests/e2e/registration-account-dashboard.mobile-a11y.spec.ts",
|
|
axe: true,
|
|
mobileOverflow: true,
|
|
requiredSnippets: [
|
|
"E2E_ALLOW_REGISTRATION_CREATE",
|
|
"E2E_ALLOW_ACCOUNT_CREATE",
|
|
"E2E_ALLOW_NO_TENANT_ACCOUNT_CREATE",
|
|
"guards seeded tenant account membership before dashboard and account smoke",
|
|
"replays new-account login redirects and returns to the dashboard on mobile",
|
|
"can create a tenant account and reach its dashboard when explicitly enabled",
|
|
"can onboard a registered user with no tenant through account creation when explicitly enabled",
|
|
"can register a new user and stop at the email verification state when explicitly enabled",
|
|
"E2E_REGISTRATION_EXPECTS_EMAIL_VERIFICATION",
|
|
"zeroTenantAccountStatePattern",
|
|
"/login?redirectTo=",
|
|
"[data-ui-id=\"ui.account.create.form\"]",
|
|
"failOnStripeNetworkRequests(",
|
|
"tenantSettingsSubscriptionPath(",
|
|
"[data-ui-id=\"ui.subscription.page\"]",
|
|
"[data-ui-id=\"ui.subscription.billing.portal\"]",
|
|
],
|
|
},
|
|
"plugin-settings": {
|
|
file: "tests/e2e/plugin-settings.mobile-a11y.spec.ts",
|
|
axe: true,
|
|
mobileOverflow: true,
|
|
requiredSnippets: [
|
|
"lifecycle controls submit through a non-mutating interception contract",
|
|
"lifecycle errors stay announced and keep controls retryable",
|
|
"lifecycle controls persist enable and disable state for the seeded lifecycle plugin",
|
|
"app.plugins.use users cannot submit forged lifecycle mutations without manage permission",
|
|
"settings iframe exposes sandbox and CSP contract when available",
|
|
"settings iframe persists values across reload when write slice is enabled",
|
|
"[data-ui-id=\"ui.plugins.lifecycle.form\"]",
|
|
"page.waitForURL(/\\/unauthorized\\/app\\.plugins\\.manage",
|
|
"[data-ui-id=\"ui.plugins.settings.frame\"]",
|
|
],
|
|
},
|
|
"tenant-api-docs-logs": {
|
|
file: "tests/e2e/tenant-api-docs-logs.mobile-a11y.spec.ts",
|
|
axe: true,
|
|
mobileOverflow: true,
|
|
},
|
|
"tenant-api-keys": {
|
|
file: "tests/e2e/tenant-api-keys.mobile-a11y.spec.ts",
|
|
axe: true,
|
|
mobileOverflow: true,
|
|
requiredSnippets: [
|
|
"opens create form without mobile overflow or critical a11y violations",
|
|
"submits create contract with alias, entity permissions, and csrf without mutating data",
|
|
"E2E_TENANT_API_KEYS_PATH",
|
|
"interceptCurrentRoutePost(",
|
|
'form:has(input[name="action"][value="create"])',
|
|
'input[name="alias"]',
|
|
'input[name="entities[]"]',
|
|
],
|
|
},
|
|
"tenant-audit-logs": {
|
|
file: "tests/e2e/tenant-audit-logs.mobile-a11y.spec.ts",
|
|
axe: true,
|
|
mobileOverflow: true,
|
|
requiredSnippets: [
|
|
"keeps audit logs usable at",
|
|
"keeps audit event filters and table reachable on mobile",
|
|
"compactMobileViewport",
|
|
"[data-ui-id=\"ui.settings.logs.page\"]",
|
|
"[data-ui-id=\"ui.settings.logs.filters\"]",
|
|
"[data-ui-id=\"ui.settings.logs.table\"]",
|
|
"[data-ui-id=\"ui.settings.logs.events.table\"]",
|
|
],
|
|
},
|
|
"tenant-audit-export": {
|
|
file: "tests/e2e/tenant-audit-export.mobile-a11y.spec.ts",
|
|
axe: true,
|
|
mobileOverflow: true,
|
|
requiredSnippets: [
|
|
"keeps compliance export filters usable at",
|
|
"compactMobileViewport",
|
|
"[data-ui-id=\"ui.settings.auditExport.page\"]",
|
|
"[data-ui-id=\"ui.settings.auditExport.form\"]",
|
|
"[data-ui-id=\"ui.settings.auditExport.downloadCsv\"]",
|
|
"[data-ui-id=\"ui.settings.auditExport.downloadPdf\"]",
|
|
],
|
|
},
|
|
"privacy-self-service": {
|
|
file: "tests/e2e/privacy-self-service.mobile-a11y.spec.ts",
|
|
axe: true,
|
|
mobileOverflow: true,
|
|
requiredSnippets: [
|
|
"renders export and deletion controls without mobile overflow or critical a11y violations",
|
|
"submits export request with intent and csrf without mutating data",
|
|
"submits deletion request with confirmation, intent and csrf without mutating data",
|
|
"interceptCurrentRoutePost(",
|
|
"[data-ui-id=\"ui.settings.privacy.page\"]",
|
|
"[data-ui-id=\"ui.settings.privacy.export.form\"]",
|
|
"[data-ui-id=\"ui.settings.privacy.delete.form\"]",
|
|
],
|
|
},
|
|
"profile-security": {
|
|
file: "tests/e2e/profile-security.mobile-a11y.spec.ts",
|
|
axe: true,
|
|
mobileOverflow: true,
|
|
requiredSnippets: [
|
|
"renders security, MFA, session, and danger controls without mobile overflow or critical a11y violations",
|
|
"submits password change contract with csrf without mutating data",
|
|
"submits MFA toggle contract with csrf without changing enrollment state",
|
|
"E2E_TENANT_SETTINGS_PROFILE_PATH",
|
|
"interceptCurrentRoutePost(",
|
|
"[data-ui-id=\"ui.settings.profile.page\"]",
|
|
"[data-ui-id=\"ui.settings.profile.mfa.panel\"]",
|
|
"[data-ui-id=\"ui.settings.profile.password.form\"]",
|
|
"[data-ui-id=\"ui.settings.profile.mfa.toggle.form\"]",
|
|
"[data-ui-id=\"ui.settings.profile.session.panel\"]",
|
|
"[data-ui-id=\"ui.settings.profile.danger.delete\"]",
|
|
],
|
|
},
|
|
"tenant-members": {
|
|
file: "tests/e2e/tenant-members.mobile-a11y.spec.ts",
|
|
axe: true,
|
|
mobileOverflow: true,
|
|
requiredSnippets: [
|
|
"renders members table at",
|
|
"submits new member invite contract with csrf without mutating data",
|
|
"compactMobileViewport",
|
|
"interceptCurrentRoutePost(",
|
|
"[data-ui-id=\"ui.settings.members.page\"]",
|
|
"[data-ui-id=\"ui.settings.members.table\"]",
|
|
"[data-ui-id=\"ui.settings.members.new.form\"]",
|
|
],
|
|
},
|
|
"tenant-notifications-settings": {
|
|
file: "tests/e2e/tenant-notifications-settings.mobile-a11y.spec.ts",
|
|
axe: true,
|
|
mobileOverflow: true,
|
|
requiredSnippets: [
|
|
"renders notification categories without mobile overflow or critical a11y violations",
|
|
"submits preference update contract with csrf without mutating data",
|
|
"interceptCurrentRoutePost(",
|
|
"[data-ui-id=\"ui.settings.notifications.page\"]",
|
|
"ui.settings.notifications.category.${category}",
|
|
"[data-ui-id=\"ui.settings.notifications.preference.team.email\"]",
|
|
],
|
|
},
|
|
"tenant-chatbot-settings": {
|
|
file: "tests/e2e/tenant-chatbot-settings.mobile-a11y.spec.ts",
|
|
axe: true,
|
|
mobileOverflow: true,
|
|
requiredSnippets: [
|
|
"renders access controls without mobile overflow or critical a11y violations",
|
|
"submits tenant default access contract with csrf without mutating data",
|
|
"interceptCurrentRoutePost(",
|
|
"[data-ui-id=\"ui.settings.chatbot.page\"]",
|
|
"[data-ui-id=\"ui.settings.chatbot.master.form\"]",
|
|
"[data-ui-id=\"ui.settings.chatbot.defaults.form\"]",
|
|
"[data-ui-id=\"ui.settings.chatbot.providers\"]",
|
|
],
|
|
},
|
|
"tenant-ai-usage": {
|
|
file: "tests/e2e/ai-usage.mobile-a11y.spec.ts",
|
|
axe: true,
|
|
mobileOverflow: true,
|
|
requiredSnippets: [
|
|
"renders quota and forecast without mobile overflow or critical a11y violations",
|
|
"deriveAiUsagePath(",
|
|
"getByRole(\"progressbar\"",
|
|
"getByRole(\"heading\"",
|
|
"/settings/ai-usage",
|
|
],
|
|
},
|
|
"workflow-builder-run": {
|
|
file: "tests/e2e/workflow-builder-run.mobile-a11y.spec.ts",
|
|
axe: true,
|
|
mobileOverflow: true,
|
|
requiredSnippets: [
|
|
"builder stays usable on mobile",
|
|
"run dropdown stays keyboard reachable without starting execution",
|
|
"manual run contract stays usable on mobile",
|
|
"stream run waiting-block continuation stays usable on mobile",
|
|
"manual run result deep-links to execution details on mobile when explicitly enabled",
|
|
"manual run reaches expected execution status when outbox-drain readiness is required",
|
|
"E2E_ALLOW_WORKFLOW_RUN",
|
|
"requireWorkflowOutboxDrainReadiness(",
|
|
"waitForWorkflowExecutionTerminal(",
|
|
"[data-ui-id=\"ui.workflows.builder.canvas\"]",
|
|
"[data-ui-id=\"ui.workflows.run.manual.form\"]",
|
|
"[data-ui-id=\"ui.workflows.run.stream.progress\"]",
|
|
"[data-ui-id=\"ui.workflows.run.manual.viewExecutionFlow\"]",
|
|
],
|
|
},
|
|
"admin-email-sequences": {
|
|
file: "tests/e2e/admin-email-sequences.mobile-a11y.spec.ts",
|
|
axe: true,
|
|
mobileOverflow: true,
|
|
requiredSnippets: [
|
|
"renders seeded sequence list, detail, and stats surfaces on mobile",
|
|
"toggle lifecycle form posts intent without mutating the seeded sequence",
|
|
"E2E_ADMIN_EMAIL_SEQUENCES_PATH",
|
|
"seq_e2e_customer_welcome",
|
|
"lead.converted_to_customer",
|
|
"interceptCurrentRoutePost(",
|
|
"postDataIncludesField(submittedPostData.value, \"intent\", /^toggle$/u)",
|
|
],
|
|
},
|
|
"portal-management": {
|
|
file: "tests/e2e/portal-management.mobile-a11y.spec.ts",
|
|
axe: true,
|
|
mobileOverflow: true,
|
|
requiredSnippets: [
|
|
"portal management pages have no mobile overflow or critical a11y violations",
|
|
"E2E_TENANT_PORTAL_ANALYTICS_PATH",
|
|
"E2E_TENANT_PORTAL_SETTINGS_PATH",
|
|
"portalManagementPaths",
|
|
"skipOrFailWhenE2eFixtureMissing(",
|
|
],
|
|
},
|
|
"readiness-golden-path": {
|
|
file: "tests/e2e/readiness-golden-path.mobile-a11y.spec.ts",
|
|
axe: true,
|
|
mobileOverflow: true,
|
|
requiredSnippets: [
|
|
"navigates authenticated mobile readiness surfaces without mutations",
|
|
"failOnStripeNetworkRequests(",
|
|
"[data-ui-id=\"ui.subscription.checkout.page\"]",
|
|
"[data-ui-id=\"ui.plugins.page\"]",
|
|
"[data-ui-id=\"ui.workflows.builder.canvas\"]",
|
|
"[data-ui-id=\"ui.workflows.run.manual.form\"]",
|
|
],
|
|
},
|
|
"subscription-smoke": {
|
|
file: "tests/e2e/subscription-smoke.spec.ts",
|
|
axe: true,
|
|
mobileOverflow: true,
|
|
requiredSnippets: [
|
|
"[data-ui-id=\"ui.subscription.checkout.submit\"]",
|
|
"[data-ui-id=\"ui.subscription.checkout.success.page\"]",
|
|
"[data-ui-id=\"ui.subscription.billing.portal\"]",
|
|
"[data-ui-id=\"ui.subscription.product.cancel.dialog\"]",
|
|
"submits cancellation action contract with CSRF when cancellation controls are rendered",
|
|
"opens cancellation confirmation by keyboard and dismisses it with Escape without posting",
|
|
"reconciles checkout success state so it persists across reload and re-login",
|
|
"failOnStripeNetworkRequests(",
|
|
],
|
|
},
|
|
"mobile-authenticated-shell": {
|
|
file: "tests/e2e/mobile-authenticated-shell.spec.ts",
|
|
axe: true,
|
|
mobileOverflow: true,
|
|
requiredSnippets: [
|
|
"traps keyboard focus inside the mobile sidebar and returns focus to the trigger",
|
|
"mobile command palette focuses search and closes with Escape",
|
|
"mobile more menu exposes profile and sign-out contracts without submitting",
|
|
"skip link moves keyboard focus to the main app region",
|
|
"signs out from the mobile more menu and protects the dashboard",
|
|
"[data-ui-id=\"ui.action.shell.sidebarTrigger\"]",
|
|
"[data-ui-id=\"ui.surface.commandPalette.portal\"]",
|
|
"skip-to-content",
|
|
],
|
|
},
|
|
"authenticated-mobile-locale": {
|
|
file: "tests/e2e/authenticated-locale.mobile.spec.ts",
|
|
axe: true,
|
|
mobileOverflow: true,
|
|
requiredSnippets: [
|
|
"persists an authenticated locale switch across mobile shell navigation",
|
|
"await page.reload({ waitUntil: \"domcontentloaded\" });",
|
|
"tenantSettingsProfilePath(dashboardPath)",
|
|
"restoreLocale(page, initialLocale)",
|
|
"[data-ui-id=\"ui.nav.app.dashboard\"]",
|
|
"[data-ui-id=\"ui.nav.app.settings\"]",
|
|
],
|
|
},
|
|
"public-auth": {
|
|
file: "tests/e2e/public-auth.mobile-a11y.spec.ts",
|
|
axe: true,
|
|
mobileOverflow: true,
|
|
},
|
|
"auth-locale": {
|
|
file: "tests/e2e/auth-locale.mobile-a11y.spec.ts",
|
|
axe: true,
|
|
mobileOverflow: true,
|
|
},
|
|
"mobile-public": {
|
|
file: "tests/e2e/mobile-public.spec.ts",
|
|
axe: true,
|
|
mobileOverflow: true,
|
|
requiredSnippets: [
|
|
"mobile public pages",
|
|
"compact mobile public pages",
|
|
"landing page fits a narrow mobile viewport",
|
|
"not-found page fits a narrow mobile viewport",
|
|
"feature-disabled page has no mobile overflow or critical a11y violations",
|
|
"public footer selectors open without mobile overflow",
|
|
"fits a 320px mobile viewport",
|
|
"feature-disabled page handles long labels on compact mobile",
|
|
"public footer selectors stay inside a 320px viewport",
|
|
"compactMobileViewport",
|
|
"\"/privacy-policy\"",
|
|
"\"/terms-and-conditions\"",
|
|
"\"/401\"",
|
|
"\"/403\"",
|
|
"\"/500\"",
|
|
],
|
|
},
|
|
"tenant-governance-settings": {
|
|
file: "tests/e2e/tenant-governance-settings.mobile-a11y.spec.ts",
|
|
axe: true,
|
|
mobileOverflow: true,
|
|
},
|
|
"tenant-roles-permissions-mobile": {
|
|
file: "tests/e2e/tenant-roles-permissions.mobile-a11y.spec.ts",
|
|
axe: true,
|
|
mobileOverflow: true,
|
|
},
|
|
};
|
|
export const REQUIRED_E2E_FLOW_SPECS = Object.fromEntries(Object.entries(REQUIRED_E2E_FLOW_ASSERTIONS).map(([flow, contract]) => [flow, contract.file]));
|
|
const ignoredDirs = new Set(["node_modules", "dist", "build", "coverage", ".react-router", "generated", "__generated__"]);
|
|
|
|
function isComponentA11yTest(name) {
|
|
return name.endsWith(".a11y.test.tsx") || name.endsWith(".axe.test.tsx");
|
|
}
|
|
|
|
function isE2eA11ySpec(name) {
|
|
return (
|
|
name.endsWith(".a11y.spec.ts") ||
|
|
name.endsWith(".a11y.spec.tsx") ||
|
|
name.endsWith(".mobile-a11y.spec.ts") ||
|
|
name.endsWith(".mobile-a11y.spec.tsx") ||
|
|
name.endsWith(".desktop-a11y.spec.ts") ||
|
|
name.endsWith(".desktop-a11y.spec.tsx")
|
|
);
|
|
}
|
|
|
|
function isE2eSpec(name) {
|
|
return name.endsWith(".spec.ts") || name.endsWith(".spec.tsx");
|
|
}
|
|
|
|
export function walk(dir, matchesFile, root = repoRoot, found = []) {
|
|
if (!fs.existsSync(dir)) return found;
|
|
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
|
|
if (ignoredDirs.has(entry.name) || entry.name.startsWith(".")) continue;
|
|
const full = path.join(dir, entry.name);
|
|
if (entry.isDirectory()) {
|
|
walk(full, matchesFile, root, found);
|
|
continue;
|
|
}
|
|
if (entry.isFile() && matchesFile(entry.name)) {
|
|
found.push(path.relative(root, full));
|
|
}
|
|
}
|
|
return found;
|
|
}
|
|
|
|
export function collectA11yCoverage(root = repoRoot) {
|
|
const componentRoot = path.join(root, "apps/web/app");
|
|
const e2eRoot = path.join(root, "tests/e2e");
|
|
const e2eSpecFiles = walk(e2eRoot, isE2eSpec, root).sort();
|
|
return {
|
|
componentFiles: walk(componentRoot, isComponentA11yTest, root).sort(),
|
|
e2eFiles: walk(e2eRoot, isE2eA11ySpec, root).sort(),
|
|
e2eSpecFiles,
|
|
e2eSpecContents: Object.fromEntries(e2eSpecFiles.map((file) => [file, fs.readFileSync(path.join(root, file), "utf8")])),
|
|
};
|
|
}
|
|
|
|
export function verifyA11yCoverage({
|
|
componentFiles,
|
|
e2eFiles,
|
|
e2eSpecFiles = e2eFiles,
|
|
e2eSpecContents = {},
|
|
minComponent = MIN_COMPONENT_A11Y_TESTS,
|
|
minE2e = MIN_E2E_A11Y_SPECS,
|
|
requiredE2eSpecs = REQUIRED_E2E_FLOW_SPECS,
|
|
requiredE2eAssertions = REQUIRED_E2E_FLOW_ASSERTIONS,
|
|
}) {
|
|
const failures = [];
|
|
if (componentFiles.length < minComponent) {
|
|
failures.push(`expected at least ${minComponent} component a11y test file(s), found ${componentFiles.length}`);
|
|
}
|
|
if (e2eFiles.length < minE2e) {
|
|
failures.push(`expected at least ${minE2e} browser a11y spec file(s), found ${e2eFiles.length}`);
|
|
}
|
|
const e2eSpecSet = new Set(e2eSpecFiles);
|
|
for (const [flow, file] of Object.entries(requiredE2eSpecs)) {
|
|
if (!e2eSpecSet.has(file)) {
|
|
failures.push(`required e2e flow "${flow}" must be covered by ${file}`);
|
|
continue;
|
|
}
|
|
const assertionContract = requiredE2eAssertions[flow];
|
|
if (!assertionContract) {
|
|
continue;
|
|
}
|
|
const content = e2eSpecContents[file] ?? "";
|
|
if (assertionContract.axe && !content.includes("expectNoCriticalA11yViolations(")) {
|
|
failures.push(`required e2e flow "${flow}" in ${file} must call expectNoCriticalA11yViolations`);
|
|
}
|
|
if (assertionContract.mobileOverflow && !content.includes("expectNoHorizontalOverflow(")) {
|
|
failures.push(`required e2e flow "${flow}" in ${file} must call expectNoHorizontalOverflow`);
|
|
}
|
|
for (const snippet of assertionContract.requiredSnippets ?? []) {
|
|
if (!content.includes(snippet)) {
|
|
failures.push(`required e2e flow "${flow}" in ${file} must include ${JSON.stringify(snippet)}`);
|
|
}
|
|
}
|
|
}
|
|
return { ok: failures.length === 0, failures };
|
|
}
|
|
|
|
export function main(root = repoRoot) {
|
|
const coverage = collectA11yCoverage(root);
|
|
for (const f of coverage.componentFiles) console.log(`component-a11y: ${f}`);
|
|
for (const f of coverage.e2eFiles) console.log(`e2e-a11y: ${f}`);
|
|
console.log(`Found ${coverage.componentFiles.length} component a11y test file(s) (min ${MIN_COMPONENT_A11Y_TESTS}).`);
|
|
console.log(`Found ${coverage.e2eFiles.length} browser a11y spec file(s) (min ${MIN_E2E_A11Y_SPECS}).`);
|
|
console.log(
|
|
`Required browser readiness/a11y specs: ${Object.entries(REQUIRED_E2E_FLOW_SPECS)
|
|
.map(([flow, file]) => `${flow}=${file}`)
|
|
.join(", ")}.`
|
|
);
|
|
const result = verifyA11yCoverage(coverage);
|
|
if (!result.ok) {
|
|
console.error(`FAIL: ${result.failures.join("; ")}.`);
|
|
process.exit(1);
|
|
}
|
|
}
|
|
|
|
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
|
|
main();
|
|
}
|