Files
HiveGrid-Wireframes/hivegrid-cloud/scripts/verify-a11y-coverage.test.mjs
T

397 lines
22 KiB
JavaScript

import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { afterEach, describe, expect, it } from "vitest";
import { collectA11yCoverage, REQUIRED_E2E_FLOW_ASSERTIONS, verifyA11yCoverage } from "./verify-a11y-coverage.mjs";
const tempRoots = [];
async function createFixtureRoot() {
const root = await mkdtemp(path.join(os.tmpdir(), "a11y-coverage-"));
tempRoots.push(root);
return root;
}
async function writeFixtureFile(root, relativePath) {
const absolutePath = path.join(root, relativePath);
await mkdir(path.dirname(absolutePath), { recursive: true });
await writeFile(absolutePath, "export {};\n", "utf8");
}
afterEach(async () => {
await Promise.all(tempRoots.splice(0).map((root) => rm(root, { recursive: true, force: true })));
});
describe("verify-a11y-coverage", () => {
it("counts component and browser-level a11y coverage separately", async () => {
const root = await createFixtureRoot();
await writeFixtureFile(root, "apps/web/app/components/Button.a11y.test.tsx");
await writeFixtureFile(root, "apps/web/app/components/Dialog.axe.test.tsx");
await writeFixtureFile(root, "tests/e2e/login.a11y.spec.ts");
await writeFixtureFile(root, "tests/e2e/workflow-engine.mobile-a11y.spec.ts");
expect(collectA11yCoverage(root)).toEqual({
componentFiles: ["apps/web/app/components/Button.a11y.test.tsx", "apps/web/app/components/Dialog.axe.test.tsx"],
e2eFiles: ["tests/e2e/login.a11y.spec.ts", "tests/e2e/workflow-engine.mobile-a11y.spec.ts"],
e2eSpecFiles: ["tests/e2e/login.a11y.spec.ts", "tests/e2e/workflow-engine.mobile-a11y.spec.ts"],
e2eSpecContents: {
"tests/e2e/login.a11y.spec.ts": "export {};\n",
"tests/e2e/workflow-engine.mobile-a11y.spec.ts": "export {};\n",
},
});
});
it("ignores generated, dependency, and non-a11y files", async () => {
const root = await createFixtureRoot();
await writeFixtureFile(root, "apps/web/app/components/Button.test.tsx");
await writeFixtureFile(root, "apps/web/app/node_modules/vendor/Card.a11y.test.tsx");
await writeFixtureFile(root, "apps/web/app/build/Button.a11y.test.tsx");
await writeFixtureFile(root, "tests/e2e/generated/login.a11y.spec.ts");
await writeFixtureFile(root, "tests/e2e/login.spec.ts");
expect(collectA11yCoverage(root)).toEqual({
componentFiles: [],
e2eFiles: [],
e2eSpecFiles: ["tests/e2e/login.spec.ts"],
e2eSpecContents: {
"tests/e2e/login.spec.ts": "export {};\n",
},
});
});
it("fails when either coverage class drops below its minimum", () => {
expect(verifyA11yCoverage({ componentFiles: ["component.a11y.test.tsx"], e2eFiles: [], minComponent: 2, minE2e: 1, requiredE2eSpecs: {} })).toEqual({
ok: false,
failures: ["expected at least 2 component a11y test file(s), found 1", "expected at least 1 browser a11y spec file(s), found 0"],
});
});
it("passes when both coverage classes meet their minima", () => {
expect(verifyA11yCoverage({ componentFiles: ["a", "b"], e2eFiles: ["c"], minComponent: 2, minE2e: 1, requiredE2eSpecs: {} })).toEqual({
ok: true,
failures: [],
});
});
it("requires named browser readiness specs in addition to raw counts", () => {
expect(
verifyA11yCoverage({
componentFiles: ["a", "b"],
e2eFiles: ["tests/e2e/registration-account-dashboard.mobile-a11y.spec.ts"],
e2eSpecFiles: ["tests/e2e/registration-account-dashboard.mobile-a11y.spec.ts", "tests/e2e/subscription-smoke.spec.ts"],
minComponent: 2,
minE2e: 1,
requiredE2eSpecs: {
"registration-account-dashboard": "tests/e2e/registration-account-dashboard.mobile-a11y.spec.ts",
"subscription-smoke": "tests/e2e/subscription-smoke.spec.ts",
"plugin-settings": "tests/e2e/plugin-settings.mobile-a11y.spec.ts",
},
requiredE2eAssertions: {},
})
).toEqual({
ok: false,
failures: ['required e2e flow "plugin-settings" must be covered by tests/e2e/plugin-settings.mobile-a11y.spec.ts'],
});
});
it("requires named browser readiness specs to contain expected a11y assertions", () => {
expect(
verifyA11yCoverage({
componentFiles: ["a", "b"],
e2eFiles: ["tests/e2e/public-auth.mobile-a11y.spec.ts"],
e2eSpecFiles: ["tests/e2e/public-auth.mobile-a11y.spec.ts"],
e2eSpecContents: {
"tests/e2e/public-auth.mobile-a11y.spec.ts": "await expectNoHorizontalOverflow(page);",
},
minComponent: 2,
minE2e: 1,
requiredE2eSpecs: {
"public-auth": "tests/e2e/public-auth.mobile-a11y.spec.ts",
},
})
).toEqual({
ok: false,
failures: ['required e2e flow "public-auth" in tests/e2e/public-auth.mobile-a11y.spec.ts must call expectNoCriticalA11yViolations'],
});
});
it("requires named browser readiness specs to contain flow-specific coverage snippets", () => {
expect(
verifyA11yCoverage({
componentFiles: ["a", "b"],
e2eFiles: ["tests/e2e/registration-account-dashboard.mobile-a11y.spec.ts"],
e2eSpecFiles: ["tests/e2e/registration-account-dashboard.mobile-a11y.spec.ts"],
e2eSpecContents: {
"tests/e2e/registration-account-dashboard.mobile-a11y.spec.ts": "await expectNoCriticalA11yViolations(page);\nawait expectNoHorizontalOverflow(page);\n",
},
minComponent: 2,
minE2e: 1,
requiredE2eSpecs: {
"registration-account-dashboard": "tests/e2e/registration-account-dashboard.mobile-a11y.spec.ts",
},
requiredE2eAssertions: {
"registration-account-dashboard": {
file: "tests/e2e/registration-account-dashboard.mobile-a11y.spec.ts",
axe: true,
mobileOverflow: true,
requiredSnippets: [
"E2E_ALLOW_NO_TENANT_ACCOUNT_CREATE",
"guards seeded tenant account membership before dashboard and account smoke",
"replays new-account login redirects and returns to the dashboard on mobile",
"can create a tenant account and reach its dashboard when explicitly enabled",
"can onboard a registered user with no tenant through account creation when explicitly enabled",
"zeroTenantAccountStatePattern",
"/login?redirectTo=",
"[data-ui-id=\"ui.account.create.form\"]",
"tenantSettingsSubscriptionPath(",
"[data-ui-id=\"ui.subscription.page\"]",
],
},
},
})
).toEqual({
ok: false,
failures: [
'required e2e flow "registration-account-dashboard" in tests/e2e/registration-account-dashboard.mobile-a11y.spec.ts must include "E2E_ALLOW_NO_TENANT_ACCOUNT_CREATE"',
'required e2e flow "registration-account-dashboard" in tests/e2e/registration-account-dashboard.mobile-a11y.spec.ts must include "guards seeded tenant account membership before dashboard and account smoke"',
'required e2e flow "registration-account-dashboard" in tests/e2e/registration-account-dashboard.mobile-a11y.spec.ts must include "replays new-account login redirects and returns to the dashboard on mobile"',
'required e2e flow "registration-account-dashboard" in tests/e2e/registration-account-dashboard.mobile-a11y.spec.ts must include "can create a tenant account and reach its dashboard when explicitly enabled"',
'required e2e flow "registration-account-dashboard" in tests/e2e/registration-account-dashboard.mobile-a11y.spec.ts must include "can onboard a registered user with no tenant through account creation when explicitly enabled"',
'required e2e flow "registration-account-dashboard" in tests/e2e/registration-account-dashboard.mobile-a11y.spec.ts must include "zeroTenantAccountStatePattern"',
'required e2e flow "registration-account-dashboard" in tests/e2e/registration-account-dashboard.mobile-a11y.spec.ts must include "/login?redirectTo="',
'required e2e flow "registration-account-dashboard" in tests/e2e/registration-account-dashboard.mobile-a11y.spec.ts must include "[data-ui-id=\\"ui.account.create.form\\"]"',
'required e2e flow "registration-account-dashboard" in tests/e2e/registration-account-dashboard.mobile-a11y.spec.ts must include "tenantSettingsSubscriptionPath("',
'required e2e flow "registration-account-dashboard" in tests/e2e/registration-account-dashboard.mobile-a11y.spec.ts must include "[data-ui-id=\\"ui.subscription.page\\"]"',
],
});
});
it("pins plugin lifecycle and workflow run coverage beyond file-level a11y checks", () => {
expect(
verifyA11yCoverage({
componentFiles: ["a", "b"],
e2eFiles: ["tests/e2e/plugin-settings.mobile-a11y.spec.ts", "tests/e2e/workflow-builder-run.mobile-a11y.spec.ts"],
e2eSpecFiles: ["tests/e2e/plugin-settings.mobile-a11y.spec.ts", "tests/e2e/workflow-builder-run.mobile-a11y.spec.ts"],
e2eSpecContents: {
"tests/e2e/plugin-settings.mobile-a11y.spec.ts": "await expectNoCriticalA11yViolations(page);\nawait expectNoHorizontalOverflow(page);\n",
"tests/e2e/workflow-builder-run.mobile-a11y.spec.ts": "await expectNoCriticalA11yViolations(page);\nawait expectNoHorizontalOverflow(page);\n",
},
minComponent: 2,
minE2e: 2,
requiredE2eSpecs: {
"plugin-settings": "tests/e2e/plugin-settings.mobile-a11y.spec.ts",
"workflow-builder-run": "tests/e2e/workflow-builder-run.mobile-a11y.spec.ts",
},
requiredE2eAssertions: {
"plugin-settings": {
file: "tests/e2e/plugin-settings.mobile-a11y.spec.ts",
axe: true,
mobileOverflow: true,
requiredSnippets: [
"lifecycle controls submit through a non-mutating interception contract",
"lifecycle controls persist enable and disable state for the seeded lifecycle plugin",
"app.plugins.use users cannot submit forged lifecycle mutations without manage permission",
"settings iframe persists values across reload when write slice is enabled",
"[data-ui-id=\"ui.plugins.lifecycle.form\"]",
"page.waitForURL(/\\/unauthorized\\/app\\.plugins\\.manage",
],
},
"workflow-builder-run": {
file: "tests/e2e/workflow-builder-run.mobile-a11y.spec.ts",
axe: true,
mobileOverflow: true,
requiredSnippets: [
"manual run contract stays usable on mobile",
"manual run result deep-links to execution details on mobile when explicitly enabled",
"manual run reaches expected execution status when outbox-drain readiness is required",
"E2E_REQUIRE_WORKFLOW_OUTBOX_DRAIN",
"[data-ui-id=\"ui.workflows.run.manual.form\"]",
"[data-ui-id=\"ui.workflows.run.manual.viewExecutionFlow\"]",
],
},
},
})
).toEqual({
ok: false,
failures: [
'required e2e flow "plugin-settings" in tests/e2e/plugin-settings.mobile-a11y.spec.ts must include "lifecycle controls submit through a non-mutating interception contract"',
'required e2e flow "plugin-settings" in tests/e2e/plugin-settings.mobile-a11y.spec.ts must include "lifecycle controls persist enable and disable state for the seeded lifecycle plugin"',
'required e2e flow "plugin-settings" in tests/e2e/plugin-settings.mobile-a11y.spec.ts must include "app.plugins.use users cannot submit forged lifecycle mutations without manage permission"',
'required e2e flow "plugin-settings" in tests/e2e/plugin-settings.mobile-a11y.spec.ts must include "settings iframe persists values across reload when write slice is enabled"',
'required e2e flow "plugin-settings" in tests/e2e/plugin-settings.mobile-a11y.spec.ts must include "[data-ui-id=\\"ui.plugins.lifecycle.form\\"]"',
'required e2e flow "plugin-settings" in tests/e2e/plugin-settings.mobile-a11y.spec.ts must include "page.waitForURL(/\\\\/unauthorized\\\\/app\\\\.plugins\\\\.manage"',
'required e2e flow "workflow-builder-run" in tests/e2e/workflow-builder-run.mobile-a11y.spec.ts must include "manual run contract stays usable on mobile"',
'required e2e flow "workflow-builder-run" in tests/e2e/workflow-builder-run.mobile-a11y.spec.ts must include "manual run result deep-links to execution details on mobile when explicitly enabled"',
'required e2e flow "workflow-builder-run" in tests/e2e/workflow-builder-run.mobile-a11y.spec.ts must include "manual run reaches expected execution status when outbox-drain readiness is required"',
'required e2e flow "workflow-builder-run" in tests/e2e/workflow-builder-run.mobile-a11y.spec.ts must include "E2E_REQUIRE_WORKFLOW_OUTBOX_DRAIN"',
'required e2e flow "workflow-builder-run" in tests/e2e/workflow-builder-run.mobile-a11y.spec.ts must include "[data-ui-id=\\"ui.workflows.run.manual.form\\"]"',
'required e2e flow "workflow-builder-run" in tests/e2e/workflow-builder-run.mobile-a11y.spec.ts must include "[data-ui-id=\\"ui.workflows.run.manual.viewExecutionFlow\\"]"',
],
});
});
it("pins readiness golden path coverage across subscription and workflow followthrough", () => {
expect(
verifyA11yCoverage({
componentFiles: ["a", "b"],
e2eFiles: ["tests/e2e/readiness-golden-path.mobile-a11y.spec.ts"],
e2eSpecFiles: ["tests/e2e/readiness-golden-path.mobile-a11y.spec.ts"],
e2eSpecContents: {
"tests/e2e/readiness-golden-path.mobile-a11y.spec.ts":
"await expectNoCriticalA11yViolations(page);\nawait expectNoHorizontalOverflow(page);\n",
},
minComponent: 2,
minE2e: 1,
requiredE2eSpecs: {
"readiness-golden-path": "tests/e2e/readiness-golden-path.mobile-a11y.spec.ts",
},
requiredE2eAssertions: {
"readiness-golden-path": {
file: "tests/e2e/readiness-golden-path.mobile-a11y.spec.ts",
axe: true,
mobileOverflow: true,
requiredSnippets: [
"navigates authenticated mobile readiness surfaces without mutations",
"failOnStripeNetworkRequests(",
"[data-ui-id=\"ui.subscription.checkout.page\"]",
"[data-ui-id=\"ui.plugins.page\"]",
"[data-ui-id=\"ui.workflows.builder.canvas\"]",
"[data-ui-id=\"ui.workflows.run.manual.form\"]",
],
},
},
})
).toEqual({
ok: false,
failures: [
'required e2e flow "readiness-golden-path" in tests/e2e/readiness-golden-path.mobile-a11y.spec.ts must include "navigates authenticated mobile readiness surfaces without mutations"',
'required e2e flow "readiness-golden-path" in tests/e2e/readiness-golden-path.mobile-a11y.spec.ts must include "failOnStripeNetworkRequests("',
'required e2e flow "readiness-golden-path" in tests/e2e/readiness-golden-path.mobile-a11y.spec.ts must include "[data-ui-id=\\"ui.subscription.checkout.page\\"]"',
'required e2e flow "readiness-golden-path" in tests/e2e/readiness-golden-path.mobile-a11y.spec.ts must include "[data-ui-id=\\"ui.plugins.page\\"]"',
'required e2e flow "readiness-golden-path" in tests/e2e/readiness-golden-path.mobile-a11y.spec.ts must include "[data-ui-id=\\"ui.workflows.builder.canvas\\"]"',
'required e2e flow "readiness-golden-path" in tests/e2e/readiness-golden-path.mobile-a11y.spec.ts must include "[data-ui-id=\\"ui.workflows.run.manual.form\\"]"',
],
});
});
it("pins authenticated mobile locale persistence to reload, profile shell, restore, and a11y checks", () => {
expect(
verifyA11yCoverage({
componentFiles: ["a", "b"],
e2eFiles: ["tests/e2e/authenticated-locale.mobile.spec.ts"],
e2eSpecFiles: ["tests/e2e/authenticated-locale.mobile.spec.ts"],
e2eSpecContents: {
"tests/e2e/authenticated-locale.mobile.spec.ts": "await expectNoHorizontalOverflow(page);\n",
},
minComponent: 2,
minE2e: 1,
requiredE2eSpecs: {
"authenticated-mobile-locale": "tests/e2e/authenticated-locale.mobile.spec.ts",
},
requiredE2eAssertions: {
"authenticated-mobile-locale": {
file: "tests/e2e/authenticated-locale.mobile.spec.ts",
axe: true,
mobileOverflow: true,
requiredSnippets: [
"persists an authenticated locale switch across mobile shell navigation",
"await page.reload({ waitUntil: \"domcontentloaded\" });",
"tenantSettingsProfilePath(dashboardPath)",
"restoreLocale(page, initialLocale)",
"[data-ui-id=\"ui.nav.app.dashboard\"]",
"[data-ui-id=\"ui.nav.app.settings\"]",
],
},
},
})
).toEqual({
ok: false,
failures: [
'required e2e flow "authenticated-mobile-locale" in tests/e2e/authenticated-locale.mobile.spec.ts must call expectNoCriticalA11yViolations',
'required e2e flow "authenticated-mobile-locale" in tests/e2e/authenticated-locale.mobile.spec.ts must include "persists an authenticated locale switch across mobile shell navigation"',
'required e2e flow "authenticated-mobile-locale" in tests/e2e/authenticated-locale.mobile.spec.ts must include "await page.reload({ waitUntil: \\"domcontentloaded\\" });"',
'required e2e flow "authenticated-mobile-locale" in tests/e2e/authenticated-locale.mobile.spec.ts must include "tenantSettingsProfilePath(dashboardPath)"',
'required e2e flow "authenticated-mobile-locale" in tests/e2e/authenticated-locale.mobile.spec.ts must include "restoreLocale(page, initialLocale)"',
'required e2e flow "authenticated-mobile-locale" in tests/e2e/authenticated-locale.mobile.spec.ts must include "[data-ui-id=\\"ui.nav.app.dashboard\\"]"',
'required e2e flow "authenticated-mobile-locale" in tests/e2e/authenticated-locale.mobile.spec.ts must include "[data-ui-id=\\"ui.nav.app.settings\\"]"',
],
});
});
it("pins seeded tenant control-plane mobile/a11y specs as named readiness flows", () => {
const flows = [
"tenant-api-keys",
"tenant-audit-logs",
"tenant-audit-export",
"privacy-self-service",
"profile-security",
"tenant-members",
"tenant-notifications-settings",
"tenant-chatbot-settings",
"tenant-ai-usage",
];
const requiredE2eAssertions = Object.fromEntries(flows.map((flow) => [flow, REQUIRED_E2E_FLOW_ASSERTIONS[flow]]));
const requiredE2eSpecs = Object.fromEntries(flows.map((flow) => [flow, REQUIRED_E2E_FLOW_ASSERTIONS[flow].file]));
const files = Object.values(requiredE2eSpecs);
const contentsWithoutFlowContracts = Object.fromEntries(
files.map((file) => [file, "await expectNoCriticalA11yViolations(page);\nawait expectNoHorizontalOverflow(page);\n"]),
);
for (const flow of flows) {
expect(REQUIRED_E2E_FLOW_ASSERTIONS[flow]).toMatchObject({
file: expect.stringMatching(/^tests\/e2e\/.+\.spec\.ts$/u),
axe: true,
mobileOverflow: true,
});
expect(REQUIRED_E2E_FLOW_ASSERTIONS[flow].requiredSnippets.length).toBeGreaterThan(0);
}
const result = verifyA11yCoverage({
componentFiles: ["a", "b"],
e2eFiles: files,
e2eSpecFiles: files,
e2eSpecContents: contentsWithoutFlowContracts,
minComponent: 2,
minE2e: files.length,
requiredE2eSpecs,
requiredE2eAssertions,
});
expect(result.ok).toBe(false);
expect(result.failures).toEqual(
expect.arrayContaining([
'required e2e flow "tenant-api-keys" in tests/e2e/tenant-api-keys.mobile-a11y.spec.ts must include "opens create form without mobile overflow or critical a11y violations"',
'required e2e flow "tenant-audit-logs" in tests/e2e/tenant-audit-logs.mobile-a11y.spec.ts must include "keeps audit logs usable at"',
'required e2e flow "tenant-audit-export" in tests/e2e/tenant-audit-export.mobile-a11y.spec.ts must include "keeps compliance export filters usable at"',
'required e2e flow "privacy-self-service" in tests/e2e/privacy-self-service.mobile-a11y.spec.ts must include "renders export and deletion controls without mobile overflow or critical a11y violations"',
'required e2e flow "profile-security" in tests/e2e/profile-security.mobile-a11y.spec.ts must include "renders security, MFA, session, and danger controls without mobile overflow or critical a11y violations"',
'required e2e flow "tenant-members" in tests/e2e/tenant-members.mobile-a11y.spec.ts must include "renders members table at"',
'required e2e flow "tenant-notifications-settings" in tests/e2e/tenant-notifications-settings.mobile-a11y.spec.ts must include "renders notification categories without mobile overflow or critical a11y violations"',
'required e2e flow "tenant-chatbot-settings" in tests/e2e/tenant-chatbot-settings.mobile-a11y.spec.ts must include "renders access controls without mobile overflow or critical a11y violations"',
'required e2e flow "tenant-ai-usage" in tests/e2e/ai-usage.mobile-a11y.spec.ts must include "renders quota and forecast without mobile overflow or critical a11y violations"',
]),
);
});
it("pins public mobile overflow and a11y coverage as a named readiness flow", () => {
const flow = REQUIRED_E2E_FLOW_ASSERTIONS["mobile-public"];
expect(flow).toMatchObject({
file: "tests/e2e/mobile-public.spec.ts",
axe: true,
mobileOverflow: true,
});
expect(flow.requiredSnippets).toEqual(
expect.arrayContaining([
"mobile public pages",
"compact mobile public pages",
"feature-disabled page has no mobile overflow or critical a11y violations",
"public footer selectors stay inside a 320px viewport",
"compactMobileViewport",
"\"/privacy-policy\"",
])
);
});
});