HiveGrid-Wireframes/hivegrid-cloud/scripts/verify-a11y-baseline.mjs

#!/usr/bin/env node
import { spawnSync } from "node:child_process";
import fs from "node:fs";
import path from "node:path";

const BASELINE_PATH = ".eslintrc.a11y-baseline.json";
const DEFAULT_SCOPE = ["apps/web/app/**/*.tsx", "packages/ui/src/stories/**/*.tsx"];

function readBaseline(rootDir) {
  return JSON.parse(fs.readFileSync(path.join(rootDir, BASELINE_PATH), "utf8"));
}

function baselineScope(baseline) {
  if (Array.isArray(baseline.scope)) return baseline.scope;
  if (typeof baseline.scope === "string" && baseline.scope.trim().length > 0) return [baseline.scope];
  return DEFAULT_SCOPE;
}

function baselineRules(baseline) {
  return Object.keys(baseline.rules ?? {}).sort();
}

function runEslint(rootDir, scope, rules) {
  const ruleArgs = rules.flatMap((rule) => ["--rule", `${rule}:error`]);
  const result = spawnSync("pnpm", ["exec", "eslint", "--format", "json", "--no-ignore", "--ext", ".tsx", ...ruleArgs, ...scope], {
    cwd: rootDir,
    encoding: "utf8",
    maxBuffer: 1024 * 1024 * 20,
  });

  if (result.error) {
    throw result.error;
  }

  if (![0, 1].includes(result.status)) {
    throw new Error(result.stderr || `eslint exited with status ${result.status}`);
  }

  try {
    return JSON.parse(result.stdout || "[]");
  } catch (error) {
    throw new Error(`failed to parse ESLint JSON output: ${error.message}`);
  }
}

function countRuleMessages(eslintResults, rules) {
  const counts = Object.fromEntries(rules.map((rule) => [rule, 0]));
  const examples = Object.fromEntries(rules.map((rule) => [rule, []]));

  for (const result of eslintResults) {
    for (const message of result.messages ?? []) {
      if (!rules.includes(message.ruleId)) continue;
      counts[message.ruleId] += 1;
      if (examples[message.ruleId].length < 3) {
        examples[message.ruleId].push(`${path.relative(process.cwd(), result.filePath)}:${message.line}:${message.column}`);
      }
    }
  }

  return { counts, examples };
}

export function verifyA11yBaseline(rootDir = process.cwd()) {
  const baseline = readBaseline(rootDir);
  const scope = baselineScope(baseline);
  const rules = baselineRules(baseline);
  const eslintResults = runEslint(rootDir, scope, rules);
  const { counts, examples } = countRuleMessages(eslintResults, rules);
  const failures = [];
  const observations = [];
  const passes = [];

  for (const rule of rules) {
    const expected = Number(baseline.rules[rule]?.violations ?? 0);
    const actual = counts[rule] ?? 0;
    if (actual > expected) {
      failures.push(`${rule}: ${actual} violation(s), baseline allows ${expected}`);
      continue;
    }
    if (actual < expected) {
      observations.push(`${rule}: ${actual} violation(s), below baseline ${expected}; update ${BASELINE_PATH} when ratcheting`);
    } else {
      passes.push(`${rule}: ${actual} violation(s) matches baseline`);
    }
  }

  const totalExpected = Number(baseline.totalViolations ?? 0);
  const totalActual = Object.values(counts).reduce((sum, count) => sum + count, 0);
  if (totalActual > totalExpected) {
    failures.push(`totalViolations: ${totalActual}, baseline allows ${totalExpected}`);
  } else if (totalActual < totalExpected) {
    observations.push(`totalViolations: ${totalActual}, below baseline ${totalExpected}; update ${BASELINE_PATH} when ratcheting`);
  } else {
    passes.push(`totalViolations: ${totalActual} matches baseline`);
  }

  return { passes, observations, failures, counts, examples, scope };
}

export function formatA11yBaselineResult(result) {
  const status = result.failures.length > 0 ? "FAIL" : "PASS";
  const lines = [
    `${status} accessibility baseline verification`,
    `Scope: ${result.scope.join(", ")}`,
    ...Object.entries(result.counts).map(([rule, count]) => `COUNT ${rule}: ${count}`),
    ...result.passes.map((pass) => `PASS ${pass}`),
    ...result.observations.map((observation) => `OBSERVE ${observation}`),
    ...result.failures.map((failure) => `FAIL ${failure}`),
  ];

  if (result.failures.length > 0) {
    for (const [rule, examples] of Object.entries(result.examples)) {
      if (examples.length > 0) {
        lines.push(`EXAMPLE ${rule}: ${examples.join(", ")}`);
      }
    }
  }

  lines.push(`Summary: ${result.passes.length} passed, ${result.observations.length} observed, ${result.failures.length} failed`, "");
  return lines.join("\n");
}

const isCli = process.argv[1] && path.resolve(process.argv[1]) === new URL(import.meta.url).pathname;

if (isCli) {
  try {
    const result = verifyA11yBaseline(process.cwd());
    const output = formatA11yBaselineResult(result);
    if (result.failures.length > 0) {
      process.stderr.write(output);
      process.exitCode = 1;
    } else {
      process.stdout.write(output);
    }
  } catch (error) {
    process.stderr.write(`FAIL accessibility baseline verification\n${error.message}\n`);
    process.exitCode = 1;
  }
}