74 lines
12 KiB
JavaScript
74 lines
12 KiB
JavaScript
#!/usr/bin/env node
|
|
/* eslint-disable no-console */
|
|
import { spawn } from "node:child_process";
|
|
import { createHash } from "node:crypto";
|
|
import { mkdir, readdir, readFile, stat, writeFile } from "node:fs/promises";
|
|
import path from "node:path";
|
|
import { fileURLToPath } from "node:url";
|
|
|
|
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
const repoRoot = path.resolve(__dirname, "..");
|
|
export const reportPath = path.join(repoRoot, ".tc/baselines/settings-api-onboarding/baseline-health.md");
|
|
|
|
const ignoredDirectoryNames = new Set([".git", ".cache", ".next", ".react-router", ".tc", ".turbo", ".vite", "build", "cache", "coverage", "dist", "generated", "node_modules"]);
|
|
const ignoredFileNames = new Set(["package-lock.json", "pnpm-lock.yaml", "yarn.lock"]);
|
|
export const settingsApiOnboardingInventoryScanRoots = ["apps/web/app/routes", "apps/web/app/components", "apps/web/app/utils", "packages/db/src", "packages/entitlements/src", "prisma"];
|
|
export const settingsApiOnboardingInventoryPatterns = [
|
|
{ id: "workspace-settings", pattern: /\b(?:workspace\s*settings|workspaceSettings|tenant\s*settings|tenantSettings|settings|preferences|configuration|config)\b/gi },
|
|
{ id: "user-account-settings", pattern: /\b(?:user\s*account|account\s*settings|profile\s*settings|userSettings|profile|password|mfa|totp|session|impersonat(?:e|ion))\b/gi },
|
|
{ id: "api-keys-scopes", pattern: /\b(?:api\s*keys?|apiKey|API key|scope|scopes|permission|permissions|token|secret|hash|hashed)\b/gi },
|
|
{ id: "api-keys-rotation-revocation", pattern: /\b(?:rotate|rotation|revoke|revocation|deactivate|disable|active|expires|expiration|deleteLegacyApiKey|rotateLegacyApiKey)\b/gi },
|
|
{ id: "onboarding-steps", pattern: /\b(?:onboarding|onboard|step|steps|wizard|setup|getting\s*started|first\s*run|welcome)\b/gi },
|
|
{ id: "roles-permissions", pattern: /\b(?:role|roles|permission|permissions|policy|policies|group|groups|member|members|admin|owner)\b/gi },
|
|
{ id: "billing-notifications-policies", pattern: /\b(?:billing|subscription|plan|plans|quota|limit|notification|notifications|email|policy|policies|entitlement|entitlements)\b/gi },
|
|
];
|
|
export const gates = [
|
|
{ id: "typecheck", command: "pnpm", args: ["run", "typecheck"] },
|
|
{ id: "lint", command: "pnpm", args: ["run", "lint"] },
|
|
{ id: "ci-baseline", command: "node", args: ["scripts/verify-ci-baseline.mjs"] },
|
|
{ id: "vitest-settingsApiOnboarding-smoke", command: "pnpm", args: ["exec", "vitest", "run", "apps/web/app/routes/app.$tenant/settings/members.test.ts", "apps/web/app/routes/app.$tenant/settings/profile.mfa.test.tsx", "packages/db/src/api-keys.server.test.ts", "packages/entitlements/src/index.test.ts", "apps/web/app/routes/app.$tenant.test.tsx"] },
|
|
{ id: "settingsApiOnboarding-settings-api-onboarding-inventory", type: "settingsApiOnboarding-settings-api-onboarding-inventory" },
|
|
];
|
|
function shellCommand(gate) { return gate.type === "settingsApiOnboarding-settings-api-onboarding-inventory" ? "read-only Settings API Onboarding settings/API-key/onboarding inventory" : [gate.command, ...gate.args].join(" "); }
|
|
function outputFingerprint(output) { const normalized = output.split(/\r?\n/).map((line) => line.trim()).filter(Boolean).slice(-40).join("\n"); return { hash: createHash("sha256").update(output).digest("hex").slice(0, 12), excerpt: normalized.slice(0, 1200) || "(no output)" }; }
|
|
function parseNumber(value) { const parsed = Number.parseInt(value, 10); return Number.isFinite(parsed) ? parsed : 0; }
|
|
function conservativeFailureCount(count, output) { return count > 0 ? count : output.trim() ? 1 : 0; }
|
|
function countVitestFailures(output) { const failedTests = output.match(/Tests\s+(\d+)\s+failed/i); if (failedTests) return parseNumber(failedTests[1]); const failedFiles = output.match(/Test Files\s+(\d+)\s+failed/i); return failedFiles ? parseNumber(failedFiles[1]) : 0; }
|
|
export function failureCount(gateId, stdout, stderr, exitCode = 1) {
|
|
if (exitCode === 0 || gateId === "settingsApiOnboarding-settings-api-onboarding-inventory") return 0;
|
|
const output = `${stdout}\n${stderr}`;
|
|
if (gateId === "typecheck") return conservativeFailureCount((output.match(/error TS\d+/g) ?? []).length, output);
|
|
if (gateId === "lint") { const problemsMatch = output.match(/✖\s+(\d+)\s+problems?\s+\((\d+)\s+errors?/i); const compactMatch = output.match(/(\d+)\s+errors?/i); return conservativeFailureCount(problemsMatch ? parseNumber(problemsMatch[2]) : compactMatch ? parseNumber(compactMatch[1]) : 0, output); }
|
|
if (gateId === "vitest-settingsApiOnboarding-smoke") return conservativeFailureCount(countVitestFailures(output), output);
|
|
if (gateId === "ci-baseline") { const summaryMatch = output.match(/Summary:\s+\d+\s+passed,\s+\d+\s+observed,\s+(\d+)\s+failed/i); return conservativeFailureCount(summaryMatch ? parseNumber(summaryMatch[1]) : 0, output); }
|
|
return output.trim() ? 1 : 0;
|
|
}
|
|
function runShellGate(gate) { return new Promise((resolve) => { const startedAt = Date.now(); const child = spawn(gate.command, gate.args, { cwd: repoRoot, env: { ...process.env, CI: "1" }, shell: false }); let stdout = ""; let stderr = ""; child.stdout.on("data", (chunk) => { stdout += chunk; }); child.stderr.on("data", (chunk) => { stderr += chunk; }); child.on("error", (error) => { stderr += `\n${error.message}`; }); child.on("close", (code) => { const output = `${stdout}\n${stderr}`; resolve({ id: gate.id, command: shellCommand(gate), exitCode: code ?? 1, durationMs: Date.now() - startedAt, failureCount: failureCount(gate.id, stdout, stderr, code ?? 1), fingerprint: outputFingerprint(output) }); }); }); }
|
|
function isIgnoredDirectory(name, relativePath) { return ignoredDirectoryNames.has(name) || ignoredDirectoryNames.has(relativePath) || name.endsWith(".cache"); }
|
|
function isIgnoredFile(name) { return ignoredFileNames.has(name) || name.endsWith(".generated.ts") || name.endsWith(".generated.tsx") || name.endsWith(".min.js") || name.endsWith(".map"); }
|
|
async function rootExists(root, rootRelativePath) { try { await stat(path.join(root, rootRelativePath)); return true; } catch (error) { if (error?.code === "ENOENT") return false; throw error; } }
|
|
async function collectFiles(root, rootRelativePath) {
|
|
const absoluteRoot = path.join(root, rootRelativePath); const rootStat = await stat(absoluteRoot); if (rootStat.isFile()) return [absoluteRoot]; const files = [];
|
|
async function walk(directory) { const entries = await readdir(directory, { withFileTypes: true }); for (const entry of entries) { const absolutePath = path.join(directory, entry.name); const relativePath = path.relative(root, absolutePath).split(path.sep).join("/"); if (entry.isDirectory()) { if (!isIgnoredDirectory(entry.name, relativePath)) await walk(absolutePath); } else if (entry.isFile() && /\.(?:ts|tsx|js|jsx|mjs|cjs|css|json|prisma|mdx?)$/.test(entry.name) && !isIgnoredFile(entry.name)) files.push(absolutePath); } }
|
|
await walk(absoluteRoot); return files;
|
|
}
|
|
export async function runSettingsApiOnboardingInventory(root = repoRoot) {
|
|
const existingRoots = []; const missingRoots = [];
|
|
for (const rootRelativePath of settingsApiOnboardingInventoryScanRoots) { if (await rootExists(root, rootRelativePath)) existingRoots.push(rootRelativePath); else missingRoots.push(rootRelativePath); }
|
|
const files = (await Promise.all(existingRoots.map((rootRelativePath) => collectFiles(root, rootRelativePath)))).flat(); const counts = new Map(settingsApiOnboardingInventoryPatterns.map(({ id }) => [id, 0])); const matches = [];
|
|
for (const file of files) { const relativeFile = path.relative(root, file).split(path.sep).join("/"); const lines = (await readFile(file, "utf8")).split(/\r?\n/); lines.forEach((line, index) => { for (const { id, pattern } of settingsApiOnboardingInventoryPatterns) { pattern.lastIndex = 0; const found = line.match(pattern); if (!found) continue; counts.set(id, (counts.get(id) ?? 0) + found.length); matches.push(`${relativeFile}:${index + 1}: [${id}] ${line.trim()}`); } }); }
|
|
return { roots: existingRoots, missingRoots, filesScanned: files.length, counts: Object.fromEntries(counts.entries()), totalMatches: matches.length, matches };
|
|
}
|
|
async function runSettingsApiOnboardingInventoryGate(gate) { const startedAt = Date.now(); const inventory = await runSettingsApiOnboardingInventory(repoRoot); const outputLines = [`Settings API Onboarding inventory scanned roots: ${inventory.roots.join(", ") || "(none)"}`, `Settings API Onboarding inventory missing roots skipped: ${inventory.missingRoots.join(", ") || "(none)"}`, `files scanned: ${inventory.filesScanned}`, ...Object.entries(inventory.counts).map(([id, count]) => `${id}: ${count}`), `total matches: ${inventory.totalMatches}`, ...inventory.matches.slice(0, 240)]; if (inventory.matches.length > 240) outputLines.push(`... ${inventory.matches.length - 240} additional matches omitted`); const output = outputLines.join("\n"); return { id: gate.id, command: shellCommand(gate), exitCode: 0, durationMs: Date.now() - startedAt, failureCount: 0, inventory, fingerprint: outputFingerprint(output) }; }
|
|
async function runGate(gate) { return gate.type === "settingsApiOnboarding-settings-api-onboarding-inventory" ? runSettingsApiOnboardingInventoryGate(gate) : runShellGate(gate); }
|
|
function renderInventoryCounts(result) { if (!result.inventory) return []; return ["- Inventory Roots: " + (result.inventory.roots.join(", ") || "(none)"), "- Missing Roots Skipped: " + (result.inventory.missingRoots.join(", ") || "(none)"), `- Files Scanned: ${result.inventory.filesScanned}`, "- Inventory Counts:", ...Object.entries(result.inventory.counts).map(([id, count]) => ` - ${id}: ${count}`), `- Total Inventory Matches: ${result.inventory.totalMatches}`]; }
|
|
export function renderReport(results) {
|
|
const failedShellGates = results.filter((result) => result.exitCode !== 0 && result.id !== "settingsApiOnboarding-settings-api-onboarding-inventory");
|
|
const lines = ["# Settings API Onboarding Baseline Health", "", `Generated: ${new Date().toISOString()}`, "", "This report captures non-E2E baseline gates for Settings API Onboarding baseline. The runner exits non-zero when any shell gate fails, but always attempts to write this report first.", "The shell gates run with CI=1 and fingerprint TypeScript, ESLint, CI baseline, and focused Vitest smoke coverage. The Settings API Onboarding inventory gate is read-only and fingerprints workspace settings, user account settings, API-key scopes/rotation/revocation, onboarding steps, roles/permissions, billing, notifications, and policies while skipping missing roots plus generated, dependency, build, cache, coverage, swagger, and TC artifact folders.", "", `Aggregated Exitcode: ${failedShellGates.length > 0 ? 1 : 0}`, `Failed Shell Gates: ${failedShellGates.length}`, ""];
|
|
for (const result of results) lines.push(`## ${result.id}`, "", `- Command: \`${result.command}\``, `- Exitcode: ${result.exitCode}`, `- Failure Count: ${result.failureCount}`, `- Duration: ${result.durationMs}ms`, `- Output Fingerprint: ${result.fingerprint.hash}`, ...renderInventoryCounts(result), "", "```text", result.fingerprint.excerpt, "```", "");
|
|
return `${lines.join("\n")}\n`;
|
|
}
|
|
async function writeFailureReport(error) { await mkdir(path.dirname(reportPath), { recursive: true }); const output = error?.stack ?? String(error); await writeFile(reportPath, renderReport([{ id: "runner-error", command: "node scripts/verify-settings-api-onboarding-baseline.mjs", exitCode: 1, durationMs: 0, failureCount: 1, fingerprint: outputFingerprint(output) }]), "utf8"); }
|
|
async function main() { const results = []; for (const gate of gates) { console.log(`Running ${gate.id}: ${shellCommand(gate)}`); results.push(await runGate(gate)); } await mkdir(path.dirname(reportPath), { recursive: true }); await writeFile(reportPath, renderReport(results), "utf8"); console.log(`Wrote ${path.relative(repoRoot, reportPath)}`); const failedShellGates = results.filter((result) => result.exitCode !== 0 && result.id !== "settingsApiOnboarding-settings-api-onboarding-inventory"); if (failedShellGates.length > 0) process.exitCode = 1; }
|
|
if (process.argv[1] === fileURLToPath(import.meta.url)) main().catch(async (error) => { console.error(error); try { await writeFailureReport(error); } finally { process.exit(1); } });
|