Track HiveGrid Cloud planning reference
This commit is contained in:
1 parent
b3d5ea6764
commit
e341c5938a
5784 files changed
+1104929
-3
No files matched your search
@@ -0,0 +1,284 @@
|
||||
#!/usr/bin/env node
|
||||
/* eslint-disable no-console */
|
||||
import { createHash } from "node:crypto";
|
||||
import { mkdir, readdir, readFile, stat, writeFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { spawn } from "node:child_process";
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
const repoRoot = path.resolve(__dirname, "..");
|
||||
export const reportPath = path.join(repoRoot, ".tc/baselines/compliance-retention/baseline-health.md");
|
||||
|
||||
const ignoredDirectoryNames = new Set([
|
||||
".git",
|
||||
"node_modules",
|
||||
"build",
|
||||
"dist",
|
||||
"coverage",
|
||||
".react-router",
|
||||
".turbo",
|
||||
".next",
|
||||
".cache",
|
||||
".vite",
|
||||
"generated",
|
||||
"llm-docs",
|
||||
".tc",
|
||||
]);
|
||||
|
||||
export const complianceRetentionInventoryScanRoots = [
|
||||
"docs",
|
||||
"apps/web/app/routes",
|
||||
"apps/api/src",
|
||||
"apps/worker/src",
|
||||
"packages/db/src",
|
||||
"packages/queue/src",
|
||||
"packages/exports/src",
|
||||
"packages/storage/src",
|
||||
"prisma/schema",
|
||||
];
|
||||
|
||||
export const complianceRetentionInventoryPatterns = [
|
||||
{ id: "security-disclosure-marker", pattern: /\b(?:security\s+disclosure|vulnerability|responsible\s+disclosure|cve|security\s+contact|report\s+a\s+security\s+issue)\b/gi },
|
||||
{ id: "gdpr-data-inventory-marker", pattern: /\b(?:gdpr|data\s+inventory|data\s+subject|personal\s+data|data\s+export|data\s+erasure|right\s+to\s+be\s+forgotten)\b/gi },
|
||||
{ id: "retention-policy-marker", pattern: /\b(?:retention|retention\s+policy|expiresAt|expire|expiry|ttl|deleteAfter|cleanup|purge)\b/gi },
|
||||
{ id: "soft-delete-hard-delete-marker", pattern: /\b(?:soft\s*delete|hard\s*delete|deletedAt|deleteMany|forceDelete|restore|tombstone|cascade\s+delete)\b/g },
|
||||
{ id: "export-hook-marker", pattern: /\b(?:export|exports|download|downloads|dataExport|exportJob|processExport|csv|json|xlsx|pdf)\b/gi },
|
||||
{ id: "admin-pause-resume-marker", pattern: /\b(?:admin|pause|paused|resume|resumed|suspend|unsuspend|disable|enable)\b/gi },
|
||||
{ id: "audit-subscription-risk-marker", pattern: /\b(?:audit|auditable|auditLog|actorId|tenantId|workspaceId|subscription|billing|entitlement|risk)\b/g },
|
||||
{ id: "generated-client-reference", pattern: /\b(?:packages\/db\/generated|@toolctrl\/db\/generated|generated\/prisma|PrismaClient|client\/models|client\/enums)\b/g },
|
||||
];
|
||||
|
||||
export const gates = [
|
||||
{ id: "typecheck", command: "pnpm", args: ["run", "typecheck"] },
|
||||
{ id: "lint", command: "pnpm", args: ["run", "lint"] },
|
||||
{
|
||||
id: "vitest-docs-schema-export",
|
||||
command: "pnpm",
|
||||
args: [
|
||||
"exec",
|
||||
"vitest",
|
||||
"run",
|
||||
"packages/db/src/export-jobs.server.test.ts",
|
||||
"packages/db/src/file-object-schema.test.ts",
|
||||
"packages/exports/src/index.test.ts",
|
||||
"packages/exports/src/csv/index.test.ts",
|
||||
"packages/exports/src/json/index.test.ts",
|
||||
"packages/exports/src/xlsx/index.test.ts",
|
||||
],
|
||||
},
|
||||
{
|
||||
id: "vitest-worker-queue",
|
||||
command: "pnpm",
|
||||
args: ["exec", "vitest", "run", "packages/queue/src/producers.test.ts", "packages/queue/src/export-contract.test.ts", "apps/worker/src/handlers.test.ts", "apps/worker/src/runtime.test.ts"],
|
||||
},
|
||||
{ id: "compliance-retention-inventory", type: "compliance-retention-inventory" },
|
||||
];
|
||||
|
||||
function shellCommand(gate) {
|
||||
if (gate.type === "compliance-retention-inventory") return "read-only compliance/retention inventory";
|
||||
return [gate.command, ...gate.args].join(" ");
|
||||
}
|
||||
|
||||
function outputFingerprint(output) {
|
||||
const normalized = output
|
||||
.split(/\r?\n/)
|
||||
.map((line) => line.trim())
|
||||
.filter(Boolean)
|
||||
.slice(-40)
|
||||
.join("\n");
|
||||
return { hash: createHash("sha256").update(output).digest("hex").slice(0, 12), excerpt: normalized.slice(0, 1200) || "(no output)" };
|
||||
}
|
||||
|
||||
function parseNumber(value) {
|
||||
const parsed = Number.parseInt(value, 10);
|
||||
return Number.isFinite(parsed) ? parsed : 0;
|
||||
}
|
||||
|
||||
function conservativeFailureCount(count, output) {
|
||||
return count > 0 ? count : output.trim() ? 1 : 0;
|
||||
}
|
||||
|
||||
function countVitestFailures(output) {
|
||||
const failedTests = output.match(/Tests\s+(\d+)\s+failed/i);
|
||||
if (failedTests) return parseNumber(failedTests[1]);
|
||||
const failedFiles = output.match(/Test Files\s+(\d+)\s+failed/i);
|
||||
return failedFiles ? parseNumber(failedFiles[1]) : 0;
|
||||
}
|
||||
|
||||
export function failureCount(gateId, stdout, stderr, exitCode = 1) {
|
||||
if (exitCode === 0 || gateId === "compliance-retention-inventory") return 0;
|
||||
const output = `${stdout}\n${stderr}`;
|
||||
if (gateId === "typecheck") return conservativeFailureCount((output.match(/error TS\d+/g) ?? []).length, output);
|
||||
if (gateId === "lint") {
|
||||
const problemsMatch = output.match(/✖\s+(\d+)\s+problems?\s+\((\d+)\s+errors?/i);
|
||||
const compactMatch = output.match(/(\d+)\s+errors?/i);
|
||||
return conservativeFailureCount(problemsMatch ? parseNumber(problemsMatch[2]) : compactMatch ? parseNumber(compactMatch[1]) : 0, output);
|
||||
}
|
||||
if (gateId === "vitest-docs-schema-export" || gateId === "vitest-worker-queue") return conservativeFailureCount(countVitestFailures(output), output);
|
||||
return output.trim() ? 1 : 0;
|
||||
}
|
||||
|
||||
function runShellGate(gate) {
|
||||
return new Promise((resolve) => {
|
||||
const startedAt = Date.now();
|
||||
const child = spawn(gate.command, gate.args, { cwd: repoRoot, env: { ...process.env, CI: "1" }, shell: false });
|
||||
let stdout = "";
|
||||
let stderr = "";
|
||||
child.stdout.on("data", (chunk) => {
|
||||
stdout += chunk;
|
||||
});
|
||||
child.stderr.on("data", (chunk) => {
|
||||
stderr += chunk;
|
||||
});
|
||||
child.on("error", (error) => {
|
||||
stderr += `\n${error.message}`;
|
||||
});
|
||||
child.on("close", (code) => {
|
||||
const output = `${stdout}\n${stderr}`;
|
||||
resolve({
|
||||
id: gate.id,
|
||||
command: shellCommand(gate),
|
||||
exitCode: code ?? 1,
|
||||
durationMs: Date.now() - startedAt,
|
||||
failureCount: failureCount(gate.id, stdout, stderr, code ?? 1),
|
||||
fingerprint: outputFingerprint(output),
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function isIgnoredDirectory(name, relativePath) {
|
||||
return ignoredDirectoryNames.has(name) || ignoredDirectoryNames.has(relativePath) || name.endsWith(".cache") || relativePath === "packages/db/generated" || relativePath.startsWith("packages/db/generated/");
|
||||
}
|
||||
|
||||
async function rootExists(root, rootRelativePath) {
|
||||
try {
|
||||
await stat(path.join(root, rootRelativePath));
|
||||
return true;
|
||||
} catch (error) {
|
||||
if (error?.code === "ENOENT") return false;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function collectFiles(root, rootRelativePath) {
|
||||
const absoluteRoot = path.join(root, rootRelativePath);
|
||||
const files = [];
|
||||
async function walk(directory) {
|
||||
const entries = await readdir(directory, { withFileTypes: true });
|
||||
for (const entry of entries) {
|
||||
const absolutePath = path.join(directory, entry.name);
|
||||
const relativePath = path.relative(root, absolutePath).split(path.sep).join("/");
|
||||
if (entry.isDirectory()) {
|
||||
if (isIgnoredDirectory(entry.name, relativePath)) continue;
|
||||
await walk(absolutePath);
|
||||
} else if (entry.isFile() && /\.(?:md|mdx|ts|tsx|js|jsx|mjs|cjs|prisma|sql)$/.test(entry.name) && !entry.name.endsWith(".generated.ts") && !entry.name.endsWith(".generated.tsx")) {
|
||||
files.push(absolutePath);
|
||||
}
|
||||
}
|
||||
}
|
||||
await walk(absoluteRoot);
|
||||
return files;
|
||||
}
|
||||
|
||||
export async function runComplianceRetentionInventory(root = repoRoot) {
|
||||
const existingRoots = [];
|
||||
for (const rootRelativePath of complianceRetentionInventoryScanRoots) if (await rootExists(root, rootRelativePath)) existingRoots.push(rootRelativePath);
|
||||
const files = (await Promise.all(existingRoots.map((rootRelativePath) => collectFiles(root, rootRelativePath)))).flat();
|
||||
const counts = new Map(complianceRetentionInventoryPatterns.map(({ id }) => [id, 0]));
|
||||
const matches = [];
|
||||
for (const file of files) {
|
||||
const relativeFile = path.relative(root, file).split(path.sep).join("/");
|
||||
const lines = (await readFile(file, "utf8")).split(/\r?\n/);
|
||||
lines.forEach((line, index) => {
|
||||
for (const { id, pattern } of complianceRetentionInventoryPatterns) {
|
||||
pattern.lastIndex = 0;
|
||||
const found = line.match(pattern);
|
||||
if (!found) continue;
|
||||
counts.set(id, (counts.get(id) ?? 0) + found.length);
|
||||
matches.push(`${relativeFile}:${index + 1}: [${id}] ${line.trim().slice(0, 240)}`);
|
||||
}
|
||||
});
|
||||
}
|
||||
return { roots: existingRoots, counts: Object.fromEntries(counts.entries()), totalMatches: matches.length, matches };
|
||||
}
|
||||
|
||||
async function runComplianceRetentionInventoryGate(gate) {
|
||||
const startedAt = Date.now();
|
||||
const inventory = await runComplianceRetentionInventory(repoRoot);
|
||||
const outputLines = [
|
||||
`Compliance/retention inventory scanned roots: ${inventory.roots.join(", ")}`,
|
||||
...Object.entries(inventory.counts).map(([id, count]) => `${id}: ${count}`),
|
||||
`total matches: ${inventory.totalMatches}`,
|
||||
...inventory.matches.slice(0, 240),
|
||||
];
|
||||
if (inventory.matches.length > 240) outputLines.push(`... ${inventory.matches.length - 240} additional matches omitted`);
|
||||
const output = outputLines.join("\n");
|
||||
return { id: gate.id, command: shellCommand(gate), exitCode: 0, durationMs: Date.now() - startedAt, failureCount: 0, fingerprint: outputFingerprint(output) };
|
||||
}
|
||||
|
||||
async function runGate(gate) {
|
||||
if (gate.type === "compliance-retention-inventory") return runComplianceRetentionInventoryGate(gate);
|
||||
return runShellGate(gate);
|
||||
}
|
||||
|
||||
export function renderReport(results) {
|
||||
const lines = [
|
||||
"# Compliance Retention Baseline Health",
|
||||
"",
|
||||
`Generated: ${new Date().toISOString()}`,
|
||||
"",
|
||||
"This report captures non-E2E baseline gates for Compliance Retention baseline. The runner exits 0 when every gate was executed and this report was written, even if individual gates fail.",
|
||||
"The compliance/retention inventory gate is read-only and fingerprints security disclosure, GDPR data inventory, retention policy, soft/hard delete, export hook, admin pause/resume, audit/subscription risk, and accidental generated Prisma client references while skipping generated, dependency, build, cache, static output, and TC artifact folders.",
|
||||
"",
|
||||
];
|
||||
for (const result of results)
|
||||
lines.push(
|
||||
`## ${result.id}`,
|
||||
"",
|
||||
`- Command: \`${result.command}\``,
|
||||
`- Exitcode: ${result.exitCode}`,
|
||||
`- Failure Count: ${result.failureCount}`,
|
||||
`- Duration: ${result.durationMs}ms`,
|
||||
`- Output Fingerprint: ${result.fingerprint.hash}`,
|
||||
"",
|
||||
"```text",
|
||||
result.fingerprint.excerpt,
|
||||
"```",
|
||||
""
|
||||
);
|
||||
return `${lines.join("\n")}\n`;
|
||||
}
|
||||
|
||||
async function writeFailureReport(error) {
|
||||
await mkdir(path.dirname(reportPath), { recursive: true });
|
||||
const output = error?.stack ?? String(error);
|
||||
await writeFile(
|
||||
reportPath,
|
||||
renderReport([{ id: "runner-error", command: "node scripts/verify-compliance-retention-baseline.mjs", exitCode: 1, durationMs: 0, failureCount: 1, fingerprint: outputFingerprint(output) }]),
|
||||
"utf8"
|
||||
);
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const results = [];
|
||||
for (const gate of gates) {
|
||||
console.log(`Running ${gate.id}: ${shellCommand(gate)}`);
|
||||
results.push(await runGate(gate));
|
||||
}
|
||||
await mkdir(path.dirname(reportPath), { recursive: true });
|
||||
await writeFile(reportPath, renderReport(results), "utf8");
|
||||
console.log(`Wrote ${path.relative(repoRoot, reportPath)}`);
|
||||
}
|
||||
|
||||
if (process.argv[1] === fileURLToPath(import.meta.url)) {
|
||||
main().catch(async (error) => {
|
||||
console.error(error);
|
||||
try {
|
||||
await writeFailureReport(error);
|
||||
} finally {
|
||||
process.exit(1);
|
||||
}
|
||||
});
|
||||
}
|
||||
Reference in new issue
Block a user