#!/usr/bin/env node /** * Secret-scanning CI gate. * * Scans every Git-tracked text file looking for hardcoded secrets: source-host * tokens, cloud/API credentials, private keys, JWT-shaped tokens, embedded URL * credentials, and high-entropy assignment literals on secret-like names. * * Findings on `.env.example` variants or test/fixture/mock/stub paths are * recorded as "reviewed" rather than failures. Lines that contain obvious * placeholder markers (`placeholder`, `example`, `your-`, `<...>`, `${...}`, * `xxxxx`, `dummy`, `REDACTED`, ...) are skipped entirely. * * Exit 1 on any real (non-reviewed) finding, 0 otherwise. */ import fs from "node:fs"; import path from "node:path"; import { spawnSync } from "node:child_process"; import url from "node:url"; export const SKIP_DIR_NAMES = new Set([ "node_modules", "dist", "build", "coverage", ".react-router", ".git", ".cache", "generated", ".next", ".turbo", ]); const SKIP_FILE_SUFFIXES = [ ".lock", "-lock.json", ".png", ".jpg", ".jpeg", ".svg", ".ico", ".webp", ".pdf", ".woff", ".woff2", ".ttf", ".eot", ".zip", ".gz", ".br", ".wasm", ".mp4", ]; const SKIP_FILE_NAMES = new Set(["pnpm-lock.yaml"]); // Allowlisted env-example filenames whose findings are logged as reviewed. const ALLOWLISTED_FILENAMES = new Set([ ".env.example", ".env.local.example", ".env.production.example", ".env.test", ]); // Exact path components that mark an intentionally synthetic test surface. const ALLOWLISTED_PATH_COMPONENTS = new Set([ "test", "tests", "__tests__", "fixture", "fixtures", "mock", "mocks", "stub", "stubs", ]); // Self-reference: this scanner file describes the patterns it scans for, so // matches against its own source must be treated as reviewed. const SELF_FILENAMES = new Set([ "verify-secrets.mjs", "verify-secrets.test.mjs", ]); // Placeholder markers that disqualify a line from being a real secret. const PLACEHOLDER_MARKERS = [ "placeholder", "example", "your-", "your_", "replace-with", "replace_with", "xxxxx", "dummy", "redacted", "changeme", "change-me", "change_me", "todo", "fixme", ]; export function lineHasPlaceholder(line) { const lower = line.toLowerCase(); for (const marker of PLACEHOLDER_MARKERS) { if (lower.includes(marker)) return true; } if (/<[^>]+>/.test(line)) return true; if (/\{[^}]+\}/.test(line)) return true; if (line.includes("${")) return true; return false; } export const PATTERNS = [ { id: "github-pat", regex: /\bgh[opurs]_[A-Za-z0-9_]{20,}\b/g }, { id: "github-fine-grained-pat", regex: /\bgithub_pat_[A-Za-z0-9_]{20,}\b/g }, { id: "gitlab-pat", regex: /\bglpat-[A-Za-z0-9_-]{20,}\b/g }, { id: "npm-access-token", regex: /\bnpm_[A-Za-z0-9]{30,}\b/g }, { id: "pypi-access-token", regex: /\bpypi-[A-Za-z0-9_-]{40,}\b/g }, { id: "huggingface-token", regex: /\bhf_[A-Za-z0-9]{30,}\b/g }, { id: "google-api-key", regex: /\bAIza[0-9A-Za-z_-]{35}\b/g }, { id: "aws-access-key-id", regex: /\bAKIA[0-9A-Z]{16}\b/g }, { id: "payment-secret-live", regex: /\bsk_live_[A-Za-z0-9]{20,}\b/g }, { id: "payment-webhook-live", regex: /\bwhsec_[A-Za-z0-9]{20,}\b/g, requiresLineContains: "live", }, { id: "slack-bot-token", regex: /\bxoxb-\d+-\d+-\d+-[A-Za-z0-9]+\b/g }, { id: "private-key-header", regex: /-----BEGIN (?:RSA |EC |DSA |OPENSSH |PGP )?PRIVATE KEY-----/g, }, { id: "age-secret-key", regex: /\bAGE-SECRET-KEY-1[0-9A-Z]{20,}\b/g }, { id: "jwt-like", regex: /\beyJ[A-Za-z0-9_-]{10,}\.eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b/g, }, { id: "generic-high-entropy-assignment", regex: /\b(?:password|passphrase|secret|token|auth_?token|api_?key|access_?key|client_?secret|private_?key|signing_?key|webhook_?secret)\s*[:=]\s*(?:['"][A-Za-z0-9+/=_-]{24,}['"]|(?=[A-Za-z0-9+/=_-]{32,}\b)(?=[A-Za-z0-9+/=_-]*\d)[A-Za-z0-9+/=_-]{32,})/gi, }, { id: "embedded-url-credentials", regex: /\b(?:https?|postgres(?:ql)?|redis):\/\/[^\s/:@]+:[^\s/@]{8,}@[^\s]+/gi, }, ]; export function isAllowlistedPath(absPath) { const basename = path.basename(absPath); if (SELF_FILENAMES.has(basename)) return true; if (ALLOWLISTED_FILENAMES.has(basename)) return true; const components = path.normalize(absPath).toLowerCase().split(path.sep); if (components.some((component) => ALLOWLISTED_PATH_COMPONENTS.has(component))) return true; return /\.(?:test|spec)\.[^.]+$/i.test(basename); } export function scanText(text, { filename = "" } = {}) { const findings = []; const lines = text.split(/\r?\n/); for (let i = 0; i < lines.length; i += 1) { const line = lines[i]; if (lineHasPlaceholder(line)) continue; for (const pattern of PATTERNS) { pattern.regex.lastIndex = 0; for (const match of line.matchAll(pattern.regex)) { if ( pattern.requiresLineContains && !line.toLowerCase().includes(pattern.requiresLineContains) ) { continue; } const trailingText = line.slice((match.index ?? 0) + match[0].length); if ( pattern.id === "generic-high-entropy-assignment" && /:\s*[A-Z][A-Z0-9_]*$/.test(match[0]) && /^\s*[,;)]/.test(trailingText) ) { continue; } findings.push({ file: filename, line: i + 1, patternId: pattern.id, match: match[0], }); } } } return findings; } function shouldSkipFile(absPath) { const basename = path.basename(absPath); if (SKIP_FILE_NAMES.has(basename)) return true; const lower = basename.toLowerCase(); for (const suffix of SKIP_FILE_SUFFIXES) { if (lower.endsWith(suffix)) return true; } return false; } function listFilesRecursive(dir) { if (!fs.existsSync(dir)) return []; const out = []; const stack = [dir]; while (stack.length > 0) { const current = stack.pop(); let entries; try { entries = fs.readdirSync(current, { withFileTypes: true }); } catch { continue; } for (const entry of entries) { if (entry.isDirectory()) { if (SKIP_DIR_NAMES.has(entry.name)) continue; stack.push(path.join(current, entry.name)); } else if (entry.isFile()) { const abs = path.join(current, entry.name); if (shouldSkipFile(abs)) continue; out.push(abs); } } } return out; } export function collectFiles(root) { const tracked = spawnSync("git", ["-C", root, "ls-files", "-z"], { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"], }); if (tracked.status === 0) { return tracked.stdout .split("\0") .filter(Boolean) .map((relativePath) => path.join(root, relativePath)) .filter((absolutePath) => { if (shouldSkipFile(absolutePath)) return false; try { return fs.lstatSync(absolutePath).isFile(); } catch { return false; } }); } return listFilesRecursive(root); } export function isKnownLocalCredential(finding) { if (finding.patternId !== "embedded-url-credentials") return false; const rawUrl = finding.match .split('"')[0] .split("'")[0] .replace(/^postgres:\/\//, "postgresql://"); try { const parsed = new URL(rawUrl); return ( parsed.protocol === "postgresql:" && ["localhost", "127.0.0.1", "::1"].includes(parsed.hostname) && [["postgres", "postgres"], ["hivegrid", "hivegrid"]].some( ([username, password]) => parsed.username === username && parsed.password === password, ) ); } catch { return false; } } export function evaluate(root = process.cwd()) { const files = collectFiles(root); const real = []; const reviewed = []; for (const file of files) { let content; try { content = fs.readFileSync(file, "utf8"); } catch { continue; } const findings = scanText(content, { filename: file }); if (findings.length === 0) continue; const allowlisted = isAllowlistedPath(file); for (const finding of findings) { if (allowlisted || isKnownLocalCredential(finding)) { reviewed.push(finding); } else { real.push(finding); } } } return { scanned: files.length, patterns: PATTERNS.length, real, reviewed, }; } export function formatFinding(finding, rootRel) { const rel = path.relative(rootRel, finding.file) || finding.file; return `${rel}:${finding.line}: ${finding.patternId} matched [redacted length=${finding.match.length}]`; } const isMain = import.meta.url === url.pathToFileURL(process.argv[1] ?? "").href; /* eslint-disable no-console -- CLI reporter writes to stdout/stderr. */ if (isMain) { const root = process.cwd(); const result = evaluate(root); if (result.real.length === 0) { console.log("PASS no secret-shaped strings found"); } else { for (const finding of result.real) { console.error(`FAIL ${formatFinding(finding, root)}`); } } if (result.reviewed.length > 0) { for (const finding of result.reviewed) { console.log(`reviewed ${formatFinding(finding, root)}`); } } console.log( `\nSummary: scanned files: ${result.scanned}, patterns: ${result.patterns}, real: ${result.real.length}, reviewed: ${result.reviewed.length}`, ); if (result.real.length > 0) { process.exit(1); } } /* eslint-enable no-console */