Reorganize repo structure and standards references
This commit is contained in:
1 parent
3cb93a81be
commit
be2054274d
5885 files changed
+2588
-195
No files matched your search
@@ -0,0 +1,342 @@
|
||||
/**
|
||||
* Plugin output sanitization — Phase 06 #23.
|
||||
*
|
||||
* `sanitizePluginOutput(input, strictness)` walks an arbitrary JSON-
|
||||
* serialisable value and replaces sensitive substrings inside string
|
||||
* leaves with `[REDACTED:type]` markers before the result lands in the
|
||||
* `plugin_invocations.outputSummary` audit field.
|
||||
*
|
||||
* The pattern set covers:
|
||||
* - credit-card PANs (Luhn-validated to avoid false positives)
|
||||
* - JWTs (three base64url segments, payload parseable as JSON)
|
||||
* - IBANs (country-length-aware)
|
||||
* - generic API-keys (known prefixes + long high-entropy hex/base64)
|
||||
* - emails (only in `strict` mode — legitimate plugin output often
|
||||
* contains them)
|
||||
*
|
||||
* Plus field-name awareness: keys like `password`, `token`, `iban`,
|
||||
* `card_number`, `apiKey`, … have their value redacted regardless of
|
||||
* pattern.
|
||||
*
|
||||
* Performance guardrails: walk depth is capped at 16, every string
|
||||
* leaf is truncated to 64 KB with a `[TRUNCATED]` marker. The input
|
||||
* is never mutated — fresh arrays / plain objects are returned.
|
||||
*
|
||||
* Deferred (intentionally out of scope here):
|
||||
* - OTel span sanitization
|
||||
* - tenant-admin-configurable strictness per plugin (env-default only)
|
||||
*/
|
||||
|
||||
export type SanitizationStrictness = "strict" | "moderate" | "off";
|
||||
|
||||
/** Walk depth cap — anything deeper is replaced by `[DEPTH_LIMIT]`. */
|
||||
const MAX_DEPTH = 16;
|
||||
/** Max string leaf length in bytes-ish (utf16 code units) — 64 KB. */
|
||||
const MAX_STRING_LENGTH = 64 * 1024;
|
||||
/**
|
||||
* F-014 — total accumulated string budget across one walk. Closes the
|
||||
* "10 MB nested object" case: per-string truncation alone lets a plugin
|
||||
* exfiltrate megabytes by spreading it across 1000 keys. When the
|
||||
* budget is exceeded the walk emits `[SIZE_LIMIT]` markers in place of
|
||||
* further values. Default 1 MiB — operators override via
|
||||
* `PLUGIN_SANITIZE_TOTAL_BUDGET_BYTES`.
|
||||
*/
|
||||
const DEFAULT_TOTAL_BUDGET_BYTES = 1024 * 1024;
|
||||
const TRUNCATED_MARKER = "[TRUNCATED]";
|
||||
const DEPTH_LIMIT_MARKER = "[DEPTH_LIMIT]";
|
||||
const SIZE_LIMIT_MARKER = "[SIZE_LIMIT]";
|
||||
|
||||
function resolveTotalBudget(): number {
|
||||
const raw = (globalThis as { process?: { env?: Record<string, string | undefined> } }).process?.env
|
||||
?.PLUGIN_SANITIZE_TOTAL_BUDGET_BYTES;
|
||||
if (typeof raw !== "string" || raw.length === 0) return DEFAULT_TOTAL_BUDGET_BYTES;
|
||||
const parsed = Number.parseInt(raw, 10);
|
||||
if (!Number.isFinite(parsed) || parsed <= 0) return DEFAULT_TOTAL_BUDGET_BYTES;
|
||||
return parsed;
|
||||
}
|
||||
|
||||
const FIELD_NAME_REDACT = new Set([
|
||||
"email",
|
||||
"iban",
|
||||
"card_number",
|
||||
"cardnumber",
|
||||
"creditcard",
|
||||
"credit_card",
|
||||
"cc",
|
||||
"pan",
|
||||
"secret",
|
||||
"token",
|
||||
"apikey",
|
||||
"api_key",
|
||||
"authorization",
|
||||
"password",
|
||||
]);
|
||||
|
||||
/**
|
||||
* Credit-card candidate: 13-19 digits, optionally separated by single
|
||||
* space or hyphen between every group. Anchored on word-boundaries so
|
||||
* we don't gobble surrounding digits.
|
||||
*/
|
||||
const CREDIT_CARD_PATTERN = /\b(?:\d[ -]?){12,18}\d\b/g;
|
||||
|
||||
/**
|
||||
* JWT shape: three base64url segments separated by dots. We then
|
||||
* verify the payload segment is parseable JSON to drop false matches.
|
||||
*/
|
||||
const JWT_PATTERN = /\b[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b/g;
|
||||
|
||||
/** Email RFC-lite — good enough for sanitization purposes. */
|
||||
const EMAIL_PATTERN = /\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b/g;
|
||||
|
||||
/**
|
||||
* IBAN candidate — 2 letter country + 2 digit check + up to 30
|
||||
* alphanumerics. We refine by checking the country-specific length.
|
||||
*/
|
||||
const IBAN_PATTERN = /\b[A-Z]{2}\d{2}[A-Z0-9]{10,30}\b/g;
|
||||
|
||||
/** Common DACH/EU IBAN total-length table. */
|
||||
const IBAN_LENGTHS: Record<string, number> = {
|
||||
AT: 20,
|
||||
BE: 16,
|
||||
CH: 21,
|
||||
CZ: 24,
|
||||
DE: 22,
|
||||
DK: 18,
|
||||
EE: 20,
|
||||
ES: 24,
|
||||
FI: 18,
|
||||
FR: 27,
|
||||
GB: 22,
|
||||
GR: 27,
|
||||
HR: 21,
|
||||
HU: 28,
|
||||
IE: 22,
|
||||
IT: 27,
|
||||
LI: 21,
|
||||
LT: 20,
|
||||
LU: 20,
|
||||
LV: 21,
|
||||
MT: 31,
|
||||
NL: 18,
|
||||
NO: 15,
|
||||
PL: 28,
|
||||
PT: 25,
|
||||
RO: 24,
|
||||
SE: 24,
|
||||
SI: 19,
|
||||
SK: 24,
|
||||
};
|
||||
|
||||
/** Known secret prefixes — full token after prefix gets redacted. */
|
||||
const API_KEY_PREFIX_PATTERN =
|
||||
/\b(?:sk|pk|rk)_(?:test|live)_[A-Za-z0-9]{8,}\b|\bghp_[A-Za-z0-9]{20,}\b|\bgho_[A-Za-z0-9]{20,}\b|\bghs_[A-Za-z0-9]{20,}\b|\bghu_[A-Za-z0-9]{20,}\b|\bxox[abprs]-[A-Za-z0-9-]{10,}\b|\bAKIA[0-9A-Z]{16}\b|\bAIza[0-9A-Za-z_-]{20,}\b/g;
|
||||
|
||||
/** Generic long opaque token: >= 32 hex/base64url-ish chars. */
|
||||
const GENERIC_API_KEY_PATTERN = /\b[A-Za-z0-9_-]{32,}\b/g;
|
||||
|
||||
/**
|
||||
* Sanitize an arbitrary JSON-serialisable value. Returns a new value
|
||||
* with sensitive substrings replaced by `[REDACTED:type]` markers.
|
||||
*
|
||||
* @param input arbitrary value (string / number / boolean / null /
|
||||
* undefined / array / plain object)
|
||||
* @param strictness `strict` redacts emails, `moderate` skips them,
|
||||
* `off` is the identity function
|
||||
*/
|
||||
export function sanitizePluginOutput(
|
||||
input: unknown,
|
||||
strictness: SanitizationStrictness = "moderate"
|
||||
): unknown {
|
||||
if (strictness === "off") {
|
||||
return input;
|
||||
}
|
||||
// F-023 — track object identities we've already walked so a plugin
|
||||
// returning a circular reference doesn't blow the stack. WeakSet
|
||||
// ignores primitives + only adds real object identities.
|
||||
const seen = new WeakSet<object>();
|
||||
// F-014 — total accumulated string-bytes budget. Decremented by each
|
||||
// string leaf the walk emits; once exhausted further leaves become
|
||||
// SIZE_LIMIT_MARKER.
|
||||
const budget = { remaining: resolveTotalBudget() };
|
||||
return walk(input, strictness, 0, seen, budget);
|
||||
}
|
||||
|
||||
const CIRCULAR_MARKER = "[CIRCULAR]";
|
||||
|
||||
type WalkBudget = { remaining: number };
|
||||
|
||||
function walk(
|
||||
value: unknown,
|
||||
strictness: SanitizationStrictness,
|
||||
depth: number,
|
||||
seen: WeakSet<object>,
|
||||
budget: WalkBudget,
|
||||
): unknown {
|
||||
if (depth > MAX_DEPTH) {
|
||||
return DEPTH_LIMIT_MARKER;
|
||||
}
|
||||
if (budget.remaining <= 0) {
|
||||
return SIZE_LIMIT_MARKER;
|
||||
}
|
||||
if (value === null || value === undefined) {
|
||||
return value;
|
||||
}
|
||||
const t = typeof value;
|
||||
if (t === "string") {
|
||||
const sanitized = sanitizeString(value as string, strictness);
|
||||
budget.remaining -= sanitized.length;
|
||||
return sanitized;
|
||||
}
|
||||
if (t === "number" || t === "boolean" || t === "bigint") {
|
||||
return value;
|
||||
}
|
||||
if (Array.isArray(value)) {
|
||||
if (seen.has(value)) {
|
||||
return CIRCULAR_MARKER;
|
||||
}
|
||||
seen.add(value);
|
||||
return value.map((item) => walk(item, strictness, depth + 1, seen, budget));
|
||||
}
|
||||
if (t === "object") {
|
||||
const obj = value as Record<string, unknown>;
|
||||
if (seen.has(obj)) {
|
||||
return CIRCULAR_MARKER;
|
||||
}
|
||||
seen.add(obj);
|
||||
const out: Record<string, unknown> = {};
|
||||
for (const [key, raw] of Object.entries(obj)) {
|
||||
const normalizedKey = key.toLowerCase().replace(/[-_\s]/g, "");
|
||||
const sensitiveKey =
|
||||
FIELD_NAME_REDACT.has(normalizedKey) || FIELD_NAME_REDACT.has(key.toLowerCase());
|
||||
if (sensitiveKey && typeof raw === "string") {
|
||||
if (strictness === "strict") {
|
||||
out[key] = "[REDACTED:field]";
|
||||
} else if (raw.length > 4) {
|
||||
out[key] = "[REDACTED:field]";
|
||||
} else {
|
||||
out[key] = raw;
|
||||
}
|
||||
} else {
|
||||
out[key] = walk(raw, strictness, depth + 1, seen, budget);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
// functions, symbols — drop through unchanged (shouldn't appear in JSON)
|
||||
return value;
|
||||
}
|
||||
|
||||
function sanitizeString(raw: string, strictness: SanitizationStrictness): string {
|
||||
let s = raw;
|
||||
let truncated = false;
|
||||
if (s.length > MAX_STRING_LENGTH) {
|
||||
s = s.slice(0, MAX_STRING_LENGTH);
|
||||
truncated = true;
|
||||
}
|
||||
|
||||
// Order matters: JWT before generic-api-key, IBAN before generic, etc.
|
||||
s = s.replace(API_KEY_PREFIX_PATTERN, "[REDACTED:api-key]");
|
||||
s = s.replace(JWT_PATTERN, (match) => (isLikelyJwt(match) ? "[REDACTED:jwt]" : match));
|
||||
s = s.replace(IBAN_PATTERN, (match) => (isLikelyIban(match) ? "[REDACTED:iban]" : match));
|
||||
s = s.replace(CREDIT_CARD_PATTERN, (match) => {
|
||||
const digits = match.replace(/[^0-9]/g, "");
|
||||
if (digits.length < 13 || digits.length > 19) {
|
||||
return match;
|
||||
}
|
||||
return luhnValid(digits) ? "[REDACTED:credit-card]" : match;
|
||||
});
|
||||
if (strictness === "strict") {
|
||||
s = s.replace(EMAIL_PATTERN, "[REDACTED:email]");
|
||||
}
|
||||
s = s.replace(GENERIC_API_KEY_PATTERN, (match) =>
|
||||
looksLikeSecret(match) ? "[REDACTED:api-key]" : match
|
||||
);
|
||||
|
||||
if (truncated) {
|
||||
s = `${s}${TRUNCATED_MARKER}`;
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
/** Luhn check on a digit-only string. */
|
||||
export function luhnValid(digits: string): boolean {
|
||||
if (!/^\d+$/.test(digits)) {
|
||||
return false;
|
||||
}
|
||||
let sum = 0;
|
||||
let alt = false;
|
||||
for (let i = digits.length - 1; i >= 0; i--) {
|
||||
let d = digits.charCodeAt(i) - 48;
|
||||
if (alt) {
|
||||
d *= 2;
|
||||
if (d > 9) {
|
||||
d -= 9;
|
||||
}
|
||||
}
|
||||
sum += d;
|
||||
alt = !alt;
|
||||
}
|
||||
return sum % 10 === 0 && sum > 0;
|
||||
}
|
||||
|
||||
function isLikelyJwt(token: string): boolean {
|
||||
const parts = token.split(".");
|
||||
if (parts.length !== 3) {
|
||||
return false;
|
||||
}
|
||||
const [header, payload] = parts;
|
||||
if (!header || !payload) {
|
||||
return false;
|
||||
}
|
||||
// Both header and payload must decode to JSON objects.
|
||||
return isBase64UrlJsonObject(header) && isBase64UrlJsonObject(payload);
|
||||
}
|
||||
|
||||
function isBase64UrlJsonObject(segment: string): boolean {
|
||||
try {
|
||||
const padded = segment.replace(/-/g, "+").replace(/_/g, "/");
|
||||
const pad = padded.length % 4 === 0 ? "" : "=".repeat(4 - (padded.length % 4));
|
||||
const json = Buffer.from(padded + pad, "base64").toString("utf8");
|
||||
const parsed = JSON.parse(json);
|
||||
return parsed !== null && typeof parsed === "object" && !Array.isArray(parsed);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function isLikelyIban(candidate: string): boolean {
|
||||
const country = candidate.slice(0, 2);
|
||||
const expected = IBAN_LENGTHS[country];
|
||||
if (!expected) {
|
||||
return false;
|
||||
}
|
||||
return candidate.length === expected;
|
||||
}
|
||||
|
||||
/**
|
||||
* Heuristic: a long opaque token "looks like a secret" if it has high
|
||||
* character-class diversity (mixes digits + letters, or both letter
|
||||
* cases) — pure-numeric or pure-lowercase tokens are likely IDs.
|
||||
*/
|
||||
function looksLikeSecret(token: string): boolean {
|
||||
if (token.length < 32) {
|
||||
return false;
|
||||
}
|
||||
const hasUpper = /[A-Z]/.test(token);
|
||||
const hasLower = /[a-z]/.test(token);
|
||||
const hasDigit = /\d/.test(token);
|
||||
// Pure-lowercase or pure-uppercase strings (no digits) are usually
|
||||
// identifiers / slugs / uuids-without-hyphens — leave them alone.
|
||||
if (!hasDigit) {
|
||||
return false;
|
||||
}
|
||||
// Need mixed-case + digit (high entropy signal) — single-case-plus-digit
|
||||
// covers ids like timestamps too, so require both letter cases.
|
||||
if (!(hasUpper && hasLower)) {
|
||||
return false;
|
||||
}
|
||||
// Shannon-entropy proxy: require >= 12 unique chars to weed out
|
||||
// repeating patterns.
|
||||
const unique = new Set(token).size;
|
||||
return unique >= 12;
|
||||
}
|
||||
Reference in new issue
Block a user