Reorganize repo structure and standards references
This commit is contained in:
1 parent
3cb93a81be
commit
be2054274d
5885 files changed
+2588
-195
No files matched your search
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"name": "@toolctrl/observability",
|
||||
"version": "0.0.0",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"sideEffects": false,
|
||||
"exports": {
|
||||
".": "./src/index.ts"
|
||||
},
|
||||
"scripts": {
|
||||
"test": "vitest run src",
|
||||
"typecheck": "tsc -p tsconfig.json --noEmit"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^25.6.0",
|
||||
"typescript": "^5.3.2"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
/**
|
||||
* `@toolctrl/observability` — shared OpenTelemetry helpers.
|
||||
*
|
||||
* Currently exposes the PII-scrubbing SpanProcessor. The package is
|
||||
* intentionally framework-agnostic and does not take a runtime dependency
|
||||
* on any `@opentelemetry/*` SDK package — callers wire it into their own
|
||||
* NodeSDK / WebTracerProvider span-processor chains.
|
||||
*/
|
||||
export {
|
||||
HEADER_ALLOWLIST,
|
||||
PIISpanProcessor,
|
||||
isHeaderAttribute,
|
||||
redactSpanAttributes,
|
||||
scrubDbStatement,
|
||||
scrubUrlQuery,
|
||||
type DelegateSpanProcessor,
|
||||
type PiiScrubableSpan,
|
||||
} from "./pii-span-processor.js";
|
||||
+207
@@ -0,0 +1,207 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
import {
|
||||
HEADER_ALLOWLIST,
|
||||
PIISpanProcessor,
|
||||
redactSpanAttributes,
|
||||
scrubDbStatement,
|
||||
scrubUrlQuery,
|
||||
type DelegateSpanProcessor,
|
||||
type PiiScrubableSpan,
|
||||
} from "./pii-span-processor.js";
|
||||
|
||||
describe("scrubUrlQuery", () => {
|
||||
it("replaces the query-string with the [scrubbed] placeholder", () => {
|
||||
expect(scrubUrlQuery("https://api.example.com/path?token=secret&foo=bar")).toBe("https://api.example.com/path?[scrubbed]");
|
||||
});
|
||||
|
||||
it("returns the input unchanged when no query-string is present", () => {
|
||||
expect(scrubUrlQuery("https://api.example.com/path")).toBe("https://api.example.com/path");
|
||||
});
|
||||
|
||||
it("handles empty query-strings", () => {
|
||||
expect(scrubUrlQuery("https://api.example.com/path?")).toBe("https://api.example.com/path?[scrubbed]");
|
||||
});
|
||||
});
|
||||
|
||||
describe("scrubDbStatement", () => {
|
||||
it("rewrites single-quoted literals to `?`", () => {
|
||||
expect(scrubDbStatement("SELECT * FROM users WHERE email = 'alice@example.com'")).toBe("SELECT * FROM users WHERE email = ?");
|
||||
});
|
||||
|
||||
it("rewrites double-quoted literals to `?`", () => {
|
||||
expect(scrubDbStatement('UPDATE t SET name = "Alice"')).toBe("UPDATE t SET name = ?");
|
||||
});
|
||||
|
||||
it("handles multiple literals", () => {
|
||||
expect(scrubDbStatement("INSERT INTO t (a, b) VALUES ('x', 'y')")).toBe("INSERT INTO t (a, b) VALUES (?, ?)");
|
||||
});
|
||||
|
||||
it("preserves escaped quotes inside literals without infinite loops", () => {
|
||||
expect(scrubDbStatement("SELECT 'foo\\'bar' AS x")).toBe("SELECT ? AS x");
|
||||
});
|
||||
|
||||
it("returns the input unchanged when no string literal is present", () => {
|
||||
expect(scrubDbStatement("SELECT count(*) FROM users WHERE id = $1")).toBe("SELECT count(*) FROM users WHERE id = $1");
|
||||
});
|
||||
});
|
||||
|
||||
describe("redactSpanAttributes", () => {
|
||||
it("strips query-strings from URL-like attributes (http.url, http.target, url.full, url.path)", () => {
|
||||
const attributes: Record<string, unknown> = {
|
||||
"http.url": "https://api.example.com/foo?token=secret",
|
||||
"http.target": "/foo?token=secret",
|
||||
"url.full": "https://api.example.com/foo?token=secret",
|
||||
"url.path": "/foo?token=secret",
|
||||
};
|
||||
redactSpanAttributes(attributes);
|
||||
expect(attributes["http.url"]).toBe("https://api.example.com/foo?[scrubbed]");
|
||||
expect(attributes["http.target"]).toBe("/foo?[scrubbed]");
|
||||
expect(attributes["url.full"]).toBe("https://api.example.com/foo?[scrubbed]");
|
||||
expect(attributes["url.path"]).toBe("/foo?[scrubbed]");
|
||||
});
|
||||
|
||||
it("drops authorization, cookie, set-cookie and x-api-key request headers", () => {
|
||||
const attributes: Record<string, unknown> = {
|
||||
"http.request.header.authorization": "Bearer redacted-me",
|
||||
"http.request.header.cookie": "session=abc",
|
||||
"http.request.header.x-api-key": "k-123",
|
||||
"http.response.header.set-cookie": "session=xyz",
|
||||
};
|
||||
redactSpanAttributes(attributes);
|
||||
expect(attributes).not.toHaveProperty("http.request.header.authorization");
|
||||
expect(attributes).not.toHaveProperty("http.request.header.cookie");
|
||||
expect(attributes).not.toHaveProperty("http.request.header.x-api-key");
|
||||
expect(attributes).not.toHaveProperty("http.response.header.set-cookie");
|
||||
});
|
||||
|
||||
it("keeps every header in the allowlist", () => {
|
||||
const attributes: Record<string, unknown> = {};
|
||||
for (const allowed of HEADER_ALLOWLIST) {
|
||||
attributes[`http.request.header.${allowed}`] = "value";
|
||||
attributes[`http.response.header.${allowed}`] = "value";
|
||||
}
|
||||
redactSpanAttributes(attributes);
|
||||
for (const allowed of HEADER_ALLOWLIST) {
|
||||
expect(attributes).toHaveProperty(`http.request.header.${allowed}`, "value");
|
||||
expect(attributes).toHaveProperty(`http.response.header.${allowed}`, "value");
|
||||
}
|
||||
});
|
||||
|
||||
it("scrubs db.statement literals to `?`", () => {
|
||||
const attributes: Record<string, unknown> = {
|
||||
"db.statement": "SELECT * FROM users WHERE email = 'leak@example.com'",
|
||||
};
|
||||
redactSpanAttributes(attributes);
|
||||
expect(attributes["db.statement"]).toBe("SELECT * FROM users WHERE email = ?");
|
||||
});
|
||||
|
||||
it("drops messaging.message_payload but keeps the payload-size attribute", () => {
|
||||
const attributes: Record<string, unknown> = {
|
||||
"messaging.message_payload": "raw-job-blob",
|
||||
"messaging.message_payload_size_bytes": 42,
|
||||
};
|
||||
redactSpanAttributes(attributes);
|
||||
expect(attributes).not.toHaveProperty("messaging.message_payload");
|
||||
expect(attributes["messaging.message_payload_size_bytes"]).toBe(42);
|
||||
});
|
||||
|
||||
it("is a no-op on undefined attributes", () => {
|
||||
expect(() => redactSpanAttributes(undefined)).not.toThrow();
|
||||
});
|
||||
|
||||
it("ignores non-string values on URL-like and db.statement keys", () => {
|
||||
const attributes: Record<string, unknown> = {
|
||||
"http.url": 42,
|
||||
"db.statement": null,
|
||||
};
|
||||
redactSpanAttributes(attributes);
|
||||
expect(attributes["http.url"]).toBe(42);
|
||||
expect(attributes["db.statement"]).toBeNull();
|
||||
});
|
||||
|
||||
it("lowercases the header name before matching the allowlist", () => {
|
||||
const attributes: Record<string, unknown> = {
|
||||
"http.request.header.Authorization": "Bearer redacted-me",
|
||||
"http.request.header.Content-Type": "application/json",
|
||||
};
|
||||
redactSpanAttributes(attributes);
|
||||
expect(attributes).not.toHaveProperty("http.request.header.Authorization");
|
||||
expect(attributes).toHaveProperty("http.request.header.Content-Type", "application/json");
|
||||
});
|
||||
});
|
||||
|
||||
describe("PIISpanProcessor", () => {
|
||||
function makeDelegateStub(): DelegateSpanProcessor & {
|
||||
onStart: ReturnType<typeof vi.fn>;
|
||||
onEnd: ReturnType<typeof vi.fn>;
|
||||
forceFlush: ReturnType<typeof vi.fn>;
|
||||
shutdown: ReturnType<typeof vi.fn>;
|
||||
} {
|
||||
return {
|
||||
onStart: vi.fn(),
|
||||
onEnd: vi.fn(),
|
||||
forceFlush: vi.fn(async () => undefined),
|
||||
shutdown: vi.fn(async () => undefined),
|
||||
};
|
||||
}
|
||||
|
||||
it("delegates onStart unchanged", () => {
|
||||
const delegate = makeDelegateStub();
|
||||
const processor = new PIISpanProcessor(delegate);
|
||||
const span: PiiScrubableSpan = { attributes: { "http.url": "https://example.com/?q=1" } };
|
||||
const context = {};
|
||||
processor.onStart(span, context);
|
||||
expect(delegate.onStart).toHaveBeenCalledWith(span, context);
|
||||
// onStart must NOT scrub: attributes that are added later in span lifetime
|
||||
// must still be scrubbable at onEnd.
|
||||
expect(span.attributes?.["http.url"]).toBe("https://example.com/?q=1");
|
||||
});
|
||||
|
||||
it("redacts span attributes during onEnd before delegating", () => {
|
||||
const delegate = makeDelegateStub();
|
||||
const processor = new PIISpanProcessor(delegate);
|
||||
const span: PiiScrubableSpan = {
|
||||
attributes: {
|
||||
"http.url": "https://example.com/path?token=secret",
|
||||
"http.request.header.authorization": "Bearer secret",
|
||||
"http.request.header.content-type": "application/json",
|
||||
"db.statement": "SELECT * FROM users WHERE email = 'leak@example.com'",
|
||||
"messaging.message_payload": "raw-job-blob",
|
||||
},
|
||||
};
|
||||
|
||||
processor.onEnd(span);
|
||||
|
||||
expect(delegate.onEnd).toHaveBeenCalledTimes(1);
|
||||
expect(delegate.onEnd).toHaveBeenCalledWith(span);
|
||||
expect(span.attributes?.["http.url"]).toBe("https://example.com/path?[scrubbed]");
|
||||
expect(span.attributes).not.toHaveProperty("http.request.header.authorization");
|
||||
expect(span.attributes?.["http.request.header.content-type"]).toBe("application/json");
|
||||
expect(span.attributes?.["db.statement"]).toBe("SELECT * FROM users WHERE email = ?");
|
||||
expect(span.attributes).not.toHaveProperty("messaging.message_payload");
|
||||
});
|
||||
|
||||
it("still delegates onEnd when the scrubber throws", () => {
|
||||
const delegate = makeDelegateStub();
|
||||
const processor = new PIISpanProcessor(delegate);
|
||||
// Inject a getter that throws when the scrubber iterates the attribute bag.
|
||||
const span: PiiScrubableSpan = {
|
||||
get attributes() {
|
||||
throw new Error("attributes unavailable");
|
||||
},
|
||||
} as PiiScrubableSpan;
|
||||
|
||||
expect(() => processor.onEnd(span)).not.toThrow();
|
||||
expect(delegate.onEnd).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("forwards forceFlush and shutdown to the delegate", async () => {
|
||||
const delegate = makeDelegateStub();
|
||||
const processor = new PIISpanProcessor(delegate);
|
||||
await processor.forceFlush();
|
||||
await processor.shutdown();
|
||||
expect(delegate.forceFlush).toHaveBeenCalledTimes(1);
|
||||
expect(delegate.shutdown).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
+205
@@ -0,0 +1,205 @@
|
||||
/**
|
||||
* PII-scrubbing OpenTelemetry SpanProcessor.
|
||||
*
|
||||
* Decorator pattern: wraps an existing {@link SpanProcessor} (e.g. a
|
||||
* {@link BatchSpanProcessor}) and redacts sensitive span attributes during
|
||||
* `onEnd()` before delegating. The processor is structurally typed so that
|
||||
* `@toolctrl/observability` does not need to take a hard dependency on the
|
||||
* OpenTelemetry SDK packages — runtimes that already depend on
|
||||
* `@opentelemetry/sdk-trace-base` (api, worker, web) pass their `BatchSpanProcessor`
|
||||
* instances in directly.
|
||||
*
|
||||
* Scrubbing rules (see {@link redactSpanAttributes}):
|
||||
* - `http.url`, `http.target` and the OTel v1.20+ aliases `url.full`,
|
||||
* `url.path` get their query-string replaced with `?[scrubbed]`.
|
||||
* - HTTP request/response headers are filtered through {@link HEADER_ALLOWLIST}.
|
||||
* Anything outside the allowlist (notably `authorization`, `cookie`,
|
||||
* `set-cookie`, `x-api-key`) is dropped.
|
||||
* - `db.statement` literal-string parameters (`'...'` and `"..."`) are
|
||||
* rewritten to `?` (defense-in-depth — Prisma already parametrises).
|
||||
* - `messaging.message_payload` is dropped entirely; the payload-size
|
||||
* attribute (if present) is preserved.
|
||||
*
|
||||
* The processor MUST be registered BEFORE the exporter in the span-processor
|
||||
* chain — typically by wrapping the exporter's `BatchSpanProcessor`:
|
||||
*
|
||||
* ```ts
|
||||
* spanProcessors: [new PIISpanProcessor(new BatchSpanProcessor(otlpExporter))]
|
||||
* ```
|
||||
*/
|
||||
|
||||
/**
|
||||
* Structural shape of an OpenTelemetry span passed to a SpanProcessor.
|
||||
* Only the fields we mutate / read are typed here so we don't need to take
|
||||
* a runtime dependency on `@opentelemetry/sdk-trace-base`.
|
||||
*/
|
||||
export interface PiiScrubableSpan {
|
||||
attributes?: Record<string, unknown> | undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Structural shape of `SpanProcessor` from `@opentelemetry/sdk-trace-base`.
|
||||
* Async `forceFlush()` / `shutdown()` are declared as `Promise<void>` here
|
||||
* to match the SDK shape exactly so a `BatchSpanProcessor` is assignable to
|
||||
* `DelegateSpanProcessor`.
|
||||
*/
|
||||
export interface DelegateSpanProcessor {
|
||||
onStart(span: PiiScrubableSpan, parentContext: unknown): void;
|
||||
onEnd(span: PiiScrubableSpan): void;
|
||||
forceFlush(): Promise<void>;
|
||||
shutdown(): Promise<void>;
|
||||
}
|
||||
|
||||
export const HEADER_ALLOWLIST: ReadonlySet<string> = new Set(
|
||||
[
|
||||
"content-type",
|
||||
"user-agent",
|
||||
"accept-language",
|
||||
"accept-encoding",
|
||||
"x-request-id",
|
||||
"x-forwarded-for",
|
||||
"referer",
|
||||
].map((name) => name.toLowerCase()),
|
||||
);
|
||||
|
||||
const QUERY_STRING_PLACEHOLDER = "?[scrubbed]";
|
||||
|
||||
const URL_LIKE_ATTRIBUTES = ["http.url", "http.target", "url.full", "url.path", "url.query"] as const;
|
||||
|
||||
const DROP_ATTRIBUTES = ["messaging.message_payload"] as const;
|
||||
|
||||
const HEADER_PREFIXES = ["http.request.header.", "http.response.header."] as const;
|
||||
|
||||
/**
|
||||
* Strips the query-string from a URL-like attribute, replacing it with
|
||||
* `?[scrubbed]`. Returns the input unchanged when no `?` is present.
|
||||
*/
|
||||
export function scrubUrlQuery(value: string): string {
|
||||
const queryIndex = value.indexOf("?");
|
||||
if (queryIndex < 0) return value;
|
||||
return `${value.slice(0, queryIndex)}${QUERY_STRING_PLACEHOLDER}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Replaces all single-/double-quoted string literals in a SQL statement
|
||||
* with `?`. Used as defense-in-depth on top of Prisma's parameterisation.
|
||||
*/
|
||||
export function scrubDbStatement(statement: string): string {
|
||||
// Walk character-by-character so we handle escaped quotes and avoid
|
||||
// catastrophic backtracking on adversarial input.
|
||||
let out = "";
|
||||
let i = 0;
|
||||
while (i < statement.length) {
|
||||
const ch = statement.charAt(i);
|
||||
if (ch === "'" || ch === '"') {
|
||||
const quote = ch;
|
||||
let j = i + 1;
|
||||
while (j < statement.length) {
|
||||
const inner = statement.charAt(j);
|
||||
if (inner === "\\" && j + 1 < statement.length) {
|
||||
j += 2;
|
||||
continue;
|
||||
}
|
||||
if (inner === quote) {
|
||||
break;
|
||||
}
|
||||
j += 1;
|
||||
}
|
||||
out += "?";
|
||||
i = j + 1;
|
||||
continue;
|
||||
}
|
||||
out += ch;
|
||||
i += 1;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns whether a span attribute key targets an HTTP header.
|
||||
* Both `http.request.header.<name>` and `http.response.header.<name>`
|
||||
* are matched (per OTel HTTP semantic conventions v1.20+).
|
||||
*/
|
||||
export function isHeaderAttribute(key: string): { isHeader: true; name: string } | { isHeader: false } {
|
||||
for (const prefix of HEADER_PREFIXES) {
|
||||
if (key.startsWith(prefix)) {
|
||||
return { isHeader: true, name: key.slice(prefix.length).toLowerCase() };
|
||||
}
|
||||
}
|
||||
return { isHeader: false };
|
||||
}
|
||||
|
||||
/**
|
||||
* Mutates the given attribute bag in-place to apply the PII-scrubbing rules.
|
||||
* Exported for unit-tests so we can verify each rule independently of the
|
||||
* full SpanProcessor lifecycle.
|
||||
*/
|
||||
export function redactSpanAttributes(attributes: Record<string, unknown> | undefined): void {
|
||||
if (!attributes) return;
|
||||
|
||||
for (const key of Object.keys(attributes)) {
|
||||
// Drop disallowed attributes wholesale.
|
||||
if ((DROP_ATTRIBUTES as readonly string[]).includes(key)) {
|
||||
delete attributes[key];
|
||||
continue;
|
||||
}
|
||||
|
||||
// Strip query-strings from URL-like attributes.
|
||||
if ((URL_LIKE_ATTRIBUTES as readonly string[]).includes(key)) {
|
||||
const value = attributes[key];
|
||||
if (typeof value === "string") {
|
||||
attributes[key] = scrubUrlQuery(value);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// Scrub SQL literals from db.statement.
|
||||
if (key === "db.statement") {
|
||||
const value = attributes[key];
|
||||
if (typeof value === "string") {
|
||||
attributes[key] = scrubDbStatement(value);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// Allowlist-filter HTTP request/response headers.
|
||||
const header = isHeaderAttribute(key);
|
||||
if (header.isHeader) {
|
||||
if (!HEADER_ALLOWLIST.has(header.name)) {
|
||||
delete attributes[key];
|
||||
}
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Decorator SpanProcessor that scrubs PII from every ended span before
|
||||
* forwarding to a delegate (typically a `BatchSpanProcessor`).
|
||||
*/
|
||||
export class PIISpanProcessor implements DelegateSpanProcessor {
|
||||
constructor(private readonly delegate: DelegateSpanProcessor) {}
|
||||
|
||||
onStart(span: PiiScrubableSpan, parentContext: unknown): void {
|
||||
this.delegate.onStart(span, parentContext);
|
||||
}
|
||||
|
||||
onEnd(span: PiiScrubableSpan): void {
|
||||
try {
|
||||
redactSpanAttributes(span.attributes);
|
||||
} catch {
|
||||
// Scrubber failures must never block the exporter chain; the span is
|
||||
// forwarded as-is on the assumption the exporter will reject malformed
|
||||
// payloads at its own layer.
|
||||
}
|
||||
this.delegate.onEnd(span);
|
||||
}
|
||||
|
||||
forceFlush(): Promise<void> {
|
||||
return this.delegate.forceFlush();
|
||||
}
|
||||
|
||||
shutdown(): Promise<void> {
|
||||
return this.delegate.shutdown();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"extends": "../../tsconfig.json",
|
||||
"compilerOptions": {
|
||||
"allowJs": false,
|
||||
"allowImportingTsExtensions": false,
|
||||
"lib": ["ES2022"],
|
||||
"module": "NodeNext",
|
||||
"moduleResolution": "NodeNext",
|
||||
"noEmit": true,
|
||||
"types": ["node"]
|
||||
},
|
||||
"include": ["src/**/*.ts"]
|
||||
}
|
||||
Reference in new issue
Block a user