Track HiveGrid Cloud planning reference

hivegrid-cloud/scripts/observability/scan-console.ts

@@ -0,0 +1,181 @@
+#!/usr/bin/env -S node --import tsx
+/**
+ * scan-console.ts
+ *
+ * Inventory of all `console.*` calls in server-code paths.
+ *
+ * Generates docs/observability/console-migration-inventory.md as a Markdown
+ * table with file:line + snippet for every match outside of test/spec files
+ * and outside of explicit `// eslint-disable-next-line no-console` lines.
+ *
+ * Usage:
+ *   pnpm exec tsx scripts/observability/scan-console.ts
+ *   pnpm exec tsx scripts/observability/scan-console.ts --check   # exits 1 if hits > 0
+ *
+ * Server paths scanned:
+ *   - apps/api/src/**
+ *   - apps/worker/src/**
+ *   - apps/web/app/**\/*.server.ts(x)
+ *   - apps/web/app/**\/*.server.ts (modules)
+ *   - apps/web/app/modules/**\/services/**
+ *   - apps/web/app/utils/**
+ *   - packages/*\/src/**
+ *
+ * Skipped:
+ *   - **\/*.test.ts(x), **\/*.spec.ts(x)
+ *   - lines preceded by `// eslint-disable-next-line no-console`
+ *   - dist/, build/, node_modules/
+ */
+import { execSync } from "node:child_process";
+import { readFileSync, writeFileSync, mkdirSync } from "node:fs";
+import { dirname, relative, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+const REPO_ROOT = resolve(__dirname, "..", "..");
+const OUTPUT = resolve(REPO_ROOT, "docs/observability/console-migration-inventory.md");
+
+const SERVER_GLOBS: string[] = ["apps/api/src", "apps/worker/src", "apps/web/app", "packages"];
+
+const CONSOLE_RE = /console\.(log|error|warn|info|debug|dir|trace)\s*\(/;
+const TEST_FILE_RE = /\.(test|spec)\.(ts|tsx|js|jsx|mjs|cjs)$/;
+const DISTLIKE_RE = /(^|\/)(node_modules|dist|build|coverage|\.react-router|generated)(\/|$)/;
+const SERVER_FILE_RE =
+  /(\.server\.tsx?|\/\.server\/|apps\/api\/src\/|apps\/worker\/src\/|packages\/[^/]+\/src\/|apps\/web\/app\/utils\/|apps\/web\/app\/modules\/[^/]+\/services\/|apps\/web\/app\/modules\/[^/]+\/db\/|apps\/web\/app\/routes\/.+\.server\.tsx?)/;
+
+interface Hit {
+  file: string;
+  line: number;
+  level: string;
+  snippet: string;
+}
+
+function gitTrackedFiles(): string[] {
+  const out = execSync("git ls-files", { cwd: REPO_ROOT, encoding: "utf8", maxBuffer: 32 * 1024 * 1024 });
+  return out.split("\n").filter(Boolean);
+}
+
+function isServerPath(file: string): boolean {
+  if (DISTLIKE_RE.test(file)) return false;
+  if (TEST_FILE_RE.test(file)) return false;
+  if (!SERVER_GLOBS.some((g) => file.startsWith(g + "/"))) return false;
+  // Within apps/web/app: only `.server.ts(x)` files and explicit server folders
+  if (file.startsWith("apps/web/app/")) {
+    return SERVER_FILE_RE.test(file);
+  }
+  // apps/api/src, apps/worker/src, packages/*/src are entirely server code
+  return /\.(ts|tsx|mjs|cjs|js)$/.test(file);
+}
+
+function scanFile(absPath: string, relPath: string): Hit[] {
+  let content: string;
+  try {
+    content = readFileSync(absPath, "utf8");
+  } catch {
+    return [];
+  }
+  const hits: Hit[] = [];
+  const lines = content.split("\n");
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i];
+    const m = line.match(CONSOLE_RE);
+    if (!m) continue;
+    // Skip if line is inside a block comment-only context — too expensive to AST.
+    // Skip if line is a single-line comment (//) preceding the console call.
+    const trimmed = line.trimStart();
+    if (trimmed.startsWith("//") || trimmed.startsWith("*")) continue;
+    // Skip if previous line is eslint-disable-next-line no-console
+    const prev = i > 0 ? lines[i - 1].trim() : "";
+    if (/eslint-disable-next-line[^\n]*no-console/.test(prev)) continue;
+    // Skip same-line disable
+    if (/eslint-disable-line[^\n]*no-console/.test(line)) continue;
+    hits.push({
+      file: relPath,
+      line: i + 1,
+      level: m[1],
+      snippet: line.trim().slice(0, 160),
+    });
+  }
+  return hits;
+}
+
+function main(): void {
+  const args = process.argv.slice(2);
+  const checkMode = args.includes("--check");
+
+  const files = gitTrackedFiles().filter(isServerPath);
+  const allHits: Hit[] = [];
+  for (const rel of files) {
+    const abs = resolve(REPO_ROOT, rel);
+    allHits.push(...scanFile(abs, rel));
+  }
+
+  // Group by file for output
+  const byFile = new Map<string, Hit[]>();
+  for (const h of allHits) {
+    const arr = byFile.get(h.file) ?? [];
+    arr.push(h);
+    byFile.set(h.file, arr);
+  }
+  const sortedFiles = [...byFile.keys()].sort();
+
+  const now = new Date().toISOString();
+  const lines: string[] = [];
+  lines.push("# Console-Call Migration Inventory");
+  lines.push("");
+  lines.push(`> Generated by \`scripts/observability/scan-console.ts\` on ${now}.`);
+  lines.push("> Re-run with: \`pnpm exec tsx scripts/observability/scan-console.ts\`");
+  lines.push("");
+  lines.push(`**Total \`console.*\` hits in server-code paths: ${allHits.length}**`);
+  lines.push(`**Unique files: ${sortedFiles.length}**`);
+  lines.push("");
+  lines.push("Server-paths scanned:");
+  lines.push("");
+  lines.push("- `apps/api/src/**`");
+  lines.push("- `apps/worker/src/**`");
+  lines.push("- `apps/web/app/**/*.server.ts(x)` and `apps/web/app/modules/*/services/**` and `apps/web/app/utils/**`");
+  lines.push("- `packages/*/src/**`");
+  lines.push("");
+  lines.push("Excluded:");
+  lines.push("");
+  lines.push("- `**/*.test.ts(x)`, `**/*.spec.ts(x)`");
+  lines.push("- Lines after `// eslint-disable-next-line no-console`");
+  lines.push("- `dist/`, `build/`, `coverage/`, `node_modules/`, `.react-router/`, `generated/`");
+  lines.push("");
+  if (allHits.length === 0) {
+    lines.push("## All clear");
+    lines.push("");
+    lines.push("No `console.*` calls remain in server-code paths. The ESLint rule `no-console: error` enforces this going forward.");
+  } else {
+    lines.push("## Hits by file");
+    lines.push("");
+    lines.push("| File | Line | Level | Snippet |");
+    lines.push("|------|-----:|-------|---------|");
+    for (const file of sortedFiles) {
+      for (const h of byFile.get(file)!) {
+        const snippetEscaped = h.snippet.replace(/\|/g, "\\|").replace(/`/g, "\\`");
+        lines.push(`| \`${h.file}\` | ${h.line} | ${h.level} | \`${snippetEscaped}\` |`);
+      }
+    }
+    lines.push("");
+    lines.push("## Summary by file");
+    lines.push("");
+    lines.push("| File | Hits |");
+    lines.push("|------|-----:|");
+    for (const file of sortedFiles) {
+      lines.push(`| \`${file}\` | ${byFile.get(file)!.length} |`);
+    }
+  }
+
+  mkdirSync(dirname(OUTPUT), { recursive: true });
+  writeFileSync(OUTPUT, lines.join("\n") + "\n", "utf8");
+  // eslint-disable-next-line no-console
+  console.log(`scan-console: ${allHits.length} hit(s) across ${sortedFiles.length} file(s). Wrote ${relative(REPO_ROOT, OUTPUT)}`);
+
+  if (checkMode && allHits.length > 0) {
+    process.exit(1);
+  }
+}
+
+main();

hivegrid-cloud/scripts/observability/smoke-metrics.sh

@@ -0,0 +1,117 @@
+#!/usr/bin/env bash
+#
+# Smoke-test the /status and /metrics endpoints on a deployed boilerplate.
+#
+# Usage:
+#   METRICS_TOKEN=...   scripts/observability/smoke-metrics.sh             # uses staging defaults
+#   METRICS_TOKEN=...   scripts/observability/smoke-metrics.sh https://api.example.com https://worker.example.com:9100 https://app.example.com
+#
+# Exits non-zero on the first failing assertion. Designed to run from CI
+# after deploy and from the operator's laptop.
+
+set -euo pipefail
+
+API_BASE="${1:-https://api.boiler.saas-ctrl.com}"
+WORKER_BASE="${2:-}"
+WEB_BASE="${3:-https://app.boiler.saas-ctrl.com}"
+
+if [ -z "${METRICS_TOKEN:-}" ]; then
+  echo "FAIL: METRICS_TOKEN env var is required." >&2
+  exit 2
+fi
+
+EXIT_CODE=0
+
+probe_status() {
+  local label="$1"
+  local url="$2"
+
+  local response status body
+  response=$(curl --max-time 10 --silent --show-error --write-out '\n%{http_code}' "$url" || true)
+  status=$(printf '%s' "$response" | tail -n1)
+  body=$(printf '%s' "$response" | sed '$d')
+
+  if [ "$status" != "200" ] && [ "$status" != "503" ]; then
+    echo "FAIL: ${label} /status returned ${status} (expected 200 or 503)" >&2
+    echo "Body: ${body}" >&2
+    EXIT_CODE=1
+    return
+  fi
+
+  if ! printf '%s' "$body" | grep -q '"service":"' ; then
+    echo "FAIL: ${label} /status JSON missing service field" >&2
+    echo "Body: ${body}" >&2
+    EXIT_CODE=1
+    return
+  fi
+
+  echo "OK:   ${label} /status (HTTP ${status})"
+}
+
+probe_metrics_unauth() {
+  local label="$1"
+  local url="$2"
+
+  local status
+  status=$(curl --max-time 10 --silent --output /dev/null --write-out '%{http_code}' "$url" || true)
+  if [ "$status" != "401" ] && [ "$status" != "404" ]; then
+    echo "FAIL: ${label} /metrics without token returned ${status} (expected 401 or 404)" >&2
+    EXIT_CODE=1
+    return
+  fi
+  echo "OK:   ${label} /metrics without token (HTTP ${status})"
+}
+
+probe_metrics_authed() {
+  local label="$1"
+  local url="$2"
+
+  local response status body
+  response=$(curl --max-time 10 --silent --show-error -H "Authorization: Bearer ${METRICS_TOKEN}" \
+    --write-out '\n%{http_code}' "$url" || true)
+  status=$(printf '%s' "$response" | tail -n1)
+  body=$(printf '%s' "$response" | sed '$d')
+
+  if [ "$status" != "200" ]; then
+    echo "FAIL: ${label} /metrics returned ${status} (expected 200)" >&2
+    echo "Body (first 200 chars): $(printf '%s' "$body" | head -c 200)" >&2
+    EXIT_CODE=1
+    return
+  fi
+
+  for expected in "http_requests_total" "process_cpu_seconds_total"; do
+    if ! printf '%s' "$body" | grep -q "$expected" ; then
+      echo "FAIL: ${label} /metrics missing metric '${expected}'" >&2
+      EXIT_CODE=1
+      return
+    fi
+  done
+
+  echo "OK:   ${label} /metrics (HTTP 200, expected metric names found)"
+}
+
+# API
+probe_status         "api"    "${API_BASE}/status"
+probe_metrics_unauth "api"    "${API_BASE}/metrics"
+probe_metrics_authed "api"    "${API_BASE}/metrics"
+
+# Web (uses /api/status + /api/metrics resource routes)
+probe_status         "web"    "${WEB_BASE}/api/status"
+probe_metrics_unauth "web"    "${WEB_BASE}/api/metrics"
+probe_metrics_authed "web"    "${WEB_BASE}/api/metrics"
+
+# Worker — only probed when the caller passed a URL (worker admin server is
+# typically on a private network without TLS).
+if [ -n "$WORKER_BASE" ]; then
+  probe_status         "worker" "${WORKER_BASE}/status"
+  probe_metrics_unauth "worker" "${WORKER_BASE}/metrics"
+  probe_metrics_authed "worker" "${WORKER_BASE}/metrics"
+else
+  echo "SKIP: worker /status + /metrics (pass a URL as the second positional arg to enable)"
+fi
+
+if [ $EXIT_CODE -eq 0 ]; then
+  echo "All smoke checks passed."
+fi
+
+exit $EXIT_CODE

hivegrid-cloud/scripts/observability/smoke-otel.ts

@@ -0,0 +1,161 @@
+#!/usr/bin/env tsx
+/**
+ * OpenTelemetry OTLP HTTP smoke test.
+ *
+ * Sends a synthetic trace (one parent + two children) directly to the
+ * configured OTLP HTTP endpoint via `fetch` (no SDK), so an operator can
+ * verify the end-to-end pipeline without spinning up the full api/worker
+ * stack:
+ *   - DNS + TLS to the OTLP endpoint
+ *   - Gateway / Relay routes `POST /v1/traces` (or `/<path>/v1/traces`)
+ *   - Sentry/Tempo/Jaeger persists the trace
+ *
+ * Run from the repo root:
+ *   OTEL_EXPORTER_OTLP_ENDPOINT=https://otel.example.com/v1/traces \
+ *     pnpm exec tsx scripts/observability/smoke-otel.ts
+ *
+ * Exits 0 on 2xx response, 1 on non-2xx, 2 on missing endpoint config.
+ *
+ * @see docs/improvement-plan/02-observability/opentelemetry.md (Step 10)
+ */
+import { randomBytes } from "node:crypto";
+
+/* eslint-disable no-console */
+
+interface SmokeSpan {
+  traceId: string;
+  spanId: string;
+  parentSpanId: string | undefined;
+  name: string;
+  kind: number;
+  startTimeUnixNano: string;
+  endTimeUnixNano: string;
+  attributes: Array<{ key: string; value: { stringValue?: string; intValue?: string } }>;
+}
+
+function hex(bytes: number): string {
+  return randomBytes(bytes).toString("hex");
+}
+
+function buildSpans(): SmokeSpan[] {
+  const traceId = hex(16);
+  const parentSpanId = hex(8);
+  const apiSpanId = hex(8);
+  const workerSpanId = hex(8);
+
+  const start = Date.now() - 1000;
+  return [
+    {
+      traceId,
+      spanId: parentSpanId,
+      parentSpanId: undefined,
+      name: "smoke-otel.root",
+      kind: 1, // INTERNAL
+      startTimeUnixNano: String(BigInt(start) * 1_000_000n),
+      endTimeUnixNano: String(BigInt(start + 800) * 1_000_000n),
+      attributes: [
+        { key: "service.name", value: { stringValue: "boilerplate-smoke" } },
+        { key: "smoke.kind", value: { stringValue: "root" } },
+      ],
+    },
+    {
+      traceId,
+      spanId: apiSpanId,
+      parentSpanId,
+      name: "smoke-otel.api",
+      kind: 2, // SERVER
+      startTimeUnixNano: String(BigInt(start + 100) * 1_000_000n),
+      endTimeUnixNano: String(BigInt(start + 500) * 1_000_000n),
+      attributes: [
+        { key: "service.name", value: { stringValue: "boilerplate-smoke-api" } },
+        { key: "http.method", value: { stringValue: "GET" } },
+        { key: "http.target", value: { stringValue: "/healthz" } },
+      ],
+    },
+    {
+      traceId,
+      spanId: workerSpanId,
+      parentSpanId: apiSpanId,
+      name: "smoke-otel.worker",
+      kind: 5, // CONSUMER
+      startTimeUnixNano: String(BigInt(start + 250) * 1_000_000n),
+      endTimeUnixNano: String(BigInt(start + 600) * 1_000_000n),
+      attributes: [
+        { key: "service.name", value: { stringValue: "boilerplate-smoke-worker" } },
+        { key: "messaging.system", value: { stringValue: "bullmq" } },
+        { key: "messaging.destination", value: { stringValue: "smoke.queue" } },
+      ],
+    },
+  ];
+}
+
+function parseHeaders(raw: string | undefined): Record<string, string> {
+  const out: Record<string, string> = {};
+  if (!raw) return out;
+  for (const pair of raw.split(",")) {
+    const eq = pair.indexOf("=");
+    if (eq < 0) continue;
+    const key = pair.slice(0, eq).trim();
+    const value = pair.slice(eq + 1).trim();
+    if (key) out[key] = value;
+  }
+  return out;
+}
+
+async function main(): Promise<void> {
+  const rawEndpoint = process.env.OTEL_EXPORTER_OTLP_ENDPOINT;
+  if (!rawEndpoint) {
+    console.error("[smoke-otel] OTEL_EXPORTER_OTLP_ENDPOINT is not set; nothing to do.");
+    process.exit(2);
+  }
+  // Normalise: callers commonly pass the base endpoint without the `/v1/traces`
+  // path because the SDK appends it automatically. Replicate that here.
+  let url = rawEndpoint;
+  if (!/\/v1\/traces\/?$/.test(url)) {
+    url = url.replace(/\/$/, "") + "/v1/traces";
+  }
+
+  const spans = buildSpans();
+  const traceId = spans[0]?.traceId ?? "(none)";
+  console.log(`[smoke-otel] POST ${url} — trace_id=${traceId}, spans=${spans.length}`);
+
+  const payload = {
+    resourceSpans: [
+      {
+        resource: {
+          attributes: [
+            { key: "service.name", value: { stringValue: "boilerplate-smoke" } },
+            { key: "service.namespace", value: { stringValue: "toolctrl" } },
+          ],
+        },
+        scopeSpans: [
+          {
+            scope: { name: "scripts/observability/smoke-otel.ts" },
+            spans,
+          },
+        ],
+      },
+    ],
+  };
+
+  const headers: Record<string, string> = { "content-type": "application/json", ...parseHeaders(process.env.OTEL_EXPORTER_OTLP_HEADERS) };
+
+  let response: Response;
+  try {
+    response = await fetch(url, { method: "POST", headers, body: JSON.stringify(payload) });
+  } catch (cause) {
+    console.error(`[smoke-otel] fetch failed: ${(cause as Error)?.message ?? String(cause)}`);
+    process.exit(1);
+  }
+
+  const text = await response.text();
+  if (!response.ok) {
+    console.error(`[smoke-otel] non-2xx response: ${response.status} ${response.statusText}\n${text.slice(0, 800)}`);
+    process.exit(1);
+  }
+
+  console.log(`[smoke-otel] ok: ${response.status}; trace_id=${traceId}`);
+  console.log(`[smoke-otel] body: ${text.slice(0, 400)}`);
+}
+
+void main();