import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import assert from "node:assert/strict"; import { execFileSync } from "node:child_process"; import { afterEach, describe, it } from "vitest"; import { formatReleaseDocsReleaseDocsResult, verifyReleaseDocsReleaseDocs } from "./verify-release-docs.mjs"; const fixtures = []; const VALID_DOCS = { "docs/release-checklist.md": [ "# Release Checklist", "", "## Pre-release verification", "Run focused non-E2E checks and link [deployment](deployment.md) evidence before tagging.", "", "## Changelog and version tagging", "Confirm SemVer versioning and changelog entries match the release tag.", "", "## Artifacts and rollback", "Record release artifact names and the rollback owner before promotion.", "", "## Manual runtime evidence", "Manual-only runtime smoke evidence is recorded after deployment, not run automatically.", "", "## Redaction requirements", "All release notes and logs use redacted or masked evidence only.", ].join("\n"), "docs/boilerplate-version-tracking.md": [ "# Boilerplate Version Tracking", "", "## SemVer source version", "Track the boilerplate version with SemVer source version metadata.", "", "## Downstream version ledger", "Each downstream project keeps a version tracking ledger for adopted source version changes.", "", "## Changelog alignment", "The changelog explains which artifact and migration registry entries changed.", "", "## Upgrade and rollback notes", "Upgrade notes include rollback instructions for failed adoption.", "", "## Redacted evidence", "Evidence uses redacted values such as API_KEY=.", ].join("\n"), "docs/production-checklist.md": [ "# Production Checklist", "", "## Production readiness", "Production readiness requires release checklist sign-off and non-E2E verification.", "", "## Deployment boundary", "Deployment steps describe the manual runtime service boundary.", "", "## Manual runtime checks", "Manual runtime checks are recorded only after explicit approval.", "", "## Rollback plan", "Rollback plan owners confirm artifact availability before release.", "", "## Redacted release evidence", "Redaction rules prohibit raw secrets in production readiness notes.", "Worker evidence includes worker-general, worker-mail, worker-import, worker-webhook, and worker-plugin.", "The permanent mail worker uses WORKER_QUEUE_GROUP=mail.", "Each worker record includes revision evidence, queue list evidence, and execute-health evidence.", ].join("\n"), "docs/releases/README.md": "# Releases\n\nRelease index links to [release checklist](../release-checklist.md) and changelog policy.", "docs/releases/2026-01.md": "# 2026-01\n\nRelease artifact, changelog, rollback, redacted manual runtime evidence, deployment, production readiness, and SemVer notes are recorded here.", "docs/versioning.md": "# Versioning\n\nSemantic versioning and SemVer define major, minor, and patch release policy.", "docs/upgrade.md": "# Upgrade\n\nUpgrade guidance references the migration registry, changelog, artifact, and rollback procedure.", "docs/deployment.md": "# Deployment\n\nDeployment documentation separates automated non-E2E checks from manual runtime validation.", "docs/migration-registry.md": "# Migration Registry\n\nMigration registry entries are linked from changelog and boilerplate version tracking notes.", }; function writeFixtureFile(rootDir, relativePath, content) { const absolutePath = path.join(rootDir, relativePath); fs.mkdirSync(path.dirname(absolutePath), { recursive: true }); fs.writeFileSync(absolutePath, content); } function createFixture(overrides = {}) { const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), "releaseDocs-release-docs-")); fixtures.push(rootDir); for (const [relativePath, content] of Object.entries(VALID_DOCS)) { if (overrides[relativePath] === null) continue; writeFixtureFile(rootDir, relativePath, overrides[relativePath] ?? content); } for (const [relativePath, content] of Object.entries(overrides)) { if (relativePath in VALID_DOCS || content === null) continue; writeFixtureFile(rootDir, relativePath, content); } return rootDir; } afterEach(() => { while (fixtures.length > 0) { fs.rmSync(fixtures.pop(), { recursive: true, force: true }); } }); describe("verify-release-docs", () => { it("passes for documentation that satisfies the Release Docs release documentation contract", () => { const rootDir = createFixture(); const result = verifyReleaseDocsReleaseDocs(rootDir); const output = formatReleaseDocsReleaseDocsResult(result); assert.deepEqual(result.failures, []); assert.match(output, /^PASS Release Docs release documentation verification/); assert.match(output, /9 release documentation files/); assert.match(output, /docs\/release-checklist\.md/); assert.match(output, /docs\/boilerplate-version-tracking\.md/); assert.match(output, /docs\/production-checklist\.md/); }); it("fails when a required documentation file is missing", () => { const rootDir = createFixture({ "docs/production-checklist.md": null }); const result = verifyReleaseDocsReleaseDocs(rootDir); assert.match(result.failures.join("\n"), /docs\/production-checklist\.md: missing required documentation file/); }); it("fails when a required section heading is missing", () => { const rootDir = createFixture({ "docs/release-checklist.md": VALID_DOCS["docs/release-checklist.md"].replace("## Artifacts and rollback", "## Artifacts"), }); const result = verifyReleaseDocsReleaseDocs(rootDir); assert.match(result.failures.join("\n"), /docs\/release-checklist\.md: missing required section heading "## Artifacts and rollback"/); }); it("fails when a local Markdown link is unresolved", () => { const rootDir = createFixture({ "docs/releases/README.md": "# Releases\n\nSee [missing](../missing-release-page.md).", }); const result = verifyReleaseDocsReleaseDocs(rootDir); assert.match(result.failures.join("\n"), /docs\/releases\/README\.md:\d+:\d+: unresolved local Markdown link "\.\.\/missing-release-page\.md"/); }); it("fails when forbidden placeholder markers are present", () => { const rootDir = createFixture({ "docs/boilerplate-version-tracking.md": `${VALID_DOCS["docs/boilerplate-version-tracking.md"]}\ntbd after release`, }); const result = verifyReleaseDocsReleaseDocs(rootDir); assert.match(result.failures.join("\n"), /docs\/boilerplate-version-tracking\.md:\d+:\d+: forbidden placeholder marker "tbd"/); }); it("fails when worker-group release evidence is incomplete", () => { const rootDir = createFixture({ "docs/production-checklist.md": VALID_DOCS["docs/production-checklist.md"] .replace(/worker-mail/g, "mail-worker") .replace(/WORKER_QUEUE_GROUP=mail/g, "WORKER_QUEUE_GROUP=email"), }); const result = verifyReleaseDocsReleaseDocs(rootDir); assert.match(result.failures.join("\n"), /required term group "worker-group-evidence"/); }); it("fails when release entry files contain placeholder markers", () => { const rootDir = createFixture({ "docs/releases/2026-02.md": "# 2026-02\n\nRelease evidence is TBD.", }); const result = verifyReleaseDocsReleaseDocs(rootDir); assert.match(result.failures.join("\n"), /docs\/releases\/2026-02\.md:\d+:\d+: forbidden placeholder marker "TBD"/); }); it("fails for automatic E2E commands but allows manual runtime notes", () => { const rootDir = createFixture({ "docs/production-checklist.md": `${VALID_DOCS["docs/production-checklist.md"]}\nManual Playwright review is recorded by release owners.\n\`pnpm exec playwright test\``, }); const result = verifyReleaseDocsReleaseDocs(rootDir); const failures = result.failures.join("\n"); assert.match(failures, /forbidden automatic browser\/E2E command \(playwright-test-command\)/); assert.doesNotMatch(failures, /Manual Playwright review/); }); it("fails when secret-shaped evidence is present", () => { const rootDir = createFixture({ "docs/release-checklist.md": `${VALID_DOCS["docs/release-checklist.md"]}\nObserved TOKEN=prod_live_secret_value_12345 in evidence.`, }); const result = verifyReleaseDocsReleaseDocs(rootDir); assert.match(result.failures.join("\n"), /forbidden secret-shaped example \(raw-credential-assignment\)/); }); it("fails when a webhook URL contains a token-like path", () => { const rootDir = createFixture({ "docs/production-checklist.md": `${VALID_DOCS["docs/production-checklist.md"]}\nWebhook was https://example.com/webhook/123e4567-e89b-12d3-a456-426614174000`, }); const result = verifyReleaseDocsReleaseDocs(rootDir); assert.match(result.failures.join("\n"), /forbidden secret-shaped example \(webhook-secret-url\)/); }); it("exposes the package script command", () => { const packageJson = JSON.parse(fs.readFileSync(path.resolve("package.json"), "utf8")); assert.equal(packageJson.scripts["verify:release-docs"], "node scripts/verify-release-docs.mjs"); }); it("exits non-zero from the CLI when checks fail", () => { const rootDir = createFixture({ "docs/release-checklist.md": "" }); assert.throws( () => execFileSync(process.execPath, [path.resolve("scripts/verify-release-docs.mjs")], { cwd: rootDir, encoding: "utf8", stdio: "pipe" }), /documentation file is empty/, ); }); });