#!/usr/bin/env node import crypto from 'node:crypto'; import fs from 'node:fs'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); const repoRoot = path.resolve(__dirname, '..'); const DEFAULT_REQUIRED_GATE_IDS = [ 'environment', 'format', 'typecheck', 'lint', 'unit', 'build', 'bundle-budget', 'e2e', ]; const HELP = ` Mechanica validation evidence verifier Usage: node scripts/verify-validation-evidence.mjs [bundle-dir] [options] If bundle-dir is omitted, the verifier reads .validation/latest-evidence-path.txt, then falls back to the newest directory under .validation/evidence. Options: --allow-failed Allow failed validation status. Intended only for debugging bundles. --allow-missing-playwright-report Do not fail if E2E passed but no browser artifact directory was captured. --help Show this help text. `; function isTruthy(value) { return ['1', 'true', 'yes', 'y', 'on'].includes(String(value ?? '').toLowerCase()); } function parseArgs(argv) { const options = { bundleDir: null, allowFailed: isTruthy(process.env.MECHANICA_VERIFY_ALLOW_FAILED), allowMissingPlaywrightReport: isTruthy( process.env.MECHANICA_VERIFY_ALLOW_MISSING_PLAYWRIGHT_REPORT, ), }; for (let index = 0; index < argv.length; index += 1) { const arg = argv[index]; if (arg === '--help' || arg === '-h') { console.log(HELP.trim()); process.exit(0); } if (arg === '--allow-failed') { options.allowFailed = true; continue; } if (arg === '--allow-missing-playwright-report') { options.allowMissingPlaywrightReport = true; continue; } if (arg.startsWith('-')) { throw new Error(`Unknown option: ${arg}`); } if (!options.bundleDir) { options.bundleDir = arg; continue; } throw new Error(`Unexpected positional argument: ${arg}`); } return options; } function posixPath(value) { return value.split(path.sep).join('/'); } function relativeToRepo(filePath) { const relative = path.relative(repoRoot, filePath); return relative && !relative.startsWith('..') && !path.isAbsolute(relative) ? posixPath(relative) : filePath; } function resolveBundleDir(input) { if (input) { const resolved = path.resolve(repoRoot, input); if (!fs.existsSync(resolved) || !fs.statSync(resolved).isDirectory()) { throw new Error(`Evidence bundle directory does not exist: ${relativeToRepo(resolved)}`); } return resolved; } const latestPointer = path.join(repoRoot, '.validation', 'latest-evidence-path.txt'); if (fs.existsSync(latestPointer)) { const pointerValue = fs.readFileSync(latestPointer, 'utf8').trim(); if (pointerValue) { const resolved = path.resolve(repoRoot, pointerValue); if (fs.existsSync(resolved) && fs.statSync(resolved).isDirectory()) { return resolved; } } } const evidenceRoot = path.join(repoRoot, '.validation', 'evidence'); if (fs.existsSync(evidenceRoot) && fs.statSync(evidenceRoot).isDirectory()) { const candidates = fs .readdirSync(evidenceRoot, { withFileTypes: true }) .filter((entry) => entry.isDirectory()) .map((entry) => path.join(evidenceRoot, entry.name)) .sort((a, b) => fs.statSync(b).mtimeMs - fs.statSync(a).mtimeMs); if (candidates[0]) { return candidates[0]; } } throw new Error( 'No evidence bundle directory was provided and no default bundle could be found under .validation.', ); } function readJson(filePath) { return JSON.parse(fs.readFileSync(filePath, 'utf8')); } function writeJson(filePath, value) { fs.mkdirSync(path.dirname(filePath), { recursive: true }); fs.writeFileSync(`${filePath}.tmp`, `${JSON.stringify(value, null, 2)}\n`, 'utf8'); fs.renameSync(`${filePath}.tmp`, filePath); } function writeText(filePath, value) { fs.mkdirSync(path.dirname(filePath), { recursive: true }); fs.writeFileSync(`${filePath}.tmp`, value, 'utf8'); fs.renameSync(`${filePath}.tmp`, filePath); } function sha256File(filePath) { return new Promise((resolve, reject) => { const hash = crypto.createHash('sha256'); const stream = fs.createReadStream(filePath); stream.on('data', (chunk) => hash.update(chunk)); stream.on('error', reject); stream.on('end', () => resolve(hash.digest('hex'))); }); } function parseChecksums(filePath) { if (!fs.existsSync(filePath)) { return null; } const checksums = new Map(); const lines = fs.readFileSync(filePath, 'utf8').split('\n'); for (const line of lines) { const trimmed = line.trim(); if (!trimmed) { continue; } const match = /^([a-fA-F0-9]{64})\s+\*?(.+)$/.exec(trimmed); if (match) { checksums.set(match[2], match[1].toLowerCase()); } } return checksums; } function hasFiles(dirPath) { if (!fs.existsSync(dirPath) || !fs.statSync(dirPath).isDirectory()) { return false; } const entries = fs.readdirSync(dirPath, { withFileTypes: true }); for (const entry of entries) { const absolutePath = path.join(dirPath, entry.name); if (entry.isFile()) { return true; } if (entry.isDirectory() && hasFiles(absolutePath)) { return true; } } return false; } function loadRequiredJson(bundleDir, relativePath, issues) { const filePath = path.join(bundleDir, relativePath); if (!fs.existsSync(filePath)) { issues.push(`Missing required file: ${relativePath}`); return null; } try { return readJson(filePath); } catch (error) { issues.push( `Invalid JSON in ${relativePath}: ${error instanceof Error ? error.message : String(error)}`, ); return null; } } async function verifyManifestFiles(bundleDir, manifest, issues, warnings) { if (!manifest || !Array.isArray(manifest.files)) { issues.push('evidence-manifest.json does not contain a files array.'); return { checkedFiles: 0 }; } const checksumFile = path.join(bundleDir, 'checksums.sha256'); const checksums = parseChecksums(checksumFile); if (!checksums) { warnings.push('checksums.sha256 was not found; relying on evidence-manifest.json hashes only.'); } let checkedFiles = 0; for (const entry of manifest.files) { if (!entry || typeof entry.path !== 'string' || typeof entry.sha256 !== 'string') { issues.push('Manifest contains an invalid file entry.'); continue; } const absolutePath = path.join(bundleDir, entry.path); if (!fs.existsSync(absolutePath) || !fs.statSync(absolutePath).isFile()) { issues.push(`Manifest file is missing: ${entry.path}`); continue; } const stat = fs.statSync(absolutePath); if (Number.isFinite(entry.bytes) && stat.size !== entry.bytes) { issues.push(`Size mismatch for ${entry.path}: expected ${entry.bytes}, got ${stat.size}`); } const actualHash = await sha256File(absolutePath); checkedFiles += 1; if (actualHash !== entry.sha256.toLowerCase()) { issues.push(`SHA-256 mismatch for ${entry.path}`); } if (checksums) { const checksumHash = checksums.get(entry.path); if (!checksumHash) { issues.push(`checksums.sha256 is missing an entry for ${entry.path}`); } else if (checksumHash !== entry.sha256.toLowerCase()) { issues.push(`checksums.sha256 disagrees with evidence-manifest.json for ${entry.path}`); } } } return { checkedFiles }; } function verifyValidationSummary(summary, options, issues, warnings) { if (!summary) { return { gatesById: new Map(), requiredGateIds: DEFAULT_REQUIRED_GATE_IDS }; } if (summary.status !== 'passed' && !options.allowFailed) { issues.push(`Validation summary status is not passed: ${summary.status || 'unknown'}`); } if (!Array.isArray(summary.gates)) { issues.push('validation-summary.json does not contain a gates array.'); return { gatesById: new Map(), requiredGateIds: DEFAULT_REQUIRED_GATE_IDS }; } const gatesById = new Map(summary.gates.map((gate) => [gate.id, gate])); const requiredGateIds = Array.isArray(summary.requiredGateIds) && summary.requiredGateIds.length > 0 ? summary.requiredGateIds : DEFAULT_REQUIRED_GATE_IDS; for (const gateId of requiredGateIds) { const gate = gatesById.get(gateId); if (!gate) { issues.push(`Required validation gate is missing from summary: ${gateId}`); continue; } if (gate.status !== 'passed') { issues.push(`Required validation gate did not pass: ${gateId} (${gate.status})`); } } for (const gate of summary.gates) { if (!gate || !gate.id) { continue; } if (!requiredGateIds.includes(gate.id) && gate.status === 'failed') { issues.push(`Optional validation gate failed: ${gate.id}`); } if (!requiredGateIds.includes(gate.id) && gate.status === 'skipped') { warnings.push(`Optional validation gate was skipped: ${gate.id}`); } } return { gatesById, requiredGateIds }; } function verifyBundleBudget(bundleBudget, issues) { if (!bundleBudget) { return; } if (bundleBudget.status !== 'passed') { issues.push(`Bundle budget status is not passed: ${bundleBudget.status || 'unknown'}`); } if (Array.isArray(bundleBudget.checks)) { for (const check of bundleBudget.checks) { if (check && check.passed === false) { issues.push(`Bundle budget check failed: ${check.label || check.id || 'unknown check'}`); } } } } function verifyGateLogs(bundleDir, summary, issues) { if (!summary || !Array.isArray(summary.gates)) { return; } for (const gate of summary.gates) { if (!gate || !gate.id || gate.status === 'skipped') { continue; } const expectedLog = path.join(bundleDir, 'logs', `${gate.id}.log`); if (!fs.existsSync(expectedLog)) { issues.push(`Missing per-gate log: logs/${gate.id}.log`); } } } function verifyPlaywrightArtifacts(bundleDir, gatesById, options, issues, warnings) { const e2eGate = gatesById.get('e2e'); if (!e2eGate || e2eGate.status !== 'passed') { return; } const artifactDirs = ['playwright-report', 'playwright-results', 'test-results', 'blob-report']; const presentArtifactDirs = artifactDirs.filter((dir) => hasFiles(path.join(bundleDir, dir))); if (presentArtifactDirs.length === 0 && !options.allowMissingPlaywrightReport) { issues.push( 'E2E gate passed, but no Playwright report/results artifact directory was captured in the evidence bundle.', ); return; } if (!fs.existsSync(path.join(bundleDir, 'playwright-report', 'index.html'))) { warnings.push( 'playwright-report/index.html is missing. Trace/results artifacts may still be present, but the HTML report was not captured.', ); } } function verifyRequiredFiles(bundleDir, issues) { const requiredPaths = [ 'validation-summary.md', 'validation-summary.json', 'bundle-budget.md', 'bundle-budget.json', 'environment.md', 'logs', 'evidence-manifest.json', 'checksums.sha256', 'EVIDENCE_README.md', ]; for (const relativePath of requiredPaths) { const absolutePath = path.join(bundleDir, relativePath); if (!fs.existsSync(absolutePath)) { issues.push(`Missing required evidence artifact: ${relativePath}`); } } } function verificationMarkdown(result) { const gateRows = result.gates.map((gate) => { return `| \`${gate.id}\` | ${String(gate.status).toUpperCase()} | ${gate.durationMs ?? 0}ms |`; }); return [ '# Mechanica Evidence Verification Summary', '', `Generated: ${result.generatedAt}`, `Evidence bundle: \`${result.bundleDirectory}\``, `Status: **${result.status.toUpperCase()}**`, `Manifest files checked: ${result.checkedFiles}`, '', '## Gate results', '', '| Gate | Status | Duration |', '| --- | --- | ---: |', ...gateRows, '', '## Issues', '', result.issues.length === 0 ? 'No blocking issues found.' : result.issues.map((issue) => `- ${issue}`).join('\n'), '', '## Warnings', '', result.warnings.length === 0 ? 'No warnings.' : result.warnings.map((warning) => `- ${warning}`).join('\n'), '', ].join('\n'); } async function main() { const options = parseArgs(process.argv.slice(2)); const bundleDir = resolveBundleDir(options.bundleDir); const issues = []; const warnings = []; verifyRequiredFiles(bundleDir, issues); const manifest = loadRequiredJson(bundleDir, 'evidence-manifest.json', issues); const summary = loadRequiredJson(bundleDir, 'validation-summary.json', issues); const bundleBudget = loadRequiredJson(bundleDir, 'bundle-budget.json', issues); if (manifest && manifest.validationStatus !== 'passed' && !options.allowFailed) { issues.push(`Evidence manifest validationStatus is not passed: ${manifest.validationStatus}`); } const { gatesById, requiredGateIds } = verifyValidationSummary(summary, options, issues, warnings); verifyBundleBudget(bundleBudget, issues); verifyGateLogs(bundleDir, summary, issues); verifyPlaywrightArtifacts(bundleDir, gatesById, options, issues, warnings); const { checkedFiles } = await verifyManifestFiles(bundleDir, manifest, issues, warnings); const result = { schemaVersion: 1, project: 'Mechanica', generatedAt: new Date().toISOString(), bundleDirectory: relativeToRepo(bundleDir), status: issues.length === 0 ? 'passed' : 'failed', checkedFiles, requiredGateIds, gates: Array.isArray(summary?.gates) ? summary.gates : [], issues, warnings, }; writeJson(path.join(bundleDir, 'verification-summary.json'), result); writeText(path.join(bundleDir, 'verification-summary.md'), verificationMarkdown(result)); if (result.status === 'passed') { console.log('✓ Mechanica validation evidence verified.'); console.log(`Bundle: ${relativeToRepo(bundleDir)}`); console.log(`Files checked: ${checkedFiles}`); if (warnings.length > 0) { console.log(`Warnings: ${warnings.length}`); } process.exitCode = 0; } else { console.error('✗ Mechanica validation evidence verification failed.'); console.error(`Bundle: ${relativeToRepo(bundleDir)}`); for (const issue of issues) { console.error(`- ${issue}`); } console.error(`Summary: ${relativeToRepo(path.join(bundleDir, 'verification-summary.md'))}`); process.exitCode = 1; } } main().catch((error) => { console.error(error instanceof Error ? error.stack || error.message : String(error)); process.exitCode = 1; });