"""``ibench`` — the Incumbent Benchmark command line interface. Commands: ibench validate [DOSSIER_DIR] schema + consistency check, exit 1 on errors ibench run [DOSSIER_DIR] -o RESULTS run the full benchmark, write scorecards ibench show EVENT_ID [DOSSIER_DIR] print one event's scorecard to stdout """ from __future__ import annotations import sys from pathlib import Path import click from .harness import DossierLoadError, load_dossier, load_dossiers, run_dossier from .scorecard import ( render_event_markdown, render_summary_markdown, results_to_json, ) DEFAULT_DOSSIER_DIR = Path("dossiers") @click.group() def main() -> None: """The Incumbent Benchmark: governance, benchmarked.""" @main.command() @click.argument( "dossier_dir", type=click.Path(exists=True, file_okay=False, path_type=Path), default=DEFAULT_DOSSIER_DIR, required=False, ) def validate(dossier_dir: Path) -> None: """Validate every dossier and machine-check its kernel analysis.""" errors = 0 paths = sorted(p for p in dossier_dir.glob("*.yaml") if not p.name.startswith("_")) if not paths: click.echo(f"no dossiers found in {dossier_dir}", err=True) sys.exit(1) for path in paths: try: dossier = load_dossier(path) except DossierLoadError as exc: click.echo(f"[ERROR] {exc}", err=True) errors += 1 continue result = run_dossier(dossier) for issue in result.issues: stream_err = issue.severity == "error" click.echo(f"[{issue.severity.upper()}] {issue.message}", err=stream_err) if stream_err: errors += 1 status = "ok" if result.ok else "FAILED" click.echo(f"{dossier.id}: {status} ({len(result.evaluations)} moves)") if errors: click.echo(f"\n{errors} error(s).", err=True) sys.exit(1) click.echo(f"\nAll {len(paths)} dossiers pass validation and consistency checks.") @main.command() @click.argument( "dossier_dir", type=click.Path(exists=True, file_okay=False, path_type=Path), default=DEFAULT_DOSSIER_DIR, required=False, ) @click.option( "--out", "-o", "out_dir", type=click.Path(file_okay=False, path_type=Path), default=Path("results"), show_default=True, help="Directory for scorecards and results.json.", ) def run(dossier_dir: Path, out_dir: Path) -> None: """Run the benchmark and write per-event + aggregate scorecards.""" try: dossiers = load_dossiers(dossier_dir) except DossierLoadError as exc: click.echo(f"[ERROR] {exc}", err=True) sys.exit(1) if not dossiers: click.echo(f"no dossiers found in {dossier_dir}", err=True) sys.exit(1) out_dir.mkdir(parents=True, exist_ok=True) results = [] failed = 0 for d in dossiers: r = run_dossier(d) results.append(r) (out_dir / f"{d.id}.md").write_text( render_event_markdown(r), encoding="utf-8" ) if not r.ok: failed += 1 for issue in r.issues: if issue.severity == "error": click.echo(f"[ERROR] {issue.message}", err=True) click.echo(f"{d.id}: {'ok' if r.ok else 'FAILED'}") (out_dir / "summary.md").write_text( render_summary_markdown(results), encoding="utf-8" ) (out_dir / "results.json").write_text(results_to_json(results), encoding="utf-8") click.echo( f"\nWrote {len(results)} scorecards, summary.md, and results.json to " f"{out_dir}/" ) if failed: click.echo(f"{failed} event(s) failed consistency checks.", err=True) sys.exit(1) @main.command() @click.argument("event_id") @click.argument( "dossier_dir", type=click.Path(exists=True, file_okay=False, path_type=Path), default=DEFAULT_DOSSIER_DIR, required=False, ) def show(event_id: str, dossier_dir: Path) -> None: """Print one event's scorecard to stdout.""" path = dossier_dir / f"{event_id}.yaml" if not path.exists(): click.echo(f"no dossier at {path}", err=True) sys.exit(1) try: dossier = load_dossier(path) except DossierLoadError as exc: click.echo(f"[ERROR] {exc}", err=True) sys.exit(1) result = run_dossier(dossier) click.echo(render_event_markdown(result)) if not result.ok: sys.exit(1) if __name__ == "__main__": main()