"""Conformance runner: execute a vector suite against the reference pipeline. For each vector: 1. Decode the context ops and append them to a fresh ``OpLog``; every context op must be accepted (a rejected context op is a suite defect and counts as a failure). 2. Attempt to append the op under test. 3. Compare the outcome to ``expected``: * ``{"op_id": ...}`` -> must be accepted with exactly that op id. * ``{"error": code}`` -> must be rejected with exactly that error code. """ import base64 from dataclasses import dataclass, field from typing import List from .errors import FpcfError from .log import OpLog from .vectors import load_suite @dataclass class Result: name: str kind: str passed: bool detail: str = "" @dataclass class Report: results: List[Result] = field(default_factory=list) @property def total(self) -> int: return len(self.results) @property def passed(self) -> int: return sum(1 for r in self.results if r.passed) @property def failed(self) -> int: return self.total - self.passed @property def ok(self) -> bool: return self.failed == 0 and self.total > 0 def failures(self) -> List[Result]: return [r for r in self.results if not r.passed] def to_dict(self) -> dict: return { "total": self.total, "passed": self.passed, "failed": self.failed, "ok": self.ok, "results": [ { "name": r.name, "kind": r.kind, "passed": r.passed, "detail": r.detail, } for r in self.results ], } def summary(self) -> str: return "%d/%d vectors passed (%d failed)" % (self.passed, self.total, self.failed) def run_vector(vec: dict) -> Result: name = vec["name"] kind = vec["kind"] expected = vec["expected"] log = OpLog() for i, b64 in enumerate(vec.get("context_b64", [])): try: log.append_raw(base64.b64decode(b64)) except FpcfError as exc: return Result(name, kind, False, "context op %d rejected: %s" % (i, exc)) try: oid = log.append_raw(base64.b64decode(vec["raw_b64"])) error_code = None except FpcfError as exc: oid = None error_code = exc.code error_detail = str(exc) if "op_id" in expected: if error_code is not None: return Result( name, kind, False, "expected accept, got %s" % error_detail ) if oid != expected["op_id"]: return Result( name, kind, False, "op id mismatch: expected %s, got %s" % (expected["op_id"], oid), ) return Result(name, kind, True) if "error" in expected: if error_code is None: return Result( name, kind, False, "expected %s, but op was accepted as %s" % (expected["error"], oid) ) if error_code != expected["error"]: return Result( name, kind, False, "expected %s, got %s" % (expected["error"], error_detail), ) return Result(name, kind, True) return Result(name, kind, False, "vector has no usable 'expected' clause") def run_suite(vectors: List[dict]) -> Report: report = Report() for vec in vectors: report.results.append(run_vector(vec)) return report def run_dir(dir_path) -> Report: return run_suite(load_suite(dir_path))