"""Vector-suite generation tests: determinism, manifest integrity, coverage.""" from __future__ import annotations from fpcf import vectors as fpcf_vectors from helpers import ( DEFAULT_SEED, invalid_logs, invalid_single_ops, load_json, manifest_entries, tree_bytes, valid_logs, valid_single_ops, ) OTHER_SEED = "0bad" * 16 # --------------------------------------------------------------------------- # Determinism # --------------------------------------------------------------------------- def test_generation_is_byte_identical_for_same_seed(tmp_path): d1 = tmp_path / "a" d2 = tmp_path / "b" fpcf_vectors.generate_vectors(d1, seed=DEFAULT_SEED) fpcf_vectors.generate_vectors(d2, seed=DEFAULT_SEED) t1, t2 = tree_bytes(d1), tree_bytes(d2) assert t1.keys() == t2.keys() for rel in t1: assert t1[rel] == t2[rel], f"non-deterministic vector file: {rel}" def test_different_seed_changes_output(tmp_path): d1 = tmp_path / "a" d2 = tmp_path / "b" fpcf_vectors.generate_vectors(d1, seed=DEFAULT_SEED) fpcf_vectors.generate_vectors(d2, seed=OTHER_SEED) t1, t2 = tree_bytes(d1), tree_bytes(d2) # Keys are derived from the seed, so signatures (and therefore at least # some vector bytes) must differ. assert any(t1.get(rel) != t2.get(rel) for rel in t1) def test_generate_returns_the_manifest(tmp_path): out = tmp_path / "v" returned = fpcf_vectors.generate_vectors(out, seed=DEFAULT_SEED) on_disk = load_json(out / "manifest.json") assert returned == on_disk # --------------------------------------------------------------------------- # Manifest integrity # --------------------------------------------------------------------------- def test_manifest_format_marker(manifest): assert manifest["format"] == "fpcf-vectors/1" def test_manifest_records_the_seed(manifest): assert manifest["seed"] == DEFAULT_SEED def test_manifest_entries_are_well_formed(vectors_dir, manifest): names = set() for entry in manifest_entries(manifest): assert isinstance(entry["name"], str) and entry["name"] assert entry["name"] not in names, f"duplicate vector name {entry['name']}" names.add(entry["name"]) assert entry["expect"] in ("valid", "invalid") path = vectors_dir / entry["file"] assert path.is_file(), f"missing vector file: {entry['file']}" if entry["expect"] == "invalid": assert isinstance(entry["error"], str) assert entry["error"].startswith("ERR_"), entry["name"] def test_manifest_entries_parse_as_json(vectors_dir, manifest): for entry in manifest_entries(manifest): load_json(vectors_dir / entry["file"]) # must not raise # --------------------------------------------------------------------------- # Coverage: the suite must exercise both polarities of both vector kinds # --------------------------------------------------------------------------- def test_suite_covers_all_quadrants(vectors_dir, manifest): assert valid_single_ops(vectors_dir, manifest), "no valid operation vectors" assert invalid_single_ops(vectors_dir, manifest), "no invalid operation vectors" assert valid_logs(vectors_dir, manifest), "no valid log vectors" assert invalid_logs(vectors_dir, manifest), "no invalid log vectors" def test_suite_covers_every_operation_type(vectors_dir, manifest): """Every operation type in the spec must appear in a valid vector.""" expected = { "evidence-ingest", "claim-assert", "correction", "refutation", "permission-grant", "revocation", "inference-call", } seen = set() for _, op in valid_single_ops(vectors_dir, manifest): seen.add(op["type"]) for _, payload in valid_logs(vectors_dir, manifest): for op in payload["ops"]: seen.add(op["type"]) missing = expected - seen assert not missing, f"vector suite missing operation types: {sorted(missing)}"