"""Unit tests for deterministic episode replay. Replay is the load-bearing wall of the exploit-to-test pipeline: a recorded exploit trace is only worth converting into a regression test if replaying it reproduces the original run exactly, and if tampering is detectable. """ from __future__ import annotations import copy import pytest from fable_selfplay.kernel import load_kernel from fable_selfplay.replay import replay_episode from .conftest import episode_dicts def _trace_key(episode: dict) -> str: for key in ("actions", "trace"): if key in episode: return key raise AssertionError(f"episode log has no action trace; keys={sorted(episode)}") def test_replay_reproduces_every_episode(kernel_v02, small_honest_result): kernel = load_kernel(kernel_v02) episodes = episode_dicts(small_honest_result) assert episodes for episode in episodes: outcome = replay_episode(kernel, episode) assert not getattr(outcome, "diverged", False), ( f"deterministic replay diverged for seed {episode.get('seed')}" ) def test_replay_reproduces_adversarial_episodes(kernel_v01, small_adversarial_result): kernel = load_kernel(kernel_v01) for episode in episode_dicts(small_adversarial_result): outcome = replay_episode(kernel, episode) assert not getattr(outcome, "diverged", False) def test_tampered_trace_is_flagged(kernel_v02, small_honest_result): kernel = load_kernel(kernel_v02) episode = copy.deepcopy(episode_dicts(small_honest_result)[0]) key = _trace_key(episode) assert episode[key], "expected a non-empty action trace" episode[key] = episode[key][:-1] # drop the final recorded action try: outcome = replay_episode(kernel, episode) except Exception: return # raising on a corrupted log is an acceptable contract assert getattr(outcome, "diverged", False), ( "a truncated trace must be reported as a divergence" ) def test_replay_under_wrong_kernel_is_not_silently_identical( kernel_v01, kernel_v02, small_adversarial_result): """An episode recorded under v0.1 must not replay identically under v0.2 if it exercised any rule that v0.2 changed — or must at minimum never be reported as both 'no divergence' AND have different final metrics.""" kernel = load_kernel(kernel_v02) for episode in episode_dicts(small_adversarial_result): try: outcome = replay_episode(kernel, episode) except Exception: continue # rejected outright: acceptable if getattr(outcome, "diverged", False): continue # flagged: acceptable # If it claims clean reproduction, hold it to that claim. final = getattr(outcome, "final_metrics", None) if final is not None and episode.get("metrics") is not None: assert final == pytest.approx(episode["metrics"]) or final == episode["metrics"]