diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..731b386 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,26 @@ +name: CI + +on: + push: + pull_request: + +jobs: + test: + strategy: + matrix: + python-version: ["3.11", "3.12", "3.13"] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: pip + - run: python -m pip install -e ".[dev]" + - run: python -m pytest + - name: Install media verifier + run: sudo apt-get update && sudo apt-get install -y ffmpeg + - name: Verify source-pinned demo + run: python scripts/verify_seed_integrity_demo.py diff --git a/README.md b/README.md index 21f0dee..da7df1d 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,14 @@ We identify *loop-action bias* as a mechanistic failure mode of outcome-only cre | Real ALFWorld, Haiku (scorer diff.) | 13.3% | **34.8%** | = branch-aware | | Real ALFWorld, Qwen-7B (scorer diff.) | 3.4% | **31.8%** | = branch-aware | +## Actual workflow demo + +![CreditLab stochastic seed integrity demo](media/demo/seed-integrity.gif) + +[Watch the MP4](media/demo/seed-integrity.mp4) · [Inspect the capture receipt](media/demo/capture.json) + +This provider-free demo executes the checked-in `stochastic/v1` manifest through the real environment adapter. It shows that one seed is repeatable while seeds 0 and 1 produce distinct stochastic paths. The capture is pinned to its exact runtime source commit and guarded by `scripts/verify_seed_integrity_demo.py`. It does not exercise a language model or validate the paper's performance claims. + ## Reproducing the Results ### Prerequisites diff --git a/media/demo/capture.json b/media/demo/capture.json new file mode 100644 index 0000000..b8c4849 --- /dev/null +++ b/media/demo/capture.json @@ -0,0 +1,133 @@ +{ + "artifacts": { + "seed-integrity.gif": { + "bytes": 252820, + "sha256": "d3eb157c762bb3c4677ea37fcb66916e21837ed5b48c9a95fe51cf5aafbf00d3" + }, + "seed-integrity.mp4": { + "bytes": 164354, + "sha256": "45044b69f3d3d0650361d4c1e515d882bc2273106fb03ce7b9b4aa9ab05e2988" + } + }, + "benchmark": "stochastic/v1", + "cross_seed_paths_distinct": true, + "same_seed_replay_matches": true, + "schema_version": 1, + "scope": "provider-free stochastic manifest fixture; no model or paper-performance claim", + "seeds": [ + 0, + 1 + ], + "source_blobs": { + "benchmarks/stochastic/v1/tasks.yaml": "e2826242df4cc340b67eba5c05b700059e5a806c", + "scripts/generate_seed_integrity_demo.py": "81e1cefd3b0d5f0f0f4da68ce6d0e31cc5d18a8a", + "src/creditlab/analysis/sweep.py": "1bf34600f9c4dd140702bc3370ac903dd4383a34", + "src/creditlab/core/registry.py": "0bd254dd96460beea41035fb2c0cf9eb8d0f3b72", + "src/creditlab/envs/manifest_graph.py": "6d7f4b70422001e97ab84869df060d4e24f32b24" + }, + "source_commit": "b39ab74c07cc56f30e96fd8d955170e012167356", + "source_tree": "ffc8450ecf5b4a271e3ca3fc04ae1f45f86aeeb9", + "task_id": "stoch_gate_train_01", + "traces": { + "0": [ + { + "action": "unlock_gate", + "from": "start", + "observation": "The lock resists. The gate stays shut.", + "terminal": false, + "to": "start", + "turn": 1 + }, + { + "action": "unlock_gate", + "from": "start", + "observation": "The lock resists. The gate stays shut.", + "terminal": false, + "to": "start", + "turn": 2 + }, + { + "action": "unlock_gate", + "from": "start", + "observation": "The first gate opens.", + "terminal": false, + "to": "gate_2", + "turn": 3 + }, + { + "action": "unlock_gate", + "from": "gate_2", + "observation": "The second gate opens.", + "terminal": false, + "to": "gate_3", + "turn": 4 + }, + { + "action": "unlock_gate", + "from": "gate_3", + "observation": "The third gate opens.", + "terminal": false, + "to": "gate_4", + "turn": 5 + }, + { + "action": "unlock_gate", + "from": "gate_4", + "observation": "The final gate opens. Task complete.", + "terminal": true, + "to": "success", + "turn": 6 + } + ], + "1": [ + { + "action": "unlock_gate", + "from": "start", + "observation": "The first gate opens.", + "terminal": false, + "to": "gate_2", + "turn": 1 + }, + { + "action": "unlock_gate", + "from": "gate_2", + "observation": "The lock resists again.", + "terminal": false, + "to": "gate_2", + "turn": 2 + }, + { + "action": "unlock_gate", + "from": "gate_2", + "observation": "The lock resists again.", + "terminal": false, + "to": "gate_2", + "turn": 3 + }, + { + "action": "unlock_gate", + "from": "gate_2", + "observation": "The second gate opens.", + "terminal": false, + "to": "gate_3", + "turn": 4 + }, + { + "action": "unlock_gate", + "from": "gate_3", + "observation": "The third gate opens.", + "terminal": false, + "to": "gate_4", + "turn": 5 + }, + { + "action": "unlock_gate", + "from": "gate_4", + "observation": "The final gate opens. Task complete.", + "terminal": true, + "to": "success", + "turn": 6 + } + ] + } +} diff --git a/media/demo/seed-integrity.gif b/media/demo/seed-integrity.gif new file mode 100644 index 0000000..e6fbeee Binary files /dev/null and b/media/demo/seed-integrity.gif differ diff --git a/media/demo/seed-integrity.mp4 b/media/demo/seed-integrity.mp4 new file mode 100644 index 0000000..777ed38 Binary files /dev/null and b/media/demo/seed-integrity.mp4 differ diff --git a/pyproject.toml b/pyproject.toml index 6eb033d..f54ecf3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,6 +27,9 @@ wandb = [ paper = [ "matplotlib>=3.8,<4", ] +demo = [ + "pillow>=10,<13", +] alfworld = [ "alfworld==0.4.2", ] diff --git a/scripts/generate_seed_integrity_demo.py b/scripts/generate_seed_integrity_demo.py new file mode 100644 index 0000000..81e1cef --- /dev/null +++ b/scripts/generate_seed_integrity_demo.py @@ -0,0 +1,262 @@ +#!/usr/bin/env python3 +"""Generate the source-backed stochastic seed integrity demo. + +The displayed paths are executed through the repository's real manifest adapter. +No network service, model provider, or precomputed benchmark result is used. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import subprocess +import sys +from pathlib import Path + +from PIL import Image, ImageDraw, ImageFont + +REPO_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO_ROOT / "src")) + +from creditlab.core.config import load_experiment_config # noqa: E402 +from creditlab.core.models import ActionEvent # noqa: E402 +from creditlab.core.registry import create_env # noqa: E402 + +WIDTH, HEIGHT = 1280, 720 +FPS = 15 +DURATION_SECONDS = 12 +SEEDS = (0, 1) +SOURCE_INPUTS = ( + "src/creditlab/analysis/sweep.py", + "src/creditlab/core/registry.py", + "src/creditlab/envs/manifest_graph.py", + "benchmarks/stochastic/v1/tasks.yaml", + "scripts/generate_seed_integrity_demo.py", +) + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def git(*args: str) -> str: + return subprocess.check_output(["git", *args], cwd=REPO_ROOT, text=True).strip() + + +def font(size: int, bold: bool = False) -> ImageFont.FreeTypeFont: + candidates = ( + "/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf" if bold else "/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf", + "/usr/share/fonts/truetype/liberation2/LiberationSans-Bold.ttf" if bold else "/usr/share/fonts/truetype/liberation2/LiberationSans-Regular.ttf", + ) + for candidate in candidates: + if Path(candidate).exists(): + return ImageFont.truetype(candidate, size) + return ImageFont.load_default() + + +def collect_trace(seed: int) -> list[dict[str, object]]: + config = load_experiment_config(REPO_ROOT / "configs" / "stochastic_paper_local.yaml") + config.env.config["seed"] = seed + environment = create_env(config) + task = environment.list_tasks("train")[0] + state = environment.reset(task) + trace: list[dict[str, object]] = [] + for turn in range(8): + action_name = str(state.payload["oracle_action"]) + action = ActionEvent(action_id=f"seed-{seed}-{turn}", action_type="tool", content=action_name) + next_state, observation, terminal = environment.step(task, state, action) + trace.append( + { + "turn": turn + 1, + "from": state.payload["raw_node"], + "action": action_name, + "to": next_state.payload["raw_node"], + "observation": observation, + "terminal": terminal, + } + ) + state = next_state + if terminal: + break + return trace + + +def rounded(draw: ImageDraw.ImageDraw, box: tuple[int, int, int, int], fill: str, radius: int = 22, outline: str | None = None) -> None: + draw.rounded_rectangle(box, radius=radius, fill=fill, outline=outline, width=2 if outline else 1) + + +def gradient() -> Image.Image: + # A low-resolution field scaled once is visually smooth and keeps reproducible + # media generation comfortably fast on CPU-only machines. + sample_width, sample_height = 80, 45 + image = Image.new("RGB", (sample_width, sample_height), "#07111f") + pixels = image.load() + for y in range(sample_height): + for x in range(sample_width): + glow = max(0.0, 1.0 - (((x - 66) / 44) ** 2 + ((y - 6) / 33) ** 2)) + pixels[x, y] = (7 + int(10 * glow), 17 + int(22 * glow), 31 + int(37 * glow)) + return image.resize((WIDTH, HEIGHT), Image.Resampling.BICUBIC) + + +BACKGROUND = gradient() + + +def draw_header(draw: ImageDraw.ImageDraw, kicker: str, title: str, subtitle: str) -> None: + draw.text((74, 55), kicker.upper(), font=font(18, True), fill="#5eead4") + title_size = 46 + title_font = font(title_size, True) + while draw.textbbox((0, 0), title, font=title_font)[2] > WIDTH - 148 and title_size > 30: + title_size -= 2 + title_font = font(title_size, True) + draw.text((74, 90), title, font=title_font, fill="#f8fafc") + draw.text((76, 151), subtitle, font=font(21), fill="#a8b7cb") + + +def draw_pipeline(draw: ImageDraw.ImageDraw, active: int) -> None: + labels = ("Sweep seed", "Environment RNG", "Policy fallback", "Recorded path") + for index, label in enumerate(labels): + x = 92 + index * 292 + color = "#2dd4bf" if index <= active else "#1e3448" + rounded(draw, (x, 248, x + 218, 330), "#0d1d2e", 18, color) + draw.text((x + 24, 269), f"0{index + 1}", font=font(15, True), fill=color) + draw.text((x + 24, 294), label, font=font(17, True), fill="#f8fafc") + if index < len(labels) - 1: + draw.line((x + 226, 289, x + 278, 289), fill=color, width=4) + draw.polygon([(x + 278, 289), (x + 266, 282), (x + 266, 296)], fill=color) + + +def draw_trace_card(draw: ImageDraw.ImageDraw, x: int, y: int, seed: int, trace: list[dict[str, object]], visible: int, accent: str) -> None: + rounded(draw, (x, y, x + 535, y + 350), "#0b1a2a", 24, "#20384c") + draw.text((x + 30, y + 24), f"SEED {seed}", font=font(19, True), fill=accent) + draw.text((x + 30, y + 56), "actual manifest transitions", font=font(16), fill="#91a6bb") + for index, step in enumerate(trace[:visible]): + row_y = y + 102 + index * 39 + succeeded = step["from"] != step["to"] + dot = accent if succeeded else "#fb7185" + draw.ellipse((x + 31, row_y + 4, x + 43, row_y + 16), fill=dot) + text = f"{step['from']} → {step['to']}" + draw.text((x + 57, row_y), text, font=font(17, True), fill="#e7eef7") + draw.text((x + 337, row_y + 1), "advance" if succeeded else "retry", font=font(15), fill=dot) + if visible >= len(trace): + draw.text((x + 30, y + 316), f"{len(trace)} turns • terminal success", font=font(16, True), fill="#c7d5e3") + + +def make_frame(frame_index: int, traces: dict[int, list[dict[str, object]]], source_sha: str) -> Image.Image: + seconds = frame_index / FPS + image = BACKGROUND.copy() + draw = ImageDraw.Draw(image) + if seconds < 2.2: + draw_header(draw, "CreditLab · provider-free fixture", "Seeds now reach every random branch", "One sweep seed controls both environment transitions and fallback policy choices.") + draw_pipeline(draw, min(3, int(seconds / 0.5))) + rounded(draw, (92, 401, 1188, 582), "#0b1a2a", 24, "#20384c") + draw.text((124, 435), "Determinism contract", font=font(20, True), fill="#5eead4") + draw.text((124, 480), "same seed → same path", font=font(31, True), fill="#f8fafc") + draw.text((657, 480), "different seed → independent path", font=font(31, True), fill="#f8fafc") + elif seconds < 8.7: + draw_header(draw, "Executed against stochastic/v1", "Two reproducible paths, not two labels", "Each row below is produced by GraphManifestEnv.step() at the pinned source.") + progress = (seconds - 2.2) / 6.5 + visible0 = min(len(traces[0]), max(1, int(progress * (len(traces[0]) + 1)))) + visible1 = min(len(traces[1]), max(1, int(progress * (len(traces[1]) + 1)))) + draw_trace_card(draw, 74, 222, 0, traces[0], visible0, "#38bdf8") + draw_trace_card(draw, 671, 222, 1, traces[1], visible1, "#a78bfa") + else: + draw_header(draw, "Verified receipt", "Seeded paths are reproducible and independent.", "The demo uses local fixture data only—no model, provider, or paper-performance claim.") + rounded(draw, (74, 232, 1206, 555), "#0b1a2a", 24, "#20384c") + facts = ( + ("seed 0 replay", "MATCH", "#38bdf8"), + ("seed 1 replay", "MATCH", "#a78bfa"), + ("cross-seed paths", "DISTINCT", "#5eead4"), + ("source", source_sha[:12], "#fbbf24"), + ) + for index, (label, value, color) in enumerate(facts): + x = 107 + (index % 2) * 545 + y = 271 + (index // 2) * 128 + draw.text((x, y), label.upper(), font=font(16, True), fill="#91a6bb") + draw.text((x, y + 33), value, font=font(36, True), fill=color) + draw.text((76, 638), "ACTUAL WORKFLOW • SOURCE-PINNED • OFFLINE", font=font(16, True), fill="#5eead4") + draw.text((1108, 665), "creditlab", font=font(17, True), fill="#627b91") + return image + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--output-dir", type=Path, default=REPO_ROOT / "media" / "demo") + parser.add_argument("--source-sha", default=None, help="Commit containing the captured runtime sources") + parser.add_argument("--reuse-frames", action="store_true", help="Reuse a complete existing frame set") + args = parser.parse_args() + output_dir = args.output_dir.resolve() + output_dir.mkdir(parents=True, exist_ok=True) + source_sha = args.source_sha or git("rev-parse", "HEAD") + git("cat-file", "-e", f"{source_sha}^{{commit}}") + + traces = {seed: collect_trace(seed) for seed in SEEDS} + replays = {seed: collect_trace(seed) for seed in SEEDS} + if traces != replays: + raise RuntimeError("same-seed replay was not deterministic") + if traces[0] == traces[1]: + raise RuntimeError("demo seeds unexpectedly produced identical paths") + + frames_dir = output_dir / "frames" + frames_dir.mkdir(exist_ok=True) + expected_frames = DURATION_SECONDS * FPS + existing_frames = sorted(frames_dir.glob("frame-*.png")) + if not args.reuse_frames or len(existing_frames) != expected_frames: + for frame in existing_frames: + frame.unlink() + for frame_index in range(expected_frames): + make_frame(frame_index, traces, source_sha).save(frames_dir / f"frame-{frame_index:04d}.png") + + mp4_path = output_dir / "seed-integrity.mp4" + gif_path = output_dir / "seed-integrity.gif" + subprocess.run( + [ + "ffmpeg", "-y", "-framerate", str(FPS), "-i", str(frames_dir / "frame-%04d.png"), + "-c:v", "libx264", "-pix_fmt", "yuv420p", "-movflags", "+faststart", "-crf", "20", str(mp4_path), + ], + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + subprocess.run( + [ + "ffmpeg", "-y", "-framerate", str(FPS), "-i", str(frames_dir / "frame-%04d.png"), "-vf", + "fps=10,scale=960:-1:flags=lanczos,split[s0][s1];[s0]palettegen=max_colors=160[p];[s1][p]paletteuse=dither=bayer", + "-loop", "0", str(gif_path), + ], + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + + source_blobs = {path: git("rev-parse", f"{source_sha}:{path}") for path in SOURCE_INPUTS} + receipt = { + "schema_version": 1, + "scope": "provider-free stochastic manifest fixture; no model or paper-performance claim", + "source_commit": source_sha, + "source_tree": git("rev-parse", f"{source_sha}^{{tree}}"), + "source_blobs": source_blobs, + "benchmark": "stochastic/v1", + "task_id": "stoch_gate_train_01", + "seeds": list(SEEDS), + "same_seed_replay_matches": traces == replays, + "cross_seed_paths_distinct": traces[0] != traces[1], + "traces": {str(seed): trace for seed, trace in traces.items()}, + "artifacts": { + mp4_path.name: {"sha256": sha256(mp4_path), "bytes": mp4_path.stat().st_size}, + gif_path.name: {"sha256": sha256(gif_path), "bytes": gif_path.stat().st_size}, + }, + } + (output_dir / "capture.json").write_text(json.dumps(receipt, indent=2, sort_keys=True) + "\n", encoding="utf-8") + for frame in frames_dir.glob("*.png"): + frame.unlink() + frames_dir.rmdir() + print(json.dumps(receipt, indent=2, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/scripts/verify_seed_integrity_demo.py b/scripts/verify_seed_integrity_demo.py new file mode 100644 index 0000000..316868d --- /dev/null +++ b/scripts/verify_seed_integrity_demo.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +"""Verify media hashes, decoding, and captured source blobs.""" + +from __future__ import annotations + +import hashlib +import json +import subprocess +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +DEMO_DIR = REPO_ROOT / "media" / "demo" + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def git(*args: str) -> str: + return subprocess.check_output(["git", *args], cwd=REPO_ROOT, text=True).strip() + + +def main() -> None: + receipt = json.loads((DEMO_DIR / "capture.json").read_text(encoding="utf-8")) + source_commit = receipt["source_commit"] + git("cat-file", "-e", f"{source_commit}^{{commit}}") + assert receipt["source_tree"] == git("rev-parse", f"{source_commit}^{{tree}}") + for path, blob in receipt["source_blobs"].items(): + assert blob == git("rev-parse", f"{source_commit}:{path}"), path + assert receipt["same_seed_replay_matches"] is True + assert receipt["cross_seed_paths_distinct"] is True + for name, metadata in receipt["artifacts"].items(): + path = DEMO_DIR / name + assert path.stat().st_size == metadata["bytes"], name + assert sha256(path) == metadata["sha256"], name + subprocess.run( + ["ffmpeg", "-v", "error", "-i", str(path), "-f", "null", "-"], + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + ) + print("seed integrity demo: verified") + + +if __name__ == "__main__": + main() diff --git a/src/creditlab/analysis/sweep.py b/src/creditlab/analysis/sweep.py index 7cd0109..1bf3460 100644 --- a/src/creditlab/analysis/sweep.py +++ b/src/creditlab/analysis/sweep.py @@ -100,6 +100,7 @@ def _collect_base_prompted_run(self, run_group_id: str, seed: int) -> str: if collect_config.policy.name not in ("vllm_server",): collect_config.policy.name = "prompted" collect_config.policy.config["seed"] = seed + collect_config.env.config["seed"] = seed env = create_env(collect_config) policy = create_policy(collect_config.policy.name, collect_config.policy.config, collect_config.store.artifact_root) context = RunContext( @@ -127,6 +128,7 @@ def _evaluate_prompted(self, run_group_id: str, seed: int) -> str: eval_config = self.config.model_copy(deep=True) eval_config.policy.name = "prompted" eval_config.policy.config["seed"] = seed + eval_config.env.config["seed"] = seed env = create_env(eval_config) policy = create_policy(eval_config.policy.name, eval_config.policy.config, eval_config.store.artifact_root) context = RunContext( @@ -164,7 +166,8 @@ def _train_and_evaluate( train_result = trainer.fit([source_run], scorer_name=scorer_name, budget=train_config.experiment.budgets) eval_config = self.config.model_copy(deep=True) eval_config.policy.name = "table" - eval_config.policy.config = {"model_path": train_result["model_path"]} + eval_config.policy.config = {"model_path": train_result["model_path"], "seed": seed} + eval_config.env.config["seed"] = seed env = create_env(eval_config) policy = create_policy(eval_config.policy.name, eval_config.policy.config, eval_config.store.artifact_root) context = RunContext( diff --git a/src/creditlab/core/registry.py b/src/creditlab/core/registry.py index af871c4..0bd254d 100644 --- a/src/creditlab/core/registry.py +++ b/src/creditlab/core/registry.py @@ -23,14 +23,15 @@ def create_env(config: ExperimentConfig): manifest = config.benchmark.manifest_path task_limit = config.benchmark.task_limit + seed = int(config.env.config.get("seed", 0)) if config.env.name == "diagnostic_branch_lab": - return DiagnosticBranchLabEnv(manifest_path=manifest, task_limit=task_limit) + return DiagnosticBranchLabEnv(manifest_path=manifest, task_limit=task_limit, seed=seed) if config.env.name == "webshop": - return WebShopEnv(manifest_path=manifest, task_limit=task_limit) + return WebShopEnv(manifest_path=manifest, task_limit=task_limit, seed=seed) if config.env.name == "alfworld": - return ALFWorldEnv(manifest_path=manifest, task_limit=task_limit) + return ALFWorldEnv(manifest_path=manifest, task_limit=task_limit, seed=seed) if config.env.name == "stochastic": - return StochasticEnv(manifest_path=manifest, task_limit=task_limit) + return StochasticEnv(manifest_path=manifest, task_limit=task_limit, seed=seed) if config.env.name == "alfworld_real": from creditlab.envs.alfworld_real import RealALFWorldEnv return RealALFWorldEnv( diff --git a/tests/test_sweep_and_export.py b/tests/test_sweep_and_export.py index 6efb569..9f6d941 100644 --- a/tests/test_sweep_and_export.py +++ b/tests/test_sweep_and_export.py @@ -1,9 +1,12 @@ from pathlib import Path +import yaml + from creditlab.analysis.exporters import export_results_bundle, render_paper_assets from creditlab.analysis.sweep import SweepRunner from creditlab.core.config import ExperimentConfig -from creditlab.core.registry import create_store +from creditlab.core.models import ActionEvent +from creditlab.core.registry import create_env, create_store def make_config(tmp_path: Path) -> ExperimentConfig: @@ -58,3 +61,113 @@ def test_sweep_aggregate_and_export(tmp_path: Path): assert sweep["aggregate"]["condition_summaries"] assert Path(bundle["files"][0]).exists() assert Path(rendered["generated_results"]).exists() + + +def test_manifest_environment_seed_controls_stochastic_transitions(tmp_path: Path): + manifest_path = tmp_path / "stochastic.yaml" + manifest_path.write_text( + yaml.safe_dump( + { + "benchmark_name": "seed_probe", + "revision": "v1", + "state_abstraction_version": "v1", + "verifier_version": "v1", + "tasks": [ + { + "task_id": "seed_probe", + "split": "smoke", + "instruction": "Take the stochastic edge.", + "initial_state": "start", + "states": { + "start": { + "phase": "start", + "mode": "ready", + "progress": 0, + "oracle_action": "advance", + "edges": { + "advance": { + "next_node": "success", + "failure_node": "failure", + "success_probability": 0.5, + "observation": "advanced", + "failure_observation": "blocked", + "terminal": True, + "success": True, + "progress_delta": 1, + "state_delta": "improved", + } + }, + }, + "success": { + "phase": "done", + "mode": "success", + "progress": 1, + "edges": {}, + }, + "failure": { + "phase": "done", + "mode": "failure", + "progress": 0, + "edges": {}, + }, + }, + } + ], + } + ), + encoding="utf-8", + ) + config = make_config(tmp_path) + config.env.name = "stochastic" + config.env.split = "smoke" + config.benchmark.manifest_path = manifest_path + + def destination(seed: int) -> str: + seeded = config.model_copy(deep=True) + seeded.env.config["seed"] = seed + env = create_env(seeded) + task = env.list_tasks("smoke")[0] + state = env.reset(task) + action = ActionEvent(action_id="advance", action_type="tool", content="advance") + next_state, _, _ = env.step(task, state, action) + return next_state.payload["raw_node"] + + assert destination(0) == destination(0) == "failure" + assert destination(1) == destination(1) == "success" + + +def test_sweep_propagates_seed_to_policy_and_environment(monkeypatch, tmp_path: Path): + config = make_config(tmp_path) + runner = SweepRunner(config) + captured: dict[str, int] = {} + + class FakeTrainer: + def fit(self, run_ids, scorer_name, budget): + return {"model_path": str(tmp_path / "model.json")} + + class FakeEngine: + def __init__(self, env, policy, store, tracker): + captured["environment"] = env + captured["policy"] = policy + + def collect(self, config, run_context, split_override): + captured["env_seed"] = config.env.config["seed"] + captured["policy_seed"] = config.policy.config["seed"] + return "run_seeded" + + monkeypatch.setattr("creditlab.analysis.sweep.create_trainer", lambda *args, **kwargs: FakeTrainer()) + monkeypatch.setattr("creditlab.analysis.sweep.create_env", lambda config: config.env.config["seed"]) + monkeypatch.setattr("creditlab.analysis.sweep.create_policy", lambda name, config, root: config["seed"]) + monkeypatch.setattr("creditlab.analysis.sweep.RolloutEngine", FakeEngine) + + run_id = runner._train_and_evaluate( + run_group_id="group_seeded", + seed=29, + source_run="run_source", + condition_name="branch_aware", + scorer_name="branch_aware", + trainer_name="branch_aware", + ) + + assert run_id == "run_seeded" + assert captured == {"environment": 29, "policy": 29, "env_seed": 29, "policy_seed": 29}