From eaa6d121411a10f4591eae58ed1e10164ae1c43a Mon Sep 17 00:00:00 2001 From: David Spencer <1526975+DecisionNerd@users.noreply.github.com> Date: Sat, 29 Aug 2026 03:35:09 -0600 Subject: [PATCH 1/2] test(bench): add authoritative rung controller --- benchmarks/Makefile | 28 +- benchmarks/README.md | 38 ++ .../graphforge_bench/progressive_run.py | 388 ++++++++++++++++++ .../schemas/ordinary-ingest-capability.json | 15 + benchmarks/schemas/progressive-run-plan.json | 39 ++ .../schemas/progressive-run-result.json | 15 + benchmarks/tests/test_progressive_run.py | 241 +++++++++++ 7 files changed, 763 insertions(+), 1 deletion(-) create mode 100644 benchmarks/harness/graphforge_bench/progressive_run.py create mode 100644 benchmarks/schemas/ordinary-ingest-capability.json create mode 100644 benchmarks/schemas/progressive-run-plan.json create mode 100644 benchmarks/schemas/progressive-run-result.json create mode 100644 benchmarks/tests/test_progressive_run.py diff --git a/benchmarks/Makefile b/benchmarks/Makefile index a421865c0..a20198ebe 100644 --- a/benchmarks/Makefile +++ b/benchmarks/Makefile @@ -1,4 +1,4 @@ -.PHONY: install smoke smoke-python smoke-rust local-admission progressive-qualification-list +.PHONY: install smoke smoke-python smoke-rust local-admission progressive-qualification-list progressive-qualification-plan progressive-qualification-run progressive-qualification-project-s20 progressive-qualification-binaries install: uv sync --locked @@ -21,3 +21,29 @@ local-admission: install progressive-qualification-list: install PYTHONPATH=$(CURDIR)/harness uv run --locked reframe -C reframe/settings.py -c reframe/checks -n '^Graph500ProgressiveQualificationProfile' -l + +progressive-qualification-binaries: + cargo build --locked --release --manifest-path ../Cargo.toml -p graphforge-cli + cargo build --locked --release --manifest-path Cargo.toml --bin graphforge-benchmark-certify --bin graphforge-benchmark-graph500-generator + +progressive-qualification-plan: install progressive-qualification-binaries + test -n "$(RUNG)" && test -n "$(OUTPUT_DIR)" + PYTHONPATH=$(CURDIR)/harness uv run --locked python -m graphforge_bench.progressive_run \ + --rung "$(RUNG)" --output-dir "$(OUTPUT_DIR)" --dry-run \ + --gf "$(CURDIR)/../target/release/gf" \ + --certify "$(CURDIR)/target/release/graphforge-benchmark-certify" \ + --generator "$(CURDIR)/target/release/graphforge-benchmark-graph500-generator" + +progressive-qualification-run: install progressive-qualification-binaries + test -n "$(RUNG)" && test -n "$(OUTPUT_DIR)" + PYTHONPATH=$(CURDIR)/harness uv run --locked python -m graphforge_bench.progressive_run \ + --rung "$(RUNG)" --output-dir "$(OUTPUT_DIR)" \ + --gf "$(CURDIR)/../target/release/gf" \ + --certify "$(CURDIR)/target/release/graphforge-benchmark-certify" \ + --generator "$(CURDIR)/target/release/graphforge-benchmark-graph500-generator" + +progressive-qualification-project-s20: install + test -n "$(OUTPUT_DIR)" && test -n "$(PROVIDER_CAPACITY)" + PYTHONPATH=$(CURDIR)/harness uv run --locked python -m graphforge_bench.progressive_run \ + --project-s20 --output-dir "$(OUTPUT_DIR)" \ + --provider-capacity "$(PROVIDER_CAPACITY)" diff --git a/benchmarks/README.md b/benchmarks/README.md index ad1d5ff70..88bcc30f4 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -160,6 +160,44 @@ BenchExec definition and public certification runner. Provider cases are not valid on the local ReFrame system. Provider provisioning is a later, separate operation; listing these profiles launches nothing. +The reproducible controller builds and hashes the three exact executables, +binds the checked-out commit and profile identity, stages the BenchExec XML in +a private directory, and writes only closed sanitized documents to the explicit +evidence directory. Planning is safe on unsupported hosts and executes no rung: + +```bash +make -C benchmarks progressive-qualification-plan \ + RUNG=S18 OUTPUT_DIR=/admitted-volume/graphforge-evidence +``` + +The real one-command entry point is intentionally manual and outside normal CI: + +```bash +make -C benchmarks progressive-qualification-run \ + RUNG=S18 OUTPUT_DIR=/admitted-volume/graphforge-evidence +``` + +It accepts only S18 followed by S19, refuses duplicate or out-of-order evidence, +requires native Linux cgroups-v2 BenchExec admission, and never provisions a +provider. S19 consumes the schema-valid passed `s18-rung.json` from the same +directory. After both local rungs pass, the existing progressive projection +policy consumes them as the adjacent S20 sources with one sanitized command: + +```bash +make -C benchmarks progressive-qualification-project-s20 \ + OUTPUT_DIR=/admitted-volume/graphforge-evidence \ + PROVIDER_CAPACITY=/sanitized/provider-capacity.json +``` + +The controller also requires `ordinary-ingest-capability.json` to prove that +the ordinary `gf import-session` path uses bulk construction with at least +65,536 rows per batch and no scalar durable write loop. The current 8,192-row +decode plus per-row durable path does not satisfy that contract, so real rungs +remain refused until the ordinary product path publishes the closed capability +evidence. Missing retained/transient storage, logical I/O, reader-call, or +publication-work evidence likewise causes a typed failure; the controller never +manufactures those values from command counts or recursive file scans. + ## Native local admission deployment The command is fail-closed: only `passed` exits successfully. A typed diff --git a/benchmarks/harness/graphforge_bench/progressive_run.py b/benchmarks/harness/graphforge_bench/progressive_run.py new file mode 100644 index 000000000..2ad638f44 --- /dev/null +++ b/benchmarks/harness/graphforge_bench/progressive_run.py @@ -0,0 +1,388 @@ +"""Fail-closed controller for native-Linux progressive qualification runs. + +The controller owns ordering, immutable executable identity, safe BenchExec +staging, and evidence validation. It deliberately does not provision hosts or +invent metrics that the ordinary GraphForge lifecycle did not emit. +""" + +from __future__ import annotations + +import argparse +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +import hashlib +from importlib.metadata import PackageNotFoundError, version +import json +import os +from pathlib import Path +import platform +import re +import shutil +import subprocess +import sys +import tempfile +from typing import Any + +from jsonschema import Draft202012Validator + +from graphforge_bench.local_admission import qualify_local_host +from graphforge_bench.progressive_qualification import QualificationError, load_profiles, project + +PLAN_SCHEMA = "graphforge-progressive-run-plan/1" +RESULT_SCHEMA = "graphforge-progressive-run-result/1" +GIT_COMMIT = re.compile(r"^[0-9a-f]{40}$") +LOCAL_RUNGS = (18, 19) + + +class ControllerError(ValueError): + """The requested run is unsafe, out of order, or lacks valid evidence.""" + + +@dataclass(frozen=True) +class Executables: + gf: Path + certify: Path + generator: Path + benchexec_python: Path + + +def _json(path: Path) -> Any: + try: + return json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise ControllerError(f"invalid evidence document: {path.name}") from error + + +def _validate(root: Path, schema_name: str, document: Any) -> None: + schema = _json(root / "schemas" / schema_name) + error = next(Draft202012Validator(schema).iter_errors(document), None) + if error is not None: + raise ControllerError(f"{schema_name} validation failed: {error.message}") + + +def _digest(path: Path) -> str: + value = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + value.update(chunk) + return value.hexdigest() + + +def _resolve_executable(value: str, expected_name: str) -> Path: + candidate = Path(value) + located = str(candidate) if candidate.is_absolute() else shutil.which(value) + if located is None: + raise ControllerError(f"required executable unavailable: {expected_name}") + resolved = Path(located).resolve(strict=True) + if not resolved.is_file() or not os.access(resolved, os.X_OK): + raise ControllerError(f"required executable is not executable: {expected_name}") + return resolved + + +def resolve_executables( + *, gf: str, certify: str, generator: str, benchexec_python: str +) -> Executables: + return Executables( + gf=_resolve_executable(gf, "gf"), + certify=_resolve_executable(certify, "graphforge-benchmark-certify"), + generator=_resolve_executable(generator, "graphforge-benchmark-graph500-generator"), + benchexec_python=_resolve_executable(benchexec_python, "python"), + ) + + +def _commit(value: str) -> str: + if not GIT_COMMIT.fullmatch(value): + raise ControllerError("commit must be a lowercase full Git object ID") + return value + + +def repository_commit(root: Path) -> str: + completed = subprocess.run( + ["git", "-C", str(root.parent), "rev-parse", "HEAD"], + text=True, + capture_output=True, + check=False, + ) + if completed.returncode != 0: + raise ControllerError("repository commit unavailable") + return _commit(completed.stdout.strip()) + + +def _profile(root: Path, scale: int) -> tuple[Path, Mapping[str, Any]]: + if scale not in LOCAL_RUNGS: + raise ControllerError("authoritative local controller accepts only S18 or S19") + path = root / "profiles" / "graph500" / f"s{scale}-local.json" + document = _json(path) + _validate(root, "progressive-qualification-profile.json", document) + return path, document + + +def _passed_rung(root: Path, output_dir: Path, scale: int) -> Mapping[str, Any] | None: + path = output_dir / f"s{scale}-rung.json" + if not path.exists(): + return None + document = _json(path) + _validate(root, "progressive-qualification-rung-evidence.json", document) + if ( + document.get("scale") != scale + or document.get("status") != "passed" + or document.get("profile_id") != f"graph500-s{scale}-local" + or document.get("source") != "progressive_profile" + or document.get("live_edges") != 16 * (1 << scale) + ): + raise ControllerError(f"S{scale} evidence is not a passed matching rung") + return document + + +def require_order(root: Path, output_dir: Path, scale: int) -> None: + s18 = _passed_rung(root, output_dir, 18) + s19 = _passed_rung(root, output_dir, 19) + if scale == 18 and (s18 is not None or s19 is not None): + raise ControllerError("S18 may run only as the first incomplete rung") + if scale == 19 and (s18 is None or s19 is not None): + raise ControllerError("S19 requires exactly one passed S18 rung") + + +def build_plan( + *, + root: Path, + output_dir: Path, + scale: int, + commit: str, + executables: Executables, +) -> dict[str, Any]: + require_order(root, output_dir, scale) + _, profile = _profile(root, scale) + generator_digest = "sha256:" + _digest(root / "runners/graph500-generator/src/main.rs") + if generator_digest != profile["generator"]["identity"]: + raise ControllerError("generator source identity contradicts the checked-in profile") + try: + benchexec_version = version("BenchExec") + except PackageNotFoundError as error: + raise ControllerError("BenchExec package identity unavailable") from error + identities = { + "commit": _commit(commit), + "profile_id": profile["id"], + "profile_sha256": _digest(root / "profiles/graph500" / f"s{scale}-local.json"), + "generator": generator_digest, + "generator_executable_sha256": _digest(executables.generator), + "gf_sha256": _digest(executables.gf), + "certify_sha256": _digest(executables.certify), + "benchexec_python_sha256": _digest(executables.benchexec_python), + "benchexec_version": benchexec_version, + } + plan = { + "schema": PLAN_SCHEMA, + "rung": f"S{scale}", + "execution": "native_linux_benchexec", + "identities": identities, + "limits": {"wall_seconds": 14_400, "memory_bytes": 4_294_967_296, "cores": 16}, + "outputs": [ + f"s{scale}-benchexec.json", + f"s{scale}-graphforge.json", + f"s{scale}-rung.json", + ], + "claim": "engineering_evidence_only", + } + _validate(root, "progressive-run-plan.json", plan) + return plan + + +def _write_json(path: Path, value: Mapping[str, Any]) -> None: + path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def write_plan(output_dir: Path, plan: Mapping[str, Any]) -> Path: + output_dir.mkdir(parents=True, exist_ok=True) + path = output_dir / f"{str(plan['rung']).lower()}-plan.json" + _write_json(path, plan) + return path + + +def _safe_stage(root: Path, profile_path: Path, executables: Executables, parent: Path) -> Path: + stage = Path(tempfile.mkdtemp(prefix="gf-progressive-", dir=parent)) + shutil.copyfile(profile_path, stage / "profile.json") + shutil.copyfile( + root / "definitions/graphforge-progressive-qualification-v1.xml", stage / "benchmark.xml" + ) + bin_dir = stage / "bin" + bin_dir.mkdir() + for name, source in ( + ("gf", executables.gf), + ("graphforge-benchmark-certify", executables.certify), + ("graphforge-benchmark-graph500-generator", executables.generator), + ): + (bin_dir / name).symlink_to(source) + return stage + + +def _native_authority() -> Mapping[str, Any]: + if platform.system() != "Linux": + raise ControllerError("native Linux BenchExec authority is required") + evidence = qualify_local_host() + if evidence.get("result") != "passed": + cause = evidence.get("cause") + raise ControllerError(f"native BenchExec admission refused: {cause}") + return evidence + + +def require_bulk_ingest_capability( + root: Path, output_dir: Path, commit: str | None = None +) -> Mapping[str, Any]: + """Require ordinary import-session proof before spending a rung. + + The current scalar durable path is not silently treated as a valid scale + implementation. A later ordinary-path repair must publish this closed, + commit-bound capability document before the controller can execute. + """ + path = output_dir / "ordinary-ingest-capability.json" + if not path.is_file(): + raise ControllerError("bulk_ingest_capability_unproven") + evidence = _json(path) + _validate(root, "ordinary-ingest-capability.json", evidence) + if commit is not None and evidence.get("commit") != commit: + raise ControllerError("bulk_ingest_capability_commit_mismatch") + return evidence + + +def _run_benchexec(stage: Path, executables: Executables) -> int: + raw_output = stage / "raw" + raw_output.mkdir() + home = stage / "home" + home.mkdir() + environment = { + "HOME": str(home), + "LANG": "C.UTF-8", + "LC_ALL": "C.UTF-8", + "PATH": f"{stage / 'bin'}:{Path(sys.executable).parent}:/usr/bin:/bin", + } + command = [ + str(executables.benchexec_python), + "-m", + "benchexec", + "--no-compress-results", + "--outputpath", + str(raw_output), + "--rundefinition", + "graphforge-progressive-qualification-v1", + str(stage / "benchmark.xml"), + ] + return subprocess.run(command, env=environment, check=False).returncode + + +def validate_fixture_bundle(root: Path, bundle: Path, scale: int) -> None: + """Validate the three closed documents a real run must ultimately produce.""" + benchexec = _json(bundle / "benchexec.json") + graphforge = _json(bundle / "graphforge.json") + rung = _json(bundle / "rung.json") + _validate(root, "benchexec-run-evidence.json", benchexec) + _validate(root, "certification-evidence.json", graphforge) + _validate(root, "progressive-qualification-rung-evidence.json", rung) + if rung.get("scale") != scale or graphforge.get("profile_id") != f"graph500-s{scale}-local": + raise ControllerError("fixture evidence contradicts the selected rung") + if benchexec.get("graphforge") != graphforge: + raise ControllerError("BenchExec and GraphForge evidence disagree") + + +def run( + *, root: Path, output_dir: Path, scale: int, plan: Mapping[str, Any], executables: Executables +) -> None: + require_bulk_ingest_capability(root, output_dir, str(plan["identities"]["commit"])) + _native_authority() + profile_path, _ = _profile(root, scale) + output_dir.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix="gf-progressive-authority-") as temporary: + stage = _safe_stage(root, profile_path, executables, Path(temporary)) + status = _run_benchexec(stage, executables) + # Raw logs remain in the private temporary directory. Until the + # ordinary lifecycle emits every progressive metric, accepting a run + # would fabricate schema-valid evidence. Fail closed instead. + result = { + "schema": RESULT_SCHEMA, + "rung": f"S{scale}", + "status": "failed", + "failure": "metrics_evidence_missing" if status == 0 else "benchexec_failed", + "identities": plan["identities"], + "claim": "engineering_evidence_only", + } + _validate(root, "progressive-run-result.json", result) + _write_json(output_dir / f"s{scale}-result.json", result) + raise ControllerError(str(result["failure"])) + + +def write_s20_projection(root: Path, output_dir: Path, capacity_path: Path) -> Path: + s18 = _passed_rung(root, output_dir, 18) + s19 = _passed_rung(root, output_dir, 19) + if s18 is None or s19 is None: + raise ControllerError("S20 projection requires passed adjacent S18 and S19 rungs") + capacity = _json(capacity_path) + if not isinstance(capacity, Mapping): + raise ControllerError("provider capacity evidence must be an object") + s20 = next( + profile for profile in load_profiles(root / "profiles" / "graph500") if profile.scale == 20 + ) + evidence = project(s20, [s18, s19], capacity) + _validate(root, "progressive-qualification-evidence.json", evidence) + path = output_dir / "s20-projection.json" + _write_json(path, evidence) + return path + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + action = parser.add_mutually_exclusive_group(required=True) + action.add_argument("--rung", choices=("S18", "S19")) + action.add_argument("--project-s20", action="store_true") + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--gf") + parser.add_argument("--certify") + parser.add_argument("--generator") + parser.add_argument("--benchexec-python", default=sys.executable) + parser.add_argument("--dry-run", action="store_true") + parser.add_argument("--fixture-bundle", type=Path) + parser.add_argument("--provider-capacity", type=Path) + args = parser.parse_args(argv) + root = Path(__file__).resolve().parents[2] + try: + if args.project_s20: + if args.provider_capacity is None: + raise ControllerError("--project-s20 requires --provider-capacity") + write_s20_projection(root, args.output_dir, args.provider_capacity) + return 0 + if not all((args.gf, args.certify, args.generator)): + raise ControllerError("rung execution requires gf, certify, and generator") + if args.fixture_bundle is not None and not args.dry_run: + raise ControllerError("fixture bundles are accepted only with --dry-run") + scale = int(args.rung[1:]) + executables = resolve_executables( + gf=args.gf, + certify=args.certify, + generator=args.generator, + benchexec_python=args.benchexec_python, + ) + plan = build_plan( + root=root, + output_dir=args.output_dir, + scale=scale, + commit=repository_commit(root), + executables=executables, + ) + write_plan(args.output_dir, plan) + if args.fixture_bundle is not None: + validate_fixture_bundle(root, args.fixture_bundle, scale) + if not args.dry_run: + run( + root=root, + output_dir=args.output_dir, + scale=scale, + plan=plan, + executables=executables, + ) + return 0 + except (ControllerError, QualificationError) as error: + print(json.dumps({"schema": RESULT_SCHEMA, "status": "failed", "failure": str(error)})) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/schemas/ordinary-ingest-capability.json b/benchmarks/schemas/ordinary-ingest-capability.json new file mode 100644 index 000000000..2a6ba7d0c --- /dev/null +++ b/benchmarks/schemas/ordinary-ingest-capability.json @@ -0,0 +1,15 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "schema": "graphforge-ordinary-ingest-capability-schema/1", + "type": "object", + "additionalProperties": false, + "required": ["schema", "commit", "interface", "bulk_construction", "minimum_batch_rows", "scalar_durable_loop_absent"], + "properties": { + "schema": {"const": "graphforge-ordinary-ingest-capability/1"}, + "commit": {"type": "string", "pattern": "^[0-9a-f]{40}$"}, + "interface": {"const": "gf_import_session"}, + "bulk_construction": {"const": true}, + "minimum_batch_rows": {"type": "integer", "minimum": 65536}, + "scalar_durable_loop_absent": {"const": true} + } +} diff --git a/benchmarks/schemas/progressive-run-plan.json b/benchmarks/schemas/progressive-run-plan.json new file mode 100644 index 000000000..c5b5c01a8 --- /dev/null +++ b/benchmarks/schemas/progressive-run-plan.json @@ -0,0 +1,39 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "schema": "graphforge-progressive-run-plan-schema/1", + "type": "object", + "additionalProperties": false, + "required": ["schema", "rung", "execution", "identities", "limits", "outputs", "claim"], + "properties": { + "schema": {"const": "graphforge-progressive-run-plan/1"}, + "rung": {"enum": ["S18", "S19"]}, + "execution": {"const": "native_linux_benchexec"}, + "identities": { + "type": "object", "additionalProperties": false, + "required": ["commit", "profile_id", "profile_sha256", "generator", "generator_executable_sha256", "gf_sha256", "certify_sha256", "benchexec_python_sha256", "benchexec_version"], + "properties": { + "commit": {"type": "string", "pattern": "^[0-9a-f]{40}$"}, + "profile_id": {"enum": ["graph500-s18-local", "graph500-s19-local"]}, + "profile_sha256": {"$ref": "#/$defs/digest"}, + "generator": {"type": "string", "pattern": "^sha256:[0-9a-f]{64}$"}, + "generator_executable_sha256": {"$ref": "#/$defs/digest"}, + "gf_sha256": {"$ref": "#/$defs/digest"}, + "certify_sha256": {"$ref": "#/$defs/digest"}, + "benchexec_python_sha256": {"$ref": "#/$defs/digest"}, + "benchexec_version": {"type": "string", "pattern": "^[0-9]+(\\.[0-9]+)+$"} + } + }, + "limits": { + "type": "object", "additionalProperties": false, + "required": ["wall_seconds", "memory_bytes", "cores"], + "properties": { + "wall_seconds": {"const": 14400}, + "memory_bytes": {"const": 4294967296}, + "cores": {"const": 16} + } + }, + "outputs": {"type": "array", "minItems": 3, "maxItems": 3, "uniqueItems": true, "items": {"type": "string", "pattern": "^s(18|19)-[a-z-]+\\.json$"}}, + "claim": {"const": "engineering_evidence_only"} + }, + "$defs": {"digest": {"type": "string", "pattern": "^[0-9a-f]{64}$"}} +} diff --git a/benchmarks/schemas/progressive-run-result.json b/benchmarks/schemas/progressive-run-result.json new file mode 100644 index 000000000..c0e9171de --- /dev/null +++ b/benchmarks/schemas/progressive-run-result.json @@ -0,0 +1,15 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "schema": "graphforge-progressive-run-result-schema/1", + "type": "object", + "additionalProperties": false, + "required": ["schema", "rung", "status", "failure", "identities", "claim"], + "properties": { + "schema": {"const": "graphforge-progressive-run-result/1"}, + "rung": {"enum": ["S18", "S19"]}, + "status": {"const": "failed"}, + "failure": {"enum": ["metrics_evidence_missing", "benchexec_failed"]}, + "identities": {"type": "object"}, + "claim": {"const": "engineering_evidence_only"} + } +} diff --git a/benchmarks/tests/test_progressive_run.py b/benchmarks/tests/test_progressive_run.py new file mode 100644 index 000000000..3ba09248e --- /dev/null +++ b/benchmarks/tests/test_progressive_run.py @@ -0,0 +1,241 @@ +from __future__ import annotations + +import json +from pathlib import Path +import tempfile +import unittest + +from graphforge_bench.progressive_run import ( + ControllerError, + Executables, + build_plan, + require_bulk_ingest_capability, + require_order, + validate_fixture_bundle, + write_plan, + write_s20_projection, +) + +ROOT = Path(__file__).resolve().parents[1] +COMMIT = "78b75aed8fef71cfa3e4700b80a05d6b71e64f22" +PHASES = [ + "admission", + "generate", + "ingest", + "reopen", + "recount", + "query", + "export", + "verify", + "clean_import", + "reopen_proof", +] + + +def passed_rung(scale: int) -> dict: + return { + "profile_id": f"graph500-s{scale}-local", + "source": "progressive_profile", + "scale": scale, + "live_edges": (1 << scale) * 16, + "status": "passed", + "correctness": True, + "phases": PHASES, + "metrics": { + "wall_seconds": 10, + "peak_rss_bytes": 100, + "retained_storage_bytes": 200, + "transient_peak_storage_bytes": 300, + "logical_read_bytes": 400, + "logical_write_bytes": 500, + "physical_read_bytes": 600, + "physical_write_bytes": 700, + "reader_calls": 8, + "publication_work_units": 9, + }, + "failure": None, + } + + +def graphforge(scale: int) -> dict: + phases = [ + { + "phase": phase, + "status": "passed", + "duration_ms": 1, + "peak_rss_bytes": 100, + "exit_code": 0, + } + for phase in PHASES + ] + return { + "schema": "graphforge-public-certification/1", + "profile_id": f"graph500-s{scale}-local", + "status": "passed", + "phases": phases, + "failed_phase": None, + } + + +def benchexec(gf: dict) -> dict: + return { + "schema": "graphforge-benchexec-run/1", + "outcome": "passed", + "exit_code": 0, + "signal": None, + "authority": { + "wall_seconds": 0.01, + "cpu_seconds": 0.01, + "peak_rss_bytes": 100, + "read_bytes": 0, + "write_bytes": 0, + "pressure_cpu_seconds": 0.0, + "pressure_io_seconds": 0.0, + "pressure_memory_seconds": 0.0, + }, + "limits": { + "wall_seconds": 14400.0, + "cpu_seconds": 14400.0, + "memory_bytes": 4294967296, + "cores": list(range(16)), + }, + "graphforge": gf, + "disagreements": [], + } + + +class ProgressiveRunControllerTests(unittest.TestCase): + def setUp(self) -> None: + self.temporary = tempfile.TemporaryDirectory() + self.base = Path(self.temporary.name) + self.output = self.base / "evidence" + self.output.mkdir() + generator = self.base / "graphforge-benchmark-graph500-generator" + generator.write_bytes((ROOT / "runners/graph500-generator/src/main.rs").read_bytes()) + gf = self.base / "gf" + certify = self.base / "graphforge-benchmark-certify" + python = self.base / "python" + for path in (gf, certify, python): + path.write_bytes(b"fixture") + for path in (generator, gf, certify, python): + path.chmod(path.stat().st_mode | 0o111) + self.executables = Executables(gf, certify, generator, python) + + def tearDown(self) -> None: + self.temporary.cleanup() + + def test_dry_plan_binds_exact_immutable_identities_without_paths(self) -> None: + plan = build_plan( + root=ROOT, + output_dir=self.output, + scale=18, + commit=COMMIT, + executables=self.executables, + ) + path = write_plan(self.output, plan) + profile = json.loads((ROOT / "profiles/graph500/s18-local.json").read_text()) + self.assertEqual(plan["rung"], "S18") + self.assertEqual(plan["identities"]["commit"], COMMIT) + self.assertEqual(plan["identities"]["generator"], profile["generator"]["identity"]) + encoded = path.read_text() + self.assertNotIn(str(self.base), encoded) + self.assertNotIn("token", encoded.lower()) + + def test_s19_requires_passed_s18_and_s18_cannot_repeat(self) -> None: + with self.assertRaisesRegex(ControllerError, "requires exactly one"): + require_order(ROOT, self.output, 19) + (self.output / "s18-rung.json").write_text(json.dumps(passed_rung(18))) + require_order(ROOT, self.output, 19) + with self.assertRaisesRegex(ControllerError, "first incomplete"): + require_order(ROOT, self.output, 18) + + def test_failed_prior_evidence_refuses_progression(self) -> None: + failed = passed_rung(18) + failed.update(status="failed", correctness=False, failure="ingest") + failed["phases"] = failed["phases"][:3] + failed["metrics"] = {"wall_seconds": 1, "peak_rss_bytes": 2} + (self.output / "s18-rung.json").write_text(json.dumps(failed)) + with self.assertRaisesRegex(ControllerError, "not a passed"): + require_order(ROOT, self.output, 19) + + def test_fixture_bundle_validates_all_closed_evidence_contracts(self) -> None: + bundle = self.base / "bundle" + bundle.mkdir() + gf = graphforge(18) + (bundle / "graphforge.json").write_text(json.dumps(gf)) + (bundle / "benchexec.json").write_text(json.dumps(benchexec(gf))) + (bundle / "rung.json").write_text(json.dumps(passed_rung(18))) + validate_fixture_bundle(ROOT, bundle, 18) + changed = benchexec(gf) + changed["graphforge"]["profile_id"] = "graph500-s19-local" + (bundle / "benchexec.json").write_text(json.dumps(changed)) + with self.assertRaisesRegex(ControllerError, "disagree"): + validate_fixture_bundle(ROOT, bundle, 18) + + def test_generator_executable_digest_and_commit_are_fail_closed(self) -> None: + original = build_plan( + root=ROOT, + output_dir=self.output, + scale=18, + commit=COMMIT, + executables=self.executables, + )["identities"]["generator_executable_sha256"] + self.executables.generator.write_bytes(b"wrong") + changed = build_plan( + root=ROOT, + output_dir=self.output, + scale=18, + commit=COMMIT, + executables=self.executables, + )["identities"]["generator_executable_sha256"] + self.assertNotEqual(original, changed) + self.executables.generator.write_bytes( + (ROOT / "runners/graph500-generator/src/main.rs").read_bytes() + ) + with self.assertRaisesRegex(ControllerError, "full Git object ID"): + build_plan( + root=ROOT, + output_dir=self.output, + scale=18, + commit="HEAD", + executables=self.executables, + ) + + def test_real_run_requires_closed_bulk_ingest_capability(self) -> None: + with self.assertRaisesRegex(ControllerError, "bulk_ingest_capability_unproven"): + require_bulk_ingest_capability(ROOT, self.output) + invalid = { + "schema": "graphforge-ordinary-ingest-capability/1", + "commit": COMMIT, + "interface": "gf_import_session", + "bulk_construction": True, + "minimum_batch_rows": 8192, + "scalar_durable_loop_absent": False, + } + (self.output / "ordinary-ingest-capability.json").write_text(json.dumps(invalid)) + with self.assertRaisesRegex(ControllerError, "validation failed"): + require_bulk_ingest_capability(ROOT, self.output) + + def test_adjacent_passed_rungs_produce_schema_valid_s20_projection(self) -> None: + for scale in (18, 19): + (self.output / f"s{scale}-rung.json").write_text(json.dumps(passed_rung(scale))) + capacity = self.base / "capacity.json" + capacity.write_text( + json.dumps( + { + "physical_read_bytes_per_second": 1_000_000, + "physical_write_bytes_per_second": 1_000_000, + "reader_calls_per_second": 1_000_000, + "publication_work_per_second": 1_000_000, + "secret": "discarded", + } + ) + ) + path = write_s20_projection(ROOT, self.output, capacity) + evidence = json.loads(path.read_text()) + self.assertEqual(evidence["source_scales"], [18, 19]) + self.assertNotIn("secret", path.read_text()) + + +if __name__ == "__main__": + unittest.main() From d8a0ca28c47529bab2e9791b849bfe83eeb8cacc Mon Sep 17 00:00:00 2001 From: David Spencer <1526975+DecisionNerd@users.noreply.github.com> Date: Sat, 29 Aug 2026 09:04:58 -0600 Subject: [PATCH 2/2] fix(bench): bind staged run identities --- .../graphforge_bench/progressive_run.py | 36 +++++++++--- .../schemas/progressive-run-result.json | 20 ++++++- benchmarks/tests/test_progressive_run.py | 58 +++++++++++++++++++ 3 files changed, 103 insertions(+), 11 deletions(-) diff --git a/benchmarks/harness/graphforge_bench/progressive_run.py b/benchmarks/harness/graphforge_bench/progressive_run.py index 2ad638f44..1fd630acf 100644 --- a/benchmarks/harness/graphforge_bench/progressive_run.py +++ b/benchmarks/harness/graphforge_bench/progressive_run.py @@ -199,7 +199,13 @@ def write_plan(output_dir: Path, plan: Mapping[str, Any]) -> Path: return path -def _safe_stage(root: Path, profile_path: Path, executables: Executables, parent: Path) -> Path: +def _safe_stage( + root: Path, + profile_path: Path, + executables: Executables, + identities: Mapping[str, Any], + parent: Path, +) -> Path: stage = Path(tempfile.mkdtemp(prefix="gf-progressive-", dir=parent)) shutil.copyfile(profile_path, stage / "profile.json") shutil.copyfile( @@ -207,12 +213,19 @@ def _safe_stage(root: Path, profile_path: Path, executables: Executables, parent ) bin_dir = stage / "bin" bin_dir.mkdir() - for name, source in ( - ("gf", executables.gf), - ("graphforge-benchmark-certify", executables.certify), - ("graphforge-benchmark-graph500-generator", executables.generator), + for name, source, identity_key in ( + ("gf", executables.gf, "gf_sha256"), + ("graphforge-benchmark-certify", executables.certify, "certify_sha256"), + ( + "graphforge-benchmark-graph500-generator", + executables.generator, + "generator_executable_sha256", + ), ): - (bin_dir / name).symlink_to(source) + staged = bin_dir / name + shutil.copy2(source, staged) + if _digest(staged) != identities.get(identity_key): + raise ControllerError(f"staged executable identity mismatch: {name}") return stage @@ -245,7 +258,7 @@ def require_bulk_ingest_capability( return evidence -def _run_benchexec(stage: Path, executables: Executables) -> int: +def _run_benchexec(stage: Path, executables: Executables, identities: Mapping[str, Any]) -> int: raw_output = stage / "raw" raw_output.mkdir() home = stage / "home" @@ -267,6 +280,8 @@ def _run_benchexec(stage: Path, executables: Executables) -> int: "graphforge-progressive-qualification-v1", str(stage / "benchmark.xml"), ] + if _digest(executables.benchexec_python) != identities.get("benchexec_python_sha256"): + raise ControllerError("BenchExec Python identity changed after planning") return subprocess.run(command, env=environment, check=False).returncode @@ -292,8 +307,11 @@ def run( profile_path, _ = _profile(root, scale) output_dir.mkdir(parents=True, exist_ok=True) with tempfile.TemporaryDirectory(prefix="gf-progressive-authority-") as temporary: - stage = _safe_stage(root, profile_path, executables, Path(temporary)) - status = _run_benchexec(stage, executables) + identities = plan["identities"] + if not isinstance(identities, Mapping): + raise ControllerError("run plan identities are malformed") + stage = _safe_stage(root, profile_path, executables, identities, Path(temporary)) + status = _run_benchexec(stage, executables, identities) # Raw logs remain in the private temporary directory. Until the # ordinary lifecycle emits every progressive metric, accepting a run # would fabricate schema-valid evidence. Fail closed instead. diff --git a/benchmarks/schemas/progressive-run-result.json b/benchmarks/schemas/progressive-run-result.json index c0e9171de..33e24227d 100644 --- a/benchmarks/schemas/progressive-run-result.json +++ b/benchmarks/schemas/progressive-run-result.json @@ -9,7 +9,23 @@ "rung": {"enum": ["S18", "S19"]}, "status": {"const": "failed"}, "failure": {"enum": ["metrics_evidence_missing", "benchexec_failed"]}, - "identities": {"type": "object"}, + "identities": { + "type": "object", + "additionalProperties": false, + "required": ["commit", "profile_id", "profile_sha256", "generator", "generator_executable_sha256", "gf_sha256", "certify_sha256", "benchexec_python_sha256", "benchexec_version"], + "properties": { + "commit": {"type": "string", "pattern": "^[0-9a-f]{40}$"}, + "profile_id": {"enum": ["graph500-s18-local", "graph500-s19-local"]}, + "profile_sha256": {"$ref": "#/$defs/digest"}, + "generator": {"type": "string", "pattern": "^sha256:[0-9a-f]{64}$"}, + "generator_executable_sha256": {"$ref": "#/$defs/digest"}, + "gf_sha256": {"$ref": "#/$defs/digest"}, + "certify_sha256": {"$ref": "#/$defs/digest"}, + "benchexec_python_sha256": {"$ref": "#/$defs/digest"}, + "benchexec_version": {"type": "string", "pattern": "^[0-9]+(\\.[0-9]+)+$"} + } + }, "claim": {"const": "engineering_evidence_only"} - } + }, + "$defs": {"digest": {"type": "string", "pattern": "^[0-9a-f]{64}$"}} } diff --git a/benchmarks/tests/test_progressive_run.py b/benchmarks/tests/test_progressive_run.py index 3ba09248e..3f9d9733b 100644 --- a/benchmarks/tests/test_progressive_run.py +++ b/benchmarks/tests/test_progressive_run.py @@ -8,6 +8,9 @@ from graphforge_bench.progressive_run import ( ControllerError, Executables, + _run_benchexec, + _safe_stage, + _validate, build_plan, require_bulk_ingest_capability, require_order, @@ -236,6 +239,61 @@ def test_adjacent_passed_rungs_produce_schema_valid_s20_projection(self) -> None self.assertEqual(evidence["source_scales"], [18, 19]) self.assertNotIn("secret", path.read_text()) + def test_staged_executables_are_verified_private_copies(self) -> None: + plan = build_plan( + root=ROOT, + output_dir=self.output, + scale=18, + commit=COMMIT, + executables=self.executables, + ) + profile = ROOT / "profiles/graph500/s18-local.json" + stage = _safe_stage(ROOT, profile, self.executables, plan["identities"], self.base) + staged_gf = stage / "bin/gf" + self.assertFalse(staged_gf.is_symlink()) + original = staged_gf.read_bytes() + self.executables.gf.write_bytes(b"changed-after-planning") + self.assertEqual(staged_gf.read_bytes(), original) + with self.assertRaisesRegex(ControllerError, "staged executable identity mismatch"): + _safe_stage(ROOT, profile, self.executables, plan["identities"], self.base) + + def test_benchexec_python_is_rechecked_immediately_before_invocation(self) -> None: + plan = build_plan( + root=ROOT, + output_dir=self.output, + scale=18, + commit=COMMIT, + executables=self.executables, + ) + stage = self.base / "stage" + stage.mkdir() + (stage / "bin").mkdir() + (stage / "benchmark.xml").write_text("fixture") + self.executables.benchexec_python.write_bytes(b"changed-after-planning") + with self.assertRaisesRegex(ControllerError, "identity changed after planning"): + _run_benchexec(stage, self.executables, plan["identities"]) + + def test_failed_result_schema_requires_closed_exact_identities(self) -> None: + plan = build_plan( + root=ROOT, + output_dir=self.output, + scale=18, + commit=COMMIT, + executables=self.executables, + ) + result = { + "schema": "graphforge-progressive-run-result/1", + "rung": "S18", + "status": "failed", + "failure": "metrics_evidence_missing", + "identities": plan["identities"], + "claim": "engineering_evidence_only", + } + _validate(ROOT, "progressive-run-result.json", result) + result["identities"] = {} + with self.assertRaisesRegex(ControllerError, "validation failed"): + _validate(ROOT, "progressive-run-result.json", result) + if __name__ == "__main__": unittest.main()