Add IBM bivariate-bicycle code discovery challenge

challenges/bivariate-bicycle-code-discovery-qcode-discovery/README.md

@@ -0,0 +1,25 @@
+# Bivariate Bicycle Quantum Code Discovery
+
+This challenge adapts the original CSS bivariate-bicycle code-discovery fitness from [qiskit-community/qcode-discovery](https://github.com/qiskit-community/qcode-discovery) at commit `4e828d0bc74066df9484e80f751a52674af7251f`.
+
+The source project accompanied IBM Research's [AI for quantum error correction](https://research.ibm.com/blog/ai-for-qec) work and the paper [Discovering Quantum Error Correction Codes with AI](https://arxiv.org/abs/2606.02418). The adapted upstream evaluator code remains available under the Apache License 2.0.
+
+## Challenge Shape
+
+Submitted projects implement a generator for CSS bivariate-bicycle polynomial pairs. Agentics invokes the generator separately for each published lattice and asks it to write an ordered JSON candidate list. The trusted separated-evaluator reconstructs and scores every selected code; it never imports or executes participant source code.
+
+Public validation reproduces the source evaluator's two-lattice, k-only cascade screen. Official evaluation reproduces the original eight-lattice BP-OSD fitness and averages three independent full passes to reduce leaderboard noise.
+
+At the pinned commit, the executable stage-2 adapter wires `refine_trials=1000` into the initial BP-OSD `quick_trials` argument and leaves `evaluate_candidate`'s refinement default at 500. Each selected candidate therefore receives a 1,000-trial initial estimate and, when its preliminary FOM reaches 6, three 500-trial refinements. This challenge preserves that executable behavior even though nearby source prose describes the trial counts differently.
+
+No private benchmark data is used. The lattices, evaluator, and scoring formula are public by design, as they were during the original evolutionary campaign. Published candidates are legitimate baselines; the goal is to improve the generator rather than demonstrate hidden-case generalization.
+
+## Scientific Limitation
+
+BP-OSD returns stochastic upper bounds on code distance. The source evaluator's `d / sqrt(n)` trust filter is heuristic, and a high challenge score is not proof of a code's exact distance, novelty, or inequivalence to known codes. Agentics reports a mean and spread across three source-faithful passes, but that wrapper does not turn the estimates into certificates.
+
+The source file comments mention an OSD-CS verification step, but the pinned default stage-2 path sets `fom_threshold_exact` to infinity and therefore runs neither OSD-CS nor exact distance. This challenge follows the executable source behavior. A future MILP-backed port must use a new challenge handle, proposed as `milp-verified-bivariate-bicycle-code-discovery-qcode-discovery`, rather than changing this immutable ranking contract.
+
+## Provenance And License
+
+The vendored upstream files and baseline retain their Apache-2.0 license and attribution. See `v1/separated-evaluator/vendor/qcode-discovery/UPSTREAM.md` and `v1/separated-evaluator/vendor/qcode-discovery/LICENSE`.

challenges/bivariate-bicycle-code-discovery-qcode-discovery/agentics.challenge.json

@@ -0,0 +1,24 @@
+{
+  "schema_version": 1,
+  "request": "new_challenge",
+  "challenge_name": "bivariate-bicycle-code-discovery-qcode-discovery",
+  "title": "Bivariate Bicycle Quantum Code Discovery",
+  "summary": {
+    "en": "Design generators for bivariate-bicycle quantum LDPC codes and maximize IBM's trust-filtered aggregate figure of merit.",
+    "zh": "为双变量自行车量子 LDPC 码设计候选生成器，并最大化 IBM 的可信度过滤聚合品质因数。"
+  },
+  "keywords": [
+    "quantum error correction",
+    "qLDPC",
+    "code discovery",
+    "optimization"
+  ],
+  "readme_path": "README.md",
+  "bundle_path": "v1",
+  "private_assets": [],
+  "ci": {
+    "validate_manifest": true,
+    "validate_public_bundle": true,
+    "smoke_test_public_validation": false
+  }
+}

challenges/bivariate-bicycle-code-discovery-qcode-discovery/v1/public/lattices.json

@@ -0,0 +1,19 @@
+{
+  "schema_version": 1,
+  "max_candidates": 5000,
+  "official_repetitions": 3,
+  "validation_lattices": [
+    [6, 6],
+    [12, 6]
+  ],
+  "official_lattices": [
+    [12, 6],
+    [6, 12],
+    [12, 12],
+    [24, 6],
+    [15, 12],
+    [30, 6],
+    [16, 9],
+    [18, 8]
+  ]
+}

challenges/bivariate-bicycle-code-discovery-qcode-discovery/v1/resources/evaluator-env/pyproject.toml

@@ -0,0 +1,15 @@
+[project]
+name = "agentics-qcode-discovery-evaluator"
+version = "0.1.0"
+requires-python = ">=3.12,<3.13"
+dependencies = [
+  "galois==0.4.10",
+  "ldpc==2.4.1",
+  "numpy==2.3.5",
+  "qldpc==0.2.6",
+  "scipy==1.17.0",
+  "sympy==1.14.0",
+]
+
+[tool.uv]
+package = false

challenges/bivariate-bicycle-code-discovery-qcode-discovery/v1/separated-evaluator/run.py

@@ -0,0 +1,191 @@
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+from enum import Enum
+from pathlib import Path
+from typing import Any
+
+
+ENV_PROJECT_DIR = "evaluator-env"
+ENV_ACTIVE = "AGENTICS_QCODE_EVALUATOR_ENV_ACTIVE"
+
+
+class EvaluationMode(str, Enum):
+    VALIDATION = "validation"
+    OFFICIAL = "official"
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Score qcode-discovery generator outputs")
+    parser.add_argument("--challenge-dir", required=True)
+    parser.add_argument("--solution-runs-dir", required=True)
+    parser.add_argument("--output-path", required=True)
+    parser.add_argument("--mode", choices=[mode.value for mode in EvaluationMode], required=True)
+    parser.add_argument("--target", required=True)
+    parser.add_argument("--setup-dir", required=True)
+    parser.add_argument("--runs-file", required=True)
+    return parser.parse_args()
+
+
+def maybe_reexec(args: argparse.Namespace) -> None:
+    if os.environ.get(ENV_ACTIVE) == "1":
+        return
+    python = Path(args.setup_dir) / ENV_PROJECT_DIR / ".venv" / "bin" / "python"
+    if not python.is_file():
+        raise RuntimeError(f"missing evaluator environment at {python}")
+    env = os.environ.copy()
+    env[ENV_ACTIVE] = "1"
+    env["HOME"] = str(Path(args.output_path).parent)
+    env["TMPDIR"] = str(Path(args.output_path).parent / "tmp")
+    env["PYTHONDONTWRITEBYTECODE"] = "1"
+    Path(env["TMPDIR"]).mkdir(parents=True, exist_ok=True)
+    os.execve(str(python), [str(python), *sys.argv], env)
+
+
+def metric(name: str, value: float | int) -> dict[str, Any]:
+    return {"metric_name": name, "value": float(value)}
+
+
+def lattice_metric_name(ell: int, m: int) -> str:
+    return f"credible_fom_{ell}x{m}"
+
+
+def write_result(path: Path, payload: dict[str, Any]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(payload, indent=2), encoding="utf-8")
+
+
+def validation_payload(result: dict[str, Any], lattices: list[tuple[int, int]], warnings: tuple[str, ...]) -> dict[str, Any]:
+    covered = int(result["covered_lattices"])
+    passed = covered == len(lattices)
+    run_metrics = []
+    public_results = []
+    for lattice in lattices:
+        stats = result["per_lattice"].get(lattice, {"valid_codes": 0.0, "high_k_codes": 0.0, "best_rate": 0.0})
+        run_name = f"lattice-{lattice[0]}x{lattice[1]}"
+        valid = int(stats["valid_codes"])
+        run_metrics.append(
+            {
+                "run_name": run_name,
+                "metrics": [
+                    metric("valid_codes", valid),
+                    metric("high_k_codes", stats["high_k_codes"]),
+                ],
+            }
+        )
+        public_results.append(
+            {
+                "case_name": run_name,
+                "status": "passed" if valid > 0 else "failed",
+                "score": float(stats["best_rate"]),
+                "message": f"valid_codes={valid}, high_k_codes={int(stats['high_k_codes'])}, best_encoding_rate={stats['best_rate']:.6f}",
+            }
+        )
+    return {
+        "status": "passed" if passed else "failed",
+        "mode": EvaluationMode.VALIDATION.value,
+        "aggregate_metrics": [
+            metric("stage1_score", result["stage1_score"]),
+            metric("valid_codes", result["valid_codes"]),
+            metric("high_k_codes", result["high_k_codes"]),
+            metric("lattices_with_high_k", result["lattices_with_high_k"]),
+            metric("total_candidates", result["total_candidates"]),
+        ],
+        "run_metrics": run_metrics,
+        "public_results": public_results,
+        "validation_summary": {
+            "score": float(result["stage1_score"]),
+            "passed": covered,
+            "total": len(lattices),
+        },
+        "logs": [
+            f"IBM stage-1 score={result['stage1_score']:.6f}; covered_lattices={covered}/{len(lattices)}",
+            *warnings[:40],
+        ],
+    }
+
+
+def official_payload(result: dict[str, Any], lattices: list[tuple[int, int]], warnings: tuple[str, ...]) -> dict[str, Any]:
+    aggregate_metrics = [
+        metric("score", result["score"]),
+        metric("score_stddev", result["score_stddev"]),
+        metric("score_min", result["score_min"]),
+        metric("score_max", result["score_max"]),
+        metric("mean_best_fom", result["mean_best_fom"]),
+        metric("valid_codes", result["valid_codes"]),
+        metric("high_k_codes", result["high_k_codes"]),
+        metric("lattices_with_high_k", result["lattices_with_high_k"]),
+        metric("total_candidates", result["total_candidates"]),
+    ]
+    for ell, m in lattices:
+        aggregate_metrics.append(metric(lattice_metric_name(ell, m), result["per_lattice"][(ell, m)]))
+    return {
+        "status": "passed",
+        "mode": EvaluationMode.OFFICIAL.value,
+        "aggregate_metrics": aggregate_metrics,
+        "run_metrics": [],
+        "public_results": [],
+        "official_summary": {
+            "score": float(result["score"]),
+            "passed": int(result["lattices_with_high_k"]),
+            "total": len(lattices),
+        },
+        "logs": [
+            "three independent IBM-style BP-OSD passes completed",
+            "pass_scores=" + ",".join(f"{score:.6f}" for score in result["pass_scores"]),
+            *warnings[:40],
+        ],
+    }
+
+
+def error_payload(mode: EvaluationMode, message: str, total: int) -> dict[str, Any]:
+    payload: dict[str, Any] = {
+        "status": "error",
+        "mode": mode.value,
+        "aggregate_metrics": [metric("score", 0.0)] if mode is EvaluationMode.OFFICIAL else [],
+        "run_metrics": [],
+        "public_results": [],
+        "logs": [message if mode is EvaluationMode.VALIDATION else "evaluation failed before all repetitions completed"],
+    }
+    summary = {"score": 0.0, "passed": 0, "total": total}
+    if mode is EvaluationMode.VALIDATION:
+        payload["validation_summary"] = summary
+    else:
+        payload["official_summary"] = summary
+    return payload
+
+
+def run() -> int:
+    args = parse_args()
+    maybe_reexec(args)
+    mode = EvaluationMode(args.mode)
+    output_path = Path(args.output_path)
+    try:
+        from scoring import evaluate_official, evaluate_stage1, load_candidate_collection
+
+        runs_payload = json.loads(Path(args.runs_file).read_text(encoding="utf-8"))
+        runs = runs_payload.get("runs")
+        if not isinstance(runs, list):
+            raise ValueError("runs manifest must contain a runs array")
+        lattices = []
+        for run_spec in runs:
+            metadata = run_spec.get("metadata") if isinstance(run_spec, dict) else None
+            if not isinstance(metadata, dict):
+                raise ValueError("run metadata must contain lattice dimensions")
+            lattices.append((int(metadata["ell"]), int(metadata["m"])))
+        collection = load_candidate_collection(runs_payload, Path(args.solution_runs_dir))
+        if mode is EvaluationMode.VALIDATION:
+            payload = validation_payload(evaluate_stage1(collection, lattices), lattices, collection.warnings)
+        else:
+            payload = official_payload(evaluate_official(collection, lattices), lattices, collection.warnings)
+    except Exception as error:  # noqa: BLE001 - evaluator must always explain its terminal state.
+        payload = error_payload(mode, str(error), 2 if mode is EvaluationMode.VALIDATION else 8)
+    write_result(output_path, payload)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(run())