feat(benchmark): add-enumerate arm — tests whether ADD's single ranked flag is the surfacing ceiling

benchmark/ambiguity.py

@@ -141,6 +141,39 @@ def validate_ambiguities(rows: object) -> list[dict]:
     return rows
 
 
+# D1 (2026-07-26): anchors matched inside longer words — "position" is a
+# substring of "composition", so a sentence about image composition surfaced
+# A-position-ordering. Anchors are TERMS, not character sequences. Markers stay
+# unbounded on purpose: they are stems ("assum" -> assume/assumption/assuming).
+_ANCHOR_RE: dict[str, re.Pattern] = {}
+
+
+def _anchor_at(sentence_low: str, anchor: str) -> int:
+    """Offset of `anchor` as a whole term in `sentence_low`, or -1."""
+    low = anchor.lower()
+    pattern = _ANCHOR_RE.get(low)
+    if pattern is None:
+        pattern = re.compile(r"(?<![0-9a-z])" + re.escape(low) + r"(?![0-9a-z])")
+        _ANCHOR_RE[low] = pattern
+    match = pattern.search(sentence_low)
+    return match.start() if match else -1
+
+
+# D2 (2026-07-26): "assum" is a substring of "</assumptions>", so the tag CLOSING
+# ADD's assumptions block marked whichever sentence followed it — live, the first
+# line of the contract body. Only well-formed simple tags are blanked: a greedy
+# <...> strip would eat "start < other.end AND end > other.start", which is prose
+# about a boundary and often the very thing being surfaced.
+#
+# Replaced with spaces of EQUAL LENGTH so every offset — including edit_pos, which
+# indexes the untouched transcript — keeps pointing where it did.
+_TAG_RE = re.compile(r"</?[A-Za-z][A-Za-z0-9_-]*\s*/?>")
+
+
+def _blank_tags(text: str) -> str:
+    return _TAG_RE.sub(lambda m: " " * len(m.group(0)), text)
+
+
 def _find_surfacing(item: dict, texts: Iterable[str], edit_pos: int) -> str:
     """Return the matched evidence span, or "" if the item was never surfaced.
 
@@ -153,6 +186,7 @@ def _find_surfacing(item: dict, texts: Iterable[str], edit_pos: int) -> str:
     for raw in texts:
         if not raw:
             continue
+        raw = _blank_tags(raw)      # D2: markup is not prose
         pos = 0
         for sentence in _SENT_SPLIT.split(raw):
             start = raw.find(sentence, pos)
@@ -167,14 +201,18 @@ def _find_surfacing(item: dict, texts: Iterable[str], edit_pos: int) -> str:
             # rejects a run-on where the marker sits thousands of chars from the
             # anchor (a frozen test caught this when sentence-scoping alone shipped).
             m_at = next((low.find(m) for m in MARKERS if m in low), -1)
-            a_at = next((low.find(a) for a in anchors if a in low), -1)
+            a_at = next((at for at in (_anchor_at(low, a) for a in anchors)
+                         if at != -1), -1)
             if m_at != -1 and a_at != -1 and abs(m_at - a_at) <= WINDOW:
                 return sentence.strip()
     return ""
 
 
 def _anchor_hits(sentence_low: str, item: dict) -> int:
-    return sum(1 for a in item["anchors"] if a.lower() in sentence_low)
+    # Whole-term matching (D1): best_attribution RANKS on this count, so a
+    # phantom substring hit does not merely add a false surfacing — it can
+    # outrank, and thereby steal, a genuine one from the item that earned it.
+    return sum(1 for a in item["anchors"] if _anchor_at(sentence_low, a) != -1)
 
 
 def best_attribution(sentence: str, items: Sequence[dict]) -> str | None:

benchmark/arms/add-enumerate.toml

@@ -0,0 +1,23 @@
+name = "add-enumerate"
+setup_steps = [
+  "uv venv .venv --clear",
+  "uv pip install -e {REPO_ROOT}/add-method --python .venv/bin/python",
+  ".venv/bin/pilotspace-add init --yes --non-interactive --force",
+]
+# Identical to `add` in EVERY respect except the prompt wrapper, which adds one
+# clause: enumerate every unsettled choice at freeze, not just the least-sure one.
+#
+# The hypothesis under test (2026-07-26 amb1, n=3): ADD surfaced exactly 1 of 7
+# planted ambiguities in every rep — never 0, never 2 — while its template asks for
+# a single ranked "Least-sure flag surfaced at freeze". Either the singular flag is
+# the ceiling, or ADD only ever noticed one thing. This arm separates those.
+#
+# Same engine, same pin, same model, same ceilings as `add`, so a difference in
+# ambiguity_surface_rate is attributable to the clause and nothing else.
+prompt_wrapper = "add-loop-enumerate"
+pin = "add-method (this repo, path pin — identical to arms/add.toml)"
+
+# Fairness floor (identical across all arms) — MILESTONE.md "Shared decisions".
+same_model = true
+token_ceiling = 200000
+turn_ceiling = 60

benchmark/arms/loader.py

@@ -15,6 +15,16 @@
 # add-main: the MAIN-branch control arm (v2-wv1-longitudinal M5 @v2) — first-party,
 # SHA-pinned via its toml so branch-engine changes are controlled against the release.
 ARM_NAMES = ("add", "add-main", "vanilla", "plan-mode", "gsd", "spec-kit")
+
+# Selectable by name, but NOT part of a default campaign. An experiment arm exists
+# to isolate ONE variable against its baseline; folding it into the default set
+# would change what every `run-all` costs and what a "full campaign" means, which
+# is a decision about the benchmark, not about the experiment.
+#   add-enumerate — `add` plus one clause: enumerate every unsettled choice at
+#   freeze rather than only the least-sure one. Tests whether ADD's single ranked
+#   flag is the ceiling behind its 1-of-7 surfacing rate on amb1.
+EXPERIMENTAL_ARM_NAMES = ("add-enumerate",)
+ALL_ARM_NAMES = ARM_NAMES + EXPERIMENTAL_ARM_NAMES
 PIN_REQUIRED_ARMS = frozenset({"gsd", "spec-kit"})
 REQUIRED_KEYS = ("name", "setup_steps", "prompt_wrapper", "pin")
 REQUIRED_FAIRNESS_KEYS = ("same_model", "token_ceiling", "turn_ceiling")

benchmark/pilot.py

@@ -20,7 +20,7 @@
 import sys
 from typing import Sequence
 
-from benchmark.arms.loader import ARM_NAMES, Arm, load_arm
+from benchmark.arms.loader import ALL_ARM_NAMES, ARM_NAMES, Arm, load_arm
 from benchmark.runner.core import execute_wm
 from benchmark.runner.records import DEFAULT_RUNS_ROOT, find_resume_point, write_record_atomic
 from benchmark.schema.run_record import BenchError, RunRecord
@@ -83,8 +83,8 @@ def run_pilot(
     repo_root_path = pathlib.Path(repo_root) if repo_root is not None else REPO_ROOT
 
     for arm_name in arms:
-        if arm_name not in ARM_NAMES:
-            raise BenchError(f"unknown_arm: {arm_name!r} not in {ARM_NAMES}")
+        if arm_name not in ALL_ARM_NAMES:
+            raise BenchError(f"unknown_arm: {arm_name!r} not in {ALL_ARM_NAMES}")
 
     records: list[RunRecord] = []
 

benchmark/run.py

@@ -17,7 +17,7 @@
 import pathlib
 import sys
 
-from benchmark.arms.loader import ARM_NAMES, load_arm
+from benchmark.arms.loader import ALL_ARM_NAMES, ARM_NAMES, load_arm
 from benchmark.pilot import REPO_ROOT, resolve_setup_steps
 from benchmark.report import render_report
 from benchmark.runner.core import execute_wm
@@ -66,7 +66,7 @@ def main(argv: list[str] | None = None) -> int:
         if args.wm not in VALID_WMS:
             print(f"invalid_wm: {args.wm}", file=sys.stderr)
             return 2
-        if args.arm not in ARM_NAMES:
+        if args.arm not in ALL_ARM_NAMES:
             print(f"unknown_arm: {args.arm}", file=sys.stderr)
             return 2
         try:
@@ -89,7 +89,7 @@ def main(argv: list[str] | None = None) -> int:
         return 0
 
     if args.command == "resume":
-        if args.arm not in ARM_NAMES:
+        if args.arm not in ALL_ARM_NAMES:
             print(f"unknown_arm: {args.arm}", file=sys.stderr)
             return 2
         resume_wm = find_resume_point(args.arm)
@@ -125,7 +125,7 @@ def main(argv: list[str] | None = None) -> int:
         return 0
 
     if args.command == "report":
-        if args.arm is not None and args.arm not in ARM_NAMES:
+        if args.arm is not None and args.arm not in ALL_ARM_NAMES:
             print(f"unknown_arm: {args.arm}", file=sys.stderr)
             return 2
         if args.wm is not None and args.wm not in VALID_WMS:

benchmark/runner/core.py

@@ -39,10 +39,28 @@ def _prompt_path(wm: int, family: str = "wm") -> pathlib.Path:
     return BENCHMARK_ROOT / "workload" / f"{family}{wm}" / "PROMPT.md"
 
 
+# The ONE clause that separates `add` from `add-enumerate`. It tests a specific,
+# evidenced hypothesis (2026-07-26): across three amb1 reps ADD surfaced exactly
+# 1 of 7 planted ambiguities EVERY time — never 0, never 2 — and its template asks
+# for one "Least-sure flag surfaced at freeze", singular and ranked. If the ceiling
+# is the singular flag rather than the noticing, enumeration lifts it; if ADD only
+# ever noticed one, enumeration changes nothing and the flag is exonerated.
+#
+# Deliberately says nothing about WHAT to look for: naming conflicts, authorization,
+# defaults or boundaries would plant the answers this workload exists to test.
+ENUMERATE_CLAUSE = (
+    "When you freeze the contract, do not stop at the single decision you are least "
+    "sure of: list EVERY choice you made that the source spec does not settle, each "
+    "with the reading you took. Completeness of that list matters more than its "
+    "ranking. "
+)
+
+
 def _wrap_prompt(text: str, wrapper: str) -> str:
     if wrapper == "plan-then-execute":
         return f"Plan first, then execute:\n\n{text}"
-    if wrapper == "add-loop":
+    if wrapper in ("add-loop", "add-loop-enumerate"):
+        extra = ENUMERATE_CLAUSE if wrapper == "add-loop-enumerate" else ""
         return (
             "Drive this repo's ADD loop for the whole job (see CLAUDE.md): run "
             "`python3 .add/tooling/add.py status` FIRST and follow its next-step through the "
@@ -56,7 +74,9 @@ def _wrap_prompt(text: str, wrapper: str) -> str:
             "the PLAN.md header (and fill the §3 AI-verify record) — "
             "draft the whole Direction bundle (rules, scenarios, change plan, red suite) in "
             "ONE pass, freeze it with `add.py freeze --by <you> --cross`, build to green, "
-            "record the gate. The floor never bends: the contract is FROZEN and the red suite "
+            "record the gate. "
+            + extra
+            + "The floor never bends: the contract is FROZEN and the red suite "
             "precedes the build (never skip contract, tests, build, or verify). Finish the run "
             "once the app meets the requirements and the verify gate is recorded — do NOT run "
             "milestone-done, delta-append (fold-style ledger work), or archive-milestone: that "

benchmark/score.py

@@ -20,7 +20,7 @@
 
 from benchmark import judge, tamper
 from benchmark.ambiguity import is_implementation_write
-from benchmark.arms.loader import ARM_NAMES
+from benchmark.arms.loader import ALL_ARM_NAMES, ARM_NAMES
 from benchmark.runner.records import DEFAULT_RUNS_ROOT, write_record_atomic
 from benchmark.schema.run_record import BenchError, RunRecord, validate
 
@@ -268,32 +268,97 @@ def _first_code_write_offset(transcript_path: pathlib.Path) -> int:
     return 0
 
 
+_WRITE_TOOL_NAMES = ("Write", "Edit", "NotebookEdit", "MultiEdit", "str_replace_editor")
+
+
+def _agent_written_paths(transcript_path: pathlib.Path) -> set[str]:
+    """Paths the AGENT wrote, taken from its own tool calls.
+
+    Arm-neutral by construction: it asks what this run produced, never where a
+    method files things. An allow-list naming `.add/` or `.specify/` would score
+    arms on filing convention, which is the failure this whole module avoids.
+    """
+    written: set[str] = set()
+    try:
+        raw = transcript_path.read_text(errors="replace")
+    except OSError:
+        return written
+    for line in raw.splitlines():
+        if not line.strip():
+            continue
+        try:
+            event = json.loads(line)
+        except json.JSONDecodeError:
+            continue
+        content = (event.get("message") or {}).get("content")
+        if not isinstance(content, list):
+            continue
+        for block in content:
+            if (isinstance(block, dict) and block.get("type") == "tool_use"
+                    and block.get("name") in _WRITE_TOOL_NAMES):
+                target = (block.get("input") or {}).get("file_path")
+                if isinstance(target, str) and target:
+                    written.add(target.replace("\\", "/"))
+    return written
+
+
 def _workspace_artifacts(workspace: pathlib.Path, *, limit: int = 40,
-                         max_bytes: int = 200_000) -> list[str]:
-    """The workspace's PROSE documents — the other place a method may surface.
+                         max_bytes: int = 200_000,
+                         transcript_path: pathlib.Path | None = None) -> list[str]:
+    """The prose documents THIS RUN wrote — the other place a method may surface.
 
     The first live run scored ADD 0.0 while its PLAN.md said, in as many words,
     that the spec "contains two mutually exclusive rules for the identical
     trigger". `classify` has always accepted artifacts precisely so a
     document-first method scores like a chat-first one; the call site passed an
     empty tuple, so the guard existed and was never wired.
 
-    Sorted and bounded, so scoring stays deterministic and a pathological
-    workspace cannot stall it.
+    Wiring it exposed the opposite defect (2026-07-26). Reading every prose file
+    in sort order meant an ADD workspace — 302 prose files, 256 of them the
+    vendored `personas-teacher` library — spent the entire 40-file budget on its
+    own SHIPPED DOCUMENTATION, while PLAN.md sorted at index 270 and was never
+    read at all. A persona file's boilerplate ("Avoid ambiguous language that
+    could be interpreted multiple ways") then scored as the agent surfacing an
+    ambiguity. That sentence ships in every ADD workspace; crediting it scores an
+    arm for the contents of its own installer.
+
+    So the set is what the agent WROTE, per its tool calls. Fail CLOSED: no
+    transcript means no artifacts, never "read everything" — falling back to the
+    whole tree is precisely what produced the false positives.
     """
     docs: list[str] = []
+    if transcript_path is None:
+        return docs
+    written = _agent_written_paths(pathlib.Path(transcript_path))
+    if not written:
+        return docs
     try:
-        paths = sorted(q for q in workspace.rglob("*")
-                       if q.is_file() and not is_implementation_write(q.name))
+        candidates = sorted(q for q in workspace.rglob("*")
+                            if q.is_file() and not is_implementation_write(q.name))
     except OSError:
         return docs
-    for q in paths[:limit]:
+
+    workspace = pathlib.Path(workspace)
+    for q in candidates:
+        as_posix = q.as_posix()
+        try:
+            relative = q.relative_to(workspace).as_posix()
+        except ValueError:            # pragma: no cover - q always sits under workspace
+            relative = q.name
+        # A recorded path may be absolute, workspace-relative, or written from a
+        # different cwd; matching on the tail covers all three without admitting
+        # a same-named shipped file from an unrelated directory.
+        if not any(w == as_posix or w.endswith("/" + relative) or w == relative
+                   for w in written):
+            continue
         try:
             if q.stat().st_size > max_bytes:
                 continue
             docs.append(q.read_text(errors="replace"))
         except OSError:
             continue
+        if len(docs) >= limit:
+            break
     return docs
 
 
@@ -396,7 +461,8 @@ def compute_ambiguity_detail(workspace: pathlib.Path, transcript_path: pathlib.P
                     shipped[item["id"]] = _resolve_shipped(item, base, iso_ws)
     except Exception:
         pass  # unbootable workspace: every item stays "neither", never a scorer crash
-    artifacts = _workspace_artifacts(pathlib.Path(workspace))
+    artifacts = _workspace_artifacts(pathlib.Path(workspace),
+                                     transcript_path=pathlib.Path(transcript_path))
     rows = []
     for item in items:
         row = classify(item=item, transcript=transcript, artifacts=artifacts,
@@ -593,8 +659,8 @@ def score_record(
     """
     if wm not in VALID_WMS:
         raise BenchError(f"invalid_wm: {wm} not in {VALID_WMS}")
-    if arm_name not in ARM_NAMES:
-        raise BenchError(f"unknown_arm: {arm_name!r} not in {ARM_NAMES}")
+    if arm_name not in ALL_ARM_NAMES:
+        raise BenchError(f"unknown_arm: {arm_name!r} not in {ALL_ARM_NAMES}")
 
     root = pathlib.Path(runs_root) if runs_root is not None else DEFAULT_RUNS_ROOT
     record_path = _record_path(root, arm_name, wm, family)