diff --git a/benchmark/ambiguity.py b/benchmark/ambiguity.py index b05c5da0..21545188 100644 --- a/benchmark/ambiguity.py +++ b/benchmark/ambiguity.py @@ -141,6 +141,39 @@ def validate_ambiguities(rows: object) -> list[dict]: return rows +# D1 (2026-07-26): anchors matched inside longer words — "position" is a +# substring of "composition", so a sentence about image composition surfaced +# A-position-ordering. Anchors are TERMS, not character sequences. Markers stay +# unbounded on purpose: they are stems ("assum" -> assume/assumption/assuming). +_ANCHOR_RE: dict[str, re.Pattern] = {} + + +def _anchor_at(sentence_low: str, anchor: str) -> int: + """Offset of `anchor` as a whole term in `sentence_low`, or -1.""" + low = anchor.lower() + pattern = _ANCHOR_RE.get(low) + if pattern is None: + pattern = re.compile(r"(?", so the tag CLOSING +# ADD's assumptions block marked whichever sentence followed it — live, the first +# line of the contract body. Only well-formed simple tags are blanked: a greedy +# <...> strip would eat "start < other.end AND end > other.start", which is prose +# about a boundary and often the very thing being surfaced. +# +# Replaced with spaces of EQUAL LENGTH so every offset — including edit_pos, which +# indexes the untouched transcript — keeps pointing where it did. +_TAG_RE = re.compile(r"") + + +def _blank_tags(text: str) -> str: + return _TAG_RE.sub(lambda m: " " * len(m.group(0)), text) + + def _find_surfacing(item: dict, texts: Iterable[str], edit_pos: int) -> str: """Return the matched evidence span, or "" if the item was never surfaced. @@ -153,6 +186,7 @@ def _find_surfacing(item: dict, texts: Iterable[str], edit_pos: int) -> str: for raw in texts: if not raw: continue + raw = _blank_tags(raw) # D2: markup is not prose pos = 0 for sentence in _SENT_SPLIT.split(raw): start = raw.find(sentence, pos) @@ -167,14 +201,18 @@ def _find_surfacing(item: dict, texts: Iterable[str], edit_pos: int) -> str: # rejects a run-on where the marker sits thousands of chars from the # anchor (a frozen test caught this when sentence-scoping alone shipped). m_at = next((low.find(m) for m in MARKERS if m in low), -1) - a_at = next((low.find(a) for a in anchors if a in low), -1) + a_at = next((at for at in (_anchor_at(low, a) for a in anchors) + if at != -1), -1) if m_at != -1 and a_at != -1 and abs(m_at - a_at) <= WINDOW: return sentence.strip() return "" def _anchor_hits(sentence_low: str, item: dict) -> int: - return sum(1 for a in item["anchors"] if a.lower() in sentence_low) + # Whole-term matching (D1): best_attribution RANKS on this count, so a + # phantom substring hit does not merely add a false surfacing — it can + # outrank, and thereby steal, a genuine one from the item that earned it. + return sum(1 for a in item["anchors"] if _anchor_at(sentence_low, a) != -1) def best_attribution(sentence: str, items: Sequence[dict]) -> str | None: diff --git a/benchmark/arms/add-enumerate.toml b/benchmark/arms/add-enumerate.toml new file mode 100644 index 00000000..fdbc98e7 --- /dev/null +++ b/benchmark/arms/add-enumerate.toml @@ -0,0 +1,23 @@ +name = "add-enumerate" +setup_steps = [ + "uv venv .venv --clear", + "uv pip install -e {REPO_ROOT}/add-method --python .venv/bin/python", + ".venv/bin/pilotspace-add init --yes --non-interactive --force", +] +# Identical to `add` in EVERY respect except the prompt wrapper, which adds one +# clause: enumerate every unsettled choice at freeze, not just the least-sure one. +# +# The hypothesis under test (2026-07-26 amb1, n=3): ADD surfaced exactly 1 of 7 +# planted ambiguities in every rep — never 0, never 2 — while its template asks for +# a single ranked "Least-sure flag surfaced at freeze". Either the singular flag is +# the ceiling, or ADD only ever noticed one thing. This arm separates those. +# +# Same engine, same pin, same model, same ceilings as `add`, so a difference in +# ambiguity_surface_rate is attributable to the clause and nothing else. +prompt_wrapper = "add-loop-enumerate" +pin = "add-method (this repo, path pin — identical to arms/add.toml)" + +# Fairness floor (identical across all arms) — MILESTONE.md "Shared decisions". +same_model = true +token_ceiling = 200000 +turn_ceiling = 60 diff --git a/benchmark/arms/loader.py b/benchmark/arms/loader.py index 4a6cf6bc..67597fc1 100644 --- a/benchmark/arms/loader.py +++ b/benchmark/arms/loader.py @@ -15,6 +15,16 @@ # add-main: the MAIN-branch control arm (v2-wv1-longitudinal M5 @v2) — first-party, # SHA-pinned via its toml so branch-engine changes are controlled against the release. ARM_NAMES = ("add", "add-main", "vanilla", "plan-mode", "gsd", "spec-kit") + +# Selectable by name, but NOT part of a default campaign. An experiment arm exists +# to isolate ONE variable against its baseline; folding it into the default set +# would change what every `run-all` costs and what a "full campaign" means, which +# is a decision about the benchmark, not about the experiment. +# add-enumerate — `add` plus one clause: enumerate every unsettled choice at +# freeze rather than only the least-sure one. Tests whether ADD's single ranked +# flag is the ceiling behind its 1-of-7 surfacing rate on amb1. +EXPERIMENTAL_ARM_NAMES = ("add-enumerate",) +ALL_ARM_NAMES = ARM_NAMES + EXPERIMENTAL_ARM_NAMES PIN_REQUIRED_ARMS = frozenset({"gsd", "spec-kit"}) REQUIRED_KEYS = ("name", "setup_steps", "prompt_wrapper", "pin") REQUIRED_FAIRNESS_KEYS = ("same_model", "token_ceiling", "turn_ceiling") diff --git a/benchmark/pilot.py b/benchmark/pilot.py index 48605e60..a0e9032c 100644 --- a/benchmark/pilot.py +++ b/benchmark/pilot.py @@ -20,7 +20,7 @@ import sys from typing import Sequence -from benchmark.arms.loader import ARM_NAMES, Arm, load_arm +from benchmark.arms.loader import ALL_ARM_NAMES, ARM_NAMES, Arm, load_arm from benchmark.runner.core import execute_wm from benchmark.runner.records import DEFAULT_RUNS_ROOT, find_resume_point, write_record_atomic from benchmark.schema.run_record import BenchError, RunRecord @@ -83,8 +83,8 @@ def run_pilot( repo_root_path = pathlib.Path(repo_root) if repo_root is not None else REPO_ROOT for arm_name in arms: - if arm_name not in ARM_NAMES: - raise BenchError(f"unknown_arm: {arm_name!r} not in {ARM_NAMES}") + if arm_name not in ALL_ARM_NAMES: + raise BenchError(f"unknown_arm: {arm_name!r} not in {ALL_ARM_NAMES}") records: list[RunRecord] = [] diff --git a/benchmark/run.py b/benchmark/run.py index c0ccc385..3223f797 100644 --- a/benchmark/run.py +++ b/benchmark/run.py @@ -17,7 +17,7 @@ import pathlib import sys -from benchmark.arms.loader import ARM_NAMES, load_arm +from benchmark.arms.loader import ALL_ARM_NAMES, ARM_NAMES, load_arm from benchmark.pilot import REPO_ROOT, resolve_setup_steps from benchmark.report import render_report from benchmark.runner.core import execute_wm @@ -66,7 +66,7 @@ def main(argv: list[str] | None = None) -> int: if args.wm not in VALID_WMS: print(f"invalid_wm: {args.wm}", file=sys.stderr) return 2 - if args.arm not in ARM_NAMES: + if args.arm not in ALL_ARM_NAMES: print(f"unknown_arm: {args.arm}", file=sys.stderr) return 2 try: @@ -89,7 +89,7 @@ def main(argv: list[str] | None = None) -> int: return 0 if args.command == "resume": - if args.arm not in ARM_NAMES: + if args.arm not in ALL_ARM_NAMES: print(f"unknown_arm: {args.arm}", file=sys.stderr) return 2 resume_wm = find_resume_point(args.arm) @@ -125,7 +125,7 @@ def main(argv: list[str] | None = None) -> int: return 0 if args.command == "report": - if args.arm is not None and args.arm not in ARM_NAMES: + if args.arm is not None and args.arm not in ALL_ARM_NAMES: print(f"unknown_arm: {args.arm}", file=sys.stderr) return 2 if args.wm is not None and args.wm not in VALID_WMS: diff --git a/benchmark/runner/core.py b/benchmark/runner/core.py index f7187f76..819901b0 100644 --- a/benchmark/runner/core.py +++ b/benchmark/runner/core.py @@ -39,10 +39,28 @@ def _prompt_path(wm: int, family: str = "wm") -> pathlib.Path: return BENCHMARK_ROOT / "workload" / f"{family}{wm}" / "PROMPT.md" +# The ONE clause that separates `add` from `add-enumerate`. It tests a specific, +# evidenced hypothesis (2026-07-26): across three amb1 reps ADD surfaced exactly +# 1 of 7 planted ambiguities EVERY time — never 0, never 2 — and its template asks +# for one "Least-sure flag surfaced at freeze", singular and ranked. If the ceiling +# is the singular flag rather than the noticing, enumeration lifts it; if ADD only +# ever noticed one, enumeration changes nothing and the flag is exonerated. +# +# Deliberately says nothing about WHAT to look for: naming conflicts, authorization, +# defaults or boundaries would plant the answers this workload exists to test. +ENUMERATE_CLAUSE = ( + "When you freeze the contract, do not stop at the single decision you are least " + "sure of: list EVERY choice you made that the source spec does not settle, each " + "with the reading you took. Completeness of that list matters more than its " + "ranking. " +) + + def _wrap_prompt(text: str, wrapper: str) -> str: if wrapper == "plan-then-execute": return f"Plan first, then execute:\n\n{text}" - if wrapper == "add-loop": + if wrapper in ("add-loop", "add-loop-enumerate"): + extra = ENUMERATE_CLAUSE if wrapper == "add-loop-enumerate" else "" return ( "Drive this repo's ADD loop for the whole job (see CLAUDE.md): run " "`python3 .add/tooling/add.py status` FIRST and follow its next-step through the " @@ -56,7 +74,9 @@ def _wrap_prompt(text: str, wrapper: str) -> str: "the PLAN.md header (and fill the §3 AI-verify record) — " "draft the whole Direction bundle (rules, scenarios, change plan, red suite) in " "ONE pass, freeze it with `add.py freeze --by --cross`, build to green, " - "record the gate. The floor never bends: the contract is FROZEN and the red suite " + "record the gate. " + + extra + + "The floor never bends: the contract is FROZEN and the red suite " "precedes the build (never skip contract, tests, build, or verify). Finish the run " "once the app meets the requirements and the verify gate is recorded — do NOT run " "milestone-done, delta-append (fold-style ledger work), or archive-milestone: that " diff --git a/benchmark/score.py b/benchmark/score.py index a84e8aa7..ece75ffd 100644 --- a/benchmark/score.py +++ b/benchmark/score.py @@ -20,7 +20,7 @@ from benchmark import judge, tamper from benchmark.ambiguity import is_implementation_write -from benchmark.arms.loader import ARM_NAMES +from benchmark.arms.loader import ALL_ARM_NAMES, ARM_NAMES from benchmark.runner.records import DEFAULT_RUNS_ROOT, write_record_atomic from benchmark.schema.run_record import BenchError, RunRecord, validate @@ -268,9 +268,44 @@ def _first_code_write_offset(transcript_path: pathlib.Path) -> int: return 0 +_WRITE_TOOL_NAMES = ("Write", "Edit", "NotebookEdit", "MultiEdit", "str_replace_editor") + + +def _agent_written_paths(transcript_path: pathlib.Path) -> set[str]: + """Paths the AGENT wrote, taken from its own tool calls. + + Arm-neutral by construction: it asks what this run produced, never where a + method files things. An allow-list naming `.add/` or `.specify/` would score + arms on filing convention, which is the failure this whole module avoids. + """ + written: set[str] = set() + try: + raw = transcript_path.read_text(errors="replace") + except OSError: + return written + for line in raw.splitlines(): + if not line.strip(): + continue + try: + event = json.loads(line) + except json.JSONDecodeError: + continue + content = (event.get("message") or {}).get("content") + if not isinstance(content, list): + continue + for block in content: + if (isinstance(block, dict) and block.get("type") == "tool_use" + and block.get("name") in _WRITE_TOOL_NAMES): + target = (block.get("input") or {}).get("file_path") + if isinstance(target, str) and target: + written.add(target.replace("\\", "/")) + return written + + def _workspace_artifacts(workspace: pathlib.Path, *, limit: int = 40, - max_bytes: int = 200_000) -> list[str]: - """The workspace's PROSE documents — the other place a method may surface. + max_bytes: int = 200_000, + transcript_path: pathlib.Path | None = None) -> list[str]: + """The prose documents THIS RUN wrote — the other place a method may surface. The first live run scored ADD 0.0 while its PLAN.md said, in as many words, that the spec "contains two mutually exclusive rules for the identical @@ -278,22 +313,52 @@ def _workspace_artifacts(workspace: pathlib.Path, *, limit: int = 40, document-first method scores like a chat-first one; the call site passed an empty tuple, so the guard existed and was never wired. - Sorted and bounded, so scoring stays deterministic and a pathological - workspace cannot stall it. + Wiring it exposed the opposite defect (2026-07-26). Reading every prose file + in sort order meant an ADD workspace — 302 prose files, 256 of them the + vendored `personas-teacher` library — spent the entire 40-file budget on its + own SHIPPED DOCUMENTATION, while PLAN.md sorted at index 270 and was never + read at all. A persona file's boilerplate ("Avoid ambiguous language that + could be interpreted multiple ways") then scored as the agent surfacing an + ambiguity. That sentence ships in every ADD workspace; crediting it scores an + arm for the contents of its own installer. + + So the set is what the agent WROTE, per its tool calls. Fail CLOSED: no + transcript means no artifacts, never "read everything" — falling back to the + whole tree is precisely what produced the false positives. """ docs: list[str] = [] + if transcript_path is None: + return docs + written = _agent_written_paths(pathlib.Path(transcript_path)) + if not written: + return docs try: - paths = sorted(q for q in workspace.rglob("*") - if q.is_file() and not is_implementation_write(q.name)) + candidates = sorted(q for q in workspace.rglob("*") + if q.is_file() and not is_implementation_write(q.name)) except OSError: return docs - for q in paths[:limit]: + + workspace = pathlib.Path(workspace) + for q in candidates: + as_posix = q.as_posix() + try: + relative = q.relative_to(workspace).as_posix() + except ValueError: # pragma: no cover - q always sits under workspace + relative = q.name + # A recorded path may be absolute, workspace-relative, or written from a + # different cwd; matching on the tail covers all three without admitting + # a same-named shipped file from an unrelated directory. + if not any(w == as_posix or w.endswith("/" + relative) or w == relative + for w in written): + continue try: if q.stat().st_size > max_bytes: continue docs.append(q.read_text(errors="replace")) except OSError: continue + if len(docs) >= limit: + break return docs @@ -396,7 +461,8 @@ def compute_ambiguity_detail(workspace: pathlib.Path, transcript_path: pathlib.P shipped[item["id"]] = _resolve_shipped(item, base, iso_ws) except Exception: pass # unbootable workspace: every item stays "neither", never a scorer crash - artifacts = _workspace_artifacts(pathlib.Path(workspace)) + artifacts = _workspace_artifacts(pathlib.Path(workspace), + transcript_path=pathlib.Path(transcript_path)) rows = [] for item in items: row = classify(item=item, transcript=transcript, artifacts=artifacts, @@ -593,8 +659,8 @@ def score_record( """ if wm not in VALID_WMS: raise BenchError(f"invalid_wm: {wm} not in {VALID_WMS}") - if arm_name not in ARM_NAMES: - raise BenchError(f"unknown_arm: {arm_name!r} not in {ARM_NAMES}") + if arm_name not in ALL_ARM_NAMES: + raise BenchError(f"unknown_arm: {arm_name!r} not in {ALL_ARM_NAMES}") root = pathlib.Path(runs_root) if runs_root is not None else DEFAULT_RUNS_ROOT record_path = _record_path(root, arm_name, wm, family) diff --git a/benchmark/tests/test_ambiguity_meter_fixes.py b/benchmark/tests/test_ambiguity_meter_fixes.py index 5b56a256..9db6adb6 100644 --- a/benchmark/tests/test_ambiguity_meter_fixes.py +++ b/benchmark/tests/test_ambiguity_meter_fixes.py @@ -103,11 +103,21 @@ def test_one_sentence_credits_at_most_one_item(self): class TestArtifactsAreRead: def test_workspace_artifacts_reads_prose_not_code(self, tmp_path): + # AMENDED 2026-07-26 (D3): artifacts are now the documents the AGENT + # WROTE, not every prose file present. Reading the whole tree let an ADD + # workspace's 256 vendored persona files consume the budget and score as + # the agent's reasoning, so the transcript must vouch for each file. (tmp_path / "PLAN.md").write_text("the spec is ambiguous about the waitlist", encoding="utf-8") (tmp_path / "app.py").write_text("print('code')", encoding="utf-8") - docs = _workspace_artifacts(tmp_path) + tx = tmp_path / "t.jsonl" + tx.write_text("\n".join(json.dumps({"type": "assistant", "message": {"content": [ + {"type": "tool_use", "name": "Write", + "input": {"file_path": str(tmp_path / name), "content": "..."}}]}}) + for name in ("PLAN.md", "app.py")) + "\n", encoding="utf-8") + docs = _workspace_artifacts(tmp_path, transcript_path=tx) assert any("ambiguous" in d for d in docs) + # Still excluded — by file KIND, even though the agent wrote it too. assert not any("print('code')" in d for d in docs) def test_document_only_surfacing_scores(self): @@ -205,6 +215,14 @@ class TestArtifactWiringIsPinned: This test can only pass if compute_ambiguity_detail actually reads the workspace: the recognition exists ONLY in a file on disk, and the transcript never mentions it. + + AMENDED 2026-07-26 (D3). Artifacts are now the documents the agent WROTE, so + the transcript must record the Write — but its PAYLOAD still does not contain + the recognition. That is the realistic shape: an Edit records a replacement + slice, and after several edits the file on disk says things no single payload + ever did. The guard is unchanged in substance — pass only by reading the FILE — + while matching the contract that stops the scorer from crediting an arm for + its own installed documentation. """ def test_surfacing_only_on_disk_is_found_by_compute(self, tmp_path, monkeypatch): @@ -226,11 +244,17 @@ def test_surfacing_only_on_disk_is_found_by_compute(self, tmp_path, monkeypatch) (ws / "PLAN.md").write_text( "The spec is contradictory: it both waitlists and returns 409.", encoding="utf-8") - # A transcript that says nothing about it. + # A transcript that records WRITING the file but whose payload says + # nothing about the recognition — so only reading the file can find it. tx = tmp_path / "t.jsonl" - tx.write_text(json.dumps({"type": "assistant", "message": {"content": [ - {"type": "text", "text": "Building the service now."}]}}) + "\n", - encoding="utf-8") + tx.write_text("\n".join(json.dumps(e) for e in [ + {"type": "assistant", "message": {"content": [ + {"type": "text", "text": "Building the service now."}]}}, + {"type": "assistant", "message": {"content": [ + {"type": "tool_use", "name": "Write", + "input": {"file_path": str(ws / "PLAN.md"), + "content": "# PLAN\n(section stub)\n"}}]}}, + ]) + "\n", encoding="utf-8") detail = score.compute_ambiguity_detail(ws, tx, 9, "amb") assert detail[0]["verdict"] == "surfaced", \ diff --git a/benchmark/tests/test_detector_false_positives.py b/benchmark/tests/test_detector_false_positives.py new file mode 100644 index 00000000..ec8869cf --- /dev/null +++ b/benchmark/tests/test_detector_false_positives.py @@ -0,0 +1,149 @@ +"""Three defects that made add-enumerate's first run report 3/7 when it earned 1/7. + +All three found by READING the evidence spans of a result that moved in the +direction I was hoping for. The rate alone looked like a clean win. + +D1 — ANCHORS MATCH INSIDE LONGER WORDS. "position" is a substring of + "composition", so a sentence about image composition surfaced + A-position-ordering. + +D2 — A CLOSING XML TAG IS A MARKER. "assum" is a substring of "", + so the tag that ENDS ADD's assumptions block marks whatever sentence follows + it — in the live case, the opening line of the contract body. + +D3 — THE ARTIFACT BUDGET READS SHIPPED DOCUMENTATION. `_workspace_artifacts` + takes the first 40 prose files in sort order. An ADD workspace contains 302, + of which 256 are the vendored `personas-teacher` library, so the budget is + consumed entirely by ADD's own product documentation — and PLAN.md, the + agent's actual reasoning, sorts at index 270 and is never read at all. + + The live false positive came from + `.add/personas-teacher/design/design-image-prompt-engineer.md`, whose + boilerplate says "Avoid ambiguous language that could be interpreted multiple + ways". That sentence ships in every ADD workspace. It is not a run's output; + scoring it credits an arm for the contents of its own installer. + +D3 is the same defect as the original `artifacts = ()` bug wearing the opposite +sign: that one read none of the agent's documents, this one reads everything +EXCEPT them. Both were invisible because the surrounding tests exercised +`classify` directly and never the seam that chooses what to feed it. +""" +from __future__ import annotations + +import json +import pathlib +import sys + +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[2])) + +from benchmark.ambiguity import _anchor_hits, _find_surfacing +from benchmark.score import _workspace_artifacts + +# The LIVE strings, verbatim from runs-enum-2026-07-26-rep0's record. +COMPOSITION = ("- Consider aspect ratio and composition in every prompt\n" + "- Avoid ambiguous language that could be interpreted multiple ways") +CLOSING_TAG = ('\n```\nPOST /bookings body: { title, start_time, ' + 'end_time, room_id, priority? }') + +_POSITION = {"id": "A-position-ordering", "klass": "trap", + "anchors": ("position", "promoted next", "position 1"), + "readings": {}, "defensible": ""} +_PRIORITY = {"id": "A-priority-vs-fifo", "klass": "gap", + "anchors": ("priority", "promoted"), + "readings": {}, "defensible": ""} + + +class TestAnchorsRespectWordBoundaries: + def test_composition_does_not_surface_position_ordering(self): + assert not _find_surfacing(_POSITION, (COMPOSITION,), 10**9), ( + "'position' matched inside 'composition'") + + def test_anchor_hits_does_not_count_a_substring_match(self): + # best_attribution ranks on this count, so a phantom hit can also STEAL + # a genuine surfacing away from the item that earned it. + assert _anchor_hits(COMPOSITION.lower(), _POSITION) == 0 + + def test_a_real_mention_still_counts(self): + # Needs a MARKER as well as the anchor — "never says" is not in MARKERS, + # and a sentence without one is not a surfacing by this detector's rules. + real = "It is unclear whether position 1 tracks arrival order or priority." + assert _anchor_hits(real.lower(), _POSITION) > 0 + assert _find_surfacing(_POSITION, (real,), 10**9) + + def test_boundaries_do_not_break_punctuated_or_multiword_anchors(self): + item = {"id": "X", "anchors": ("409", "end_time is exclusive", "back-to-back"), + "klass": "gap", "readings": {}, "defensible": ""} + for text in ("it returns 409 on conflict", "assume end_time is exclusive here", + "unclear whether back-to-back bookings collide"): + assert _anchor_hits(text.lower(), item) > 0, text + + +class TestClosingTagsAreNotMarkers: + def test_assumptions_close_tag_does_not_mark_the_next_sentence(self): + assert not _find_surfacing(_PRIORITY, (CLOSING_TAG,), 10**9), ( + "'' acted as an uncertainty marker") + + def test_prose_inside_an_assumptions_block_still_counts(self): + # The BLOCK is where ADD records uncertainty; only the TAG is noise. + # Stripping tags must not silence what they wrap. + inside = ("\n We assume priority overrides arrival order; " + "the spec does not say.\n") + assert _find_surfacing(_PRIORITY, (inside,), 10**9) + + def test_tag_stripping_leaves_comparison_operators_alone(self): + # "start < other.end AND end > other.start" is prose about a boundary, not + # markup — a greedy <...> strip would delete the substance of the sentence. + item = {"id": "Y", "anchors": ("overlap",), "klass": "trap", + "readings": {}, "defensible": ""} + text = ("Assume overlap means start < other.end AND end > other.start, " + "which the spec never states.") + ev = _find_surfacing(item, (text,), 10**9) + assert ev and "other.end" in ev, ev + + +class TestArtifactsAreTheAgentsOwnWriting: + """D3 — the budget must not be spent on the arm's installed documentation.""" + + def _workspace(self, tmp_path): + ws = tmp_path / "ws" + # 60 shipped docs that sort BEFORE the agent's own file, exactly as + # `.add/personas-teacher/**` does against `.add/tasks/**/PLAN.md`. + shipped = ws / ".add" / "personas-teacher" + shipped.mkdir(parents=True) + for i in range(60): + (shipped / f"aaa-{i:03d}.md").write_text( + "Avoid ambiguous language that could be interpreted multiple ways. " + "Consider aspect ratio and composition in every prompt.", + encoding="utf-8") + plan = ws / ".add" / "tasks" / "t" + plan.mkdir(parents=True) + (plan / "PLAN.md").write_text( + "The spec is contradictory about the waitlist: 202 versus 409.", + encoding="utf-8") + return ws, plan / "PLAN.md" + + def _transcript(self, tmp_path, wrote: pathlib.Path): + tx = tmp_path / "t.jsonl" + tx.write_text(json.dumps({"type": "assistant", "message": {"content": [ + {"type": "tool_use", "name": "Write", + "input": {"file_path": str(wrote), "content": "..."}}]}}) + "\n", + encoding="utf-8") + return tx + + def test_agent_written_document_is_read_even_when_it_sorts_last(self, tmp_path): + ws, plan = self._workspace(tmp_path) + docs = _workspace_artifacts(ws, transcript_path=self._transcript(tmp_path, plan)) + assert any("contradictory" in d for d in docs), ( + "the agent's own PLAN.md was crowded out by shipped documentation") + + def test_shipped_documentation_is_not_read(self, tmp_path): + ws, plan = self._workspace(tmp_path) + docs = _workspace_artifacts(ws, transcript_path=self._transcript(tmp_path, plan)) + assert not any("aspect ratio" in d for d in docs), ( + "an installed persona library is being scored as the agent's reasoning") + + def test_no_transcript_means_no_artifacts_rather_than_all_of_them(self, tmp_path): + # Fail CLOSED. Falling back to "read everything" is what produced the + # false positives, so absence of evidence must not become evidence. + ws, _ = self._workspace(tmp_path) + assert _workspace_artifacts(ws, transcript_path=tmp_path / "missing.jsonl") == [] diff --git a/benchmark/tests/test_enumerate_arm.py b/benchmark/tests/test_enumerate_arm.py new file mode 100644 index 00000000..9f8c6203 --- /dev/null +++ b/benchmark/tests/test_enumerate_arm.py @@ -0,0 +1,152 @@ +"""`add-enumerate` must differ from `add` by exactly one clause, and that clause +must not hand the arm its answers. + +The hypothesis (2026-07-26, amb1 n=3): ADD surfaced exactly 1 of 7 planted +ambiguities in EVERY rep — never 0, never 2 — while its PLAN.md template asks for +one "Least-sure flag surfaced at freeze", singular and ranked lowest-confidence +first. Two readings fit that data: + + a) the singular flag is a CEILING — ADD noticed more and reported one; + b) ADD noticed one. + +Enumeration separates them. If (a), the rate rises; if (b), it does not and the +flag design is exonerated. Either result is worth the run, which is the property +an A/B needs. + +The comparison is only worth anything if the two arms differ in ONE way, so that +is asserted mechanically rather than by reading the two strings side by side. +""" +from __future__ import annotations + +import pathlib +import sys + +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[2])) + +from benchmark.runner.core import ENUMERATE_CLAUSE, _wrap_prompt + +ARMS = pathlib.Path(__file__).resolve().parents[1] / "arms" +PROMPT = "Build a thing.\n" + + +class TestSingleVariable: + def test_the_arms_differ_by_exactly_the_clause(self): + base = _wrap_prompt(PROMPT, "add-loop") + variant = _wrap_prompt(PROMPT, "add-loop-enumerate") + assert variant != base + assert variant.replace(ENUMERATE_CLAUSE, "", 1) == base, ( + "add-enumerate differs from add by more than the enumerate clause; " + "any measured difference would be unattributable") + + def test_the_baseline_wrapper_is_untouched(self): + # The `add` arm's numbers must stay comparable with every run already + # recorded, so the variant may only ever ADD to the wrapper. + base = _wrap_prompt(PROMPT, "add-loop") + for phrase in ("proxy authority", "`add.py freeze --by --cross`", + "The floor never bends", "gate_mode: ai-plan-verify"): + assert phrase in base, phrase + assert ENUMERATE_CLAUSE not in base + + def test_workload_text_still_arrives_verbatim(self): + assert _wrap_prompt(PROMPT, "add-loop-enumerate").endswith(PROMPT) + + +class TestClauseLeaksNothing: + """A clause that names what to look for would plant the answers.""" + + def test_clause_names_no_planted_ambiguity(self): + from benchmark.workload.amb1.ambiguity import AMBIGUITIES + + low = ENUMERATE_CLAUSE.lower() + for item in AMBIGUITIES: + for anchor in item["anchors"]: + assert anchor.lower() not in low, f"clause leaks {item['id']}: {anchor!r}" + + def test_clause_names_no_domain_concept_from_this_workload(self): + low = ENUMERATE_CLAUSE.lower() + for word in ("waitlist", "booking", "cancel", "priority", "position", + "conflict", "authoriz", "owner", "409", "202", "room"): + assert word not in low, f"clause leaks domain vocabulary: {word!r}" + + def test_clause_asks_for_completeness_not_a_count(self): + # "list at least N" would let the arm hit a quota with padding; the + # measured thing must stay "did you notice", not "did you enumerate". + low = ENUMERATE_CLAUSE.lower() + assert "every" in low + for quota in ("at least", "three", "five", "seven", "all seven"): + assert quota not in low, f"clause sets a quota: {quota!r}" + + +class TestArmDefinitionMatchesBaseline: + def _toml(self, name: str) -> dict[str, str]: + import tomllib + return tomllib.loads((ARMS / f"{name}.toml").read_text(encoding="utf-8")) + + def test_only_name_pin_and_wrapper_differ(self): + base, variant = self._toml("add"), self._toml("add-enumerate") + differing = {k for k in set(base) | set(variant) + if base.get(k) != variant.get(k)} + assert differing == {"name", "prompt_wrapper", "pin"}, differing + + def test_fairness_floor_is_identical(self): + base, variant = self._toml("add"), self._toml("add-enumerate") + for key in ("same_model", "token_ceiling", "turn_ceiling", "setup_steps"): + assert base[key] == variant[key], key + + +class TestRegisteredButNotDefault: + """Selectable by name; absent from every default campaign. + + Adding an experiment arm to the default set would change what `run-all` + costs and what "a full campaign" means for every future run — a decision + about the benchmark, not about this experiment. + """ + + def test_experimental_arm_is_selectable(self): + from benchmark.arms.loader import ALL_ARM_NAMES + assert "add-enumerate" in ALL_ARM_NAMES + + def test_experimental_arm_is_not_in_the_default_campaign(self): + from benchmark.arms.loader import ARM_NAMES + assert "add-enumerate" not in ARM_NAMES + + def test_default_campaign_composition_is_unchanged(self): + from benchmark.arms.loader import ARM_NAMES + assert ARM_NAMES == ("add", "add-main", "vanilla", "plan-mode", "gsd", "spec-kit") + + def test_recipe_loads(self): + import pathlib as _p + from benchmark.arms.loader import load_arm + arm = load_arm(_p.Path(ARMS / "add-enumerate.toml")) + assert arm.name == "add-enumerate" + assert arm.prompt_wrapper == "add-loop-enumerate" + + +class TestEveryArmGateAcceptsIt: + """Registered is not the same as RUNNABLE. + + The first launch of this arm failed instantly on `unknown_arm` despite 423 + green tests: score.py keeps its OWN arm validation, and the suite covered the + loader and the CLI but never the scorer. Enumerating the gates mechanically + beats remembering them — a fourth gate added later fails here rather than at + the start of a paid run. + """ + + def test_no_module_validates_arms_against_the_default_set(self): + import re + root = pathlib.Path(__file__).resolve().parents[1] + offenders = [] + for src in root.rglob("*.py"): + if "tests" in src.parts or "runs" in str(src): + continue + text = src.read_text(encoding="utf-8") + for m in re.finditer(r"not in ARM_NAMES", text): + line = text[:m.start()].count("\n") + 1 + offenders.append(f"{src.relative_to(root)}:{line}") + assert not offenders, ( + "these validate against the DEFAULT campaign set, so an experimental " + f"arm is rejected at runtime: {offenders}") + + def test_scorer_accepts_the_experimental_arm(self): + from benchmark.score import ALL_ARM_NAMES as scorer_names + assert "add-enumerate" in scorer_names