diff --git a/src/cooperbench/agents/mini_swe_agent_v2/agents/default.py b/src/cooperbench/agents/mini_swe_agent_v2/agents/default.py index e9f74c2c1..c12ae45b1 100644 --- a/src/cooperbench/agents/mini_swe_agent_v2/agents/default.py +++ b/src/cooperbench/agents/mini_swe_agent_v2/agents/default.py @@ -497,6 +497,10 @@ def _announce_departed_peers(self) -> None: ) ) + def _shared_git_enabled(self) -> bool: + """True only when this run attached a remote the peer can actually fetch.""" + return bool(self.extra_template_vars.get("git_enabled")) + def _peer_work_pointer(self, peer: str) -> str: """Where to find a departed peer's work — only claimed when it is really there. @@ -504,7 +508,17 @@ def _peer_work_pointer(self, peer: str) -> str: their submission when it does not would repeat exactly the failure this whole change exists to remove: telling the agent something untrue and letting it act on it. + + With shared git off, each container has its own ``origin`` (a local shim used + to record the graded pull request). ``has_published`` on that remote does not + mean the peer can read the branch. """ + if not self._shared_git_enabled(): + return ( + f"There is no shared git remote, so {GIT_REMOTE}/{peer} is not visible to you. " + "A local pull request is only a graded submission. Do not fetch or diff their " + "branch. Use the messages you actually received." + ) if self.comm and self.comm.has_published(peer): return ( f"Their submitted patch is on branch {GIT_REMOTE}/{peer}. If your changes " diff --git a/src/cooperbench/agents/mini_swe_agent_v2/config/coop.yaml b/src/cooperbench/agents/mini_swe_agent_v2/config/coop.yaml index 51c4050bd..b8e8bbb99 100644 --- a/src/cooperbench/agents/mini_swe_agent_v2/config/coop.yaml +++ b/src/cooperbench/agents/mini_swe_agent_v2/config/coop.yaml @@ -1,6 +1,6 @@ agent: system_template: | - You are a software engineer working alongside a colleague on a shared codebase. You each have your own workspace and are implementing different features in parallel. You communicate naturally — like engineers on the same team — to make sure your combined work integrates cleanly. + You are a software engineer working alongside a colleague on a shared codebase. You each have your own workspace and are implementing different features in parallel.{% if messaging_enabled %} You communicate naturally — like engineers on the same team — to make sure your combined work integrates cleanly.{% endif %} instance_template: | ## Your Task @@ -29,11 +29,22 @@ agent: 5. Test edge cases to ensure your fix is robust 6. Submit your changes — see the **Submission** section below for the exact procedure. + {% if git_enabled %} Working as a team, open your pull request as soon as you know which files you are taking, not at the end — it is how your colleague finds out what you are doing. Keep committing and pushing to it as you go. + {% else %} + {% if messaging_enabled %} + Working as a team, tell your colleague which files you are taking with send_message. + {% endif %} + Your pull request is only the graded submission. Your colleague cannot fetch your branch. + {% endif %} + {% if messaging_enabled or git_enabled %} But you are not working solo but rather as a team, so follow the workflow above for your individual tasks but you have complete freedom to communicate with your colleague in whatever way, whenever, and however often you see fit. Think about how experienced software engineers coordinate when working on the same codebase — and do that. + {% else %} + Messaging and shared Git are disabled. Work independently; your patches will still be merged for grading. + {% endif %} {% if messaging_enabled %} ## Messaging @@ -62,7 +73,12 @@ agent: - Messages from your colleague appear as: [Message from ...]: ... If they have asked you something, answer when you can — they may be waiting on your reply before continuing. - IMPORTANT: Only the system delivers messages from your colleague. Never write `[Message from ...]` yourself. - - Your colleague may finish and exit before you do. When that happens you are told so explicitly, and any further `send_message` to them fails with a non-zero exit code instead of being silently discarded. Do not keep messaging or waiting after that — reconcile against their published branch and submit. + - Your colleague may finish and exit before you do. When that happens you are told so explicitly, and any further `send_message` to them fails with a non-zero exit code instead of being silently discarded. Do not keep messaging or waiting after that. + {% if git_enabled %} + Reconcile against their published branch and submit. + {% else %} + Their branch is not shared with you. Submit from your own work and the messages you received. Do not fetch their branch. + {% endif %} {% endif %} {% if git_enabled %} @@ -86,7 +102,7 @@ agent: git fetch origin && git diff origin/main origin/{{ agents | reject('equalto', agent_id) | first }} ``` - Check whether you would collide, and settle it over messaging if so: + Check whether you would collide{% if messaging_enabled %}, and settle it over messaging if so{% endif %}: ```bash git merge --no-commit --no-ff origin/{{ agents | reject('equalto', agent_id) | first }} && git merge --abort ``` @@ -110,7 +126,7 @@ agent: **CRITICAL REQUIREMENTS:** - Your response SHOULD include reasoning text explaining what you're doing - - Your response MUST include AT LEAST ONE bash tool call — this can be a coding command, a `send_message` to your colleague, or both + - Your response MUST include AT LEAST ONE bash tool call{% if messaging_enabled %} — this can be a coding command, a `send_message` to your colleague, or both{% endif %} - Directory or environment variable changes are not persistent. Every action is executed in a new subshell. - However, you can prefix any action with `MY_ENV_VAR=MY_VALUE cd /path/to/working/dir && ...` or write/load environment variables from files - To submit your work, follow the **Submission** section below (`echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`). @@ -175,8 +191,16 @@ agent: ## Submission - Your pull request is your submission. Open it early so your colleague knows - what you are taking. + {% if git_enabled %} + Your pull request is your submission, and your colleague can read it on the shared remote. + Open it early so they know what you are taking. + {% else %} + Your pull request is your graded submission. It does not publish a branch your colleague can read. + {% if messaging_enabled %} + Tell them what you are taking with send_message. + {% endif %} + Do not ask them to fetch your branch, and do not fetch theirs. + {% endif %} ```bash git push @@ -193,10 +217,18 @@ agent: EOF ``` + {% if git_enabled %} Both PRs are merged and both feature test suites run against the result, so your PR must contain only your own work — do not commit your colleague's changes into your branch. `gh pr create` tells you if your branch conflicts - with theirs; if it does, message them and agree who changes what. + with theirs.{% if messaging_enabled %} If it does, message them and agree who changes what.{% endif %} + {% else %} + Both submissions are merged for grading and both feature test suites run against the result, so + your pull request must contain only your own work. You cannot see your colleague's branch from here. + {% if messaging_enabled %} + If you might be editing the same code, ask them with send_message. + {% endif %} + {% endif %} End the task (EXACT command required): diff --git a/src/cooperbench/agents/mini_swe_agent_v2/connectors/gh_shim.sh b/src/cooperbench/agents/mini_swe_agent_v2/connectors/gh_shim.sh index 1f1be9d77..3fad9211a 100644 --- a/src/cooperbench/agents/mini_swe_agent_v2/connectors/gh_shim.sh +++ b/src/cooperbench/agents/mini_swe_agent_v2/connectors/gh_shim.sh @@ -29,14 +29,15 @@ unsupported() { cat >&2 < title, body and summary of their PR gh pr diff the code they are proposing gh pr checkout check out their PR locally -Review happens over send_message, not PR comments. +PR comments are not available. EOF exit 2 } @@ -106,7 +107,7 @@ $body" HEAD >/dev/null echo "WARNING: your branch conflicts with $peer's." echo "Both PRs are merged before testing, so this fails both of you." echo "See where: git --no-pager diff $BASE $REMOTE/$peer" - echo "Then message them and agree who changes what." + echo "Resolve the overlap before submitting." fi done } diff --git a/tests/agents/mini_swe_agent_v2/test_gh_shim.py b/tests/agents/mini_swe_agent_v2/test_gh_shim.py index 2a8590dae..2ecada556 100644 --- a/tests/agents/mini_swe_agent_v2/test_gh_shim.py +++ b/tests/agents/mini_swe_agent_v2/test_gh_shim.py @@ -117,6 +117,8 @@ def test_unsupported_subcommands_explain_themselves(team): assert r.returncode == 2, args assert "not available in this environment" in r.stderr assert "gh pr create" in r.stderr, "error should name what IS available" + assert "only when shared Git is enabled" in r.stderr + assert "send_message" not in r.stderr def test_create_requires_a_title(team): @@ -389,6 +391,8 @@ def test_conflicting_branch_is_reported_when_the_pr_is_opened(team): out = gh("pr create --title 'palette' --body 'y'", a2, "agent2") combined = out.stdout + out.stderr assert "conflicts with agent1" in combined, f"no conflict warning:\n{combined}" + assert "Resolve the overlap" in combined + assert "message them" not in combined def test_no_conflict_warning_when_the_branches_are_disjoint(team): diff --git a/tests/agents/mini_swe_agent_v2/test_peer_exit_agent.py b/tests/agents/mini_swe_agent_v2/test_peer_exit_agent.py index bec8f2bfa..5926ce2a2 100644 --- a/tests/agents/mini_swe_agent_v2/test_peer_exit_agent.py +++ b/tests/agents/mini_swe_agent_v2/test_peer_exit_agent.py @@ -44,7 +44,7 @@ def serialize(self): def _agent(agent_id, comm): - return DefaultAgent( + agent = DefaultAgent( _StubModel(), _StubEnv(), comm=comm, @@ -52,6 +52,9 @@ def _agent(agent_id, comm): system_template="s", instance_template="i", ) + # These recovery-path tests model a run with an actual shared remote. + agent.extra_template_vars["git_enabled"] = True + return agent @pytest.fixture diff --git a/tests/agents/mini_swe_agent_v2/test_peer_visibility.py b/tests/agents/mini_swe_agent_v2/test_peer_visibility.py new file mode 100644 index 000000000..60c109e3d --- /dev/null +++ b/tests/agents/mini_swe_agent_v2/test_peer_visibility.py @@ -0,0 +1,116 @@ +"""Shared-git-off runs must not tell an agent it can read origin/.""" + +from pathlib import Path +from unittest.mock import MagicMock + +import pytest +import yaml +from jinja2 import StrictUndefined, Template + +from cooperbench.agents.mini_swe_agent_v2.agents.default import DefaultAgent + +_COOP_YAML = ( + Path(__file__).resolve().parents[3] + / "src" + / "cooperbench" + / "agents" + / "mini_swe_agent_v2" + / "config" + / "coop.yaml" +) + + +def _render(git_enabled: bool, messaging_enabled: bool = True) -> str: + raw = yaml.safe_load(_COOP_YAML.read_text()) + templates = raw["agent"]["system_template"] + raw["agent"]["instance_template"] + return Template(templates, undefined=StrictUndefined).render( + task="Implement the feature", + agent_id="agent1", + agents=["agent1", "agent2"], + git_enabled=git_enabled, + messaging_enabled=messaging_enabled, + system="Linux", + release="test", + version="0", + machine="x86_64", + ) + + +class _Comm: + def __init__(self, published: bool): + self._published = published + + def has_published(self, peer: str) -> bool: + return self._published + + +def _agent(git_enabled: bool, published: bool) -> DefaultAgent: + agent = DefaultAgent( + model=MagicMock(), + env=MagicMock(), + comm=_Comm(published), + system_template="sys", + instance_template="task", + ) + agent.extra_template_vars["git_enabled"] = git_enabled + return agent + + +def test_prompt_hides_peer_branch_when_shared_git_is_off(): + text = _render(False) + assert "git fetch origin" not in text + assert "origin/agent2" not in text + assert "published branch" not in text + assert "cannot fetch" in text + assert "gh pr create" in text + + +def test_prompt_shows_peer_branch_when_shared_git_is_on(): + text = _render(True) + assert "git fetch origin && git diff origin/main origin/agent2" in text + assert "published branch" in text + assert "colleague can read it" in text + + +@pytest.mark.parametrize("git_enabled", [False, True]) +@pytest.mark.parametrize("messaging_enabled", [False, True]) +def test_prompt_only_recommends_available_channels(git_enabled, messaging_enabled): + text = _render(git_enabled, messaging_enabled) + assert ("send_message" in text) is messaging_enabled + assert ("## Messaging" in text) is messaging_enabled + assert ("git fetch origin" in text) is git_enabled + assert ("colleague can read it" in text) is git_enabled + assert "gh pr create" in text + if not messaging_enabled: + assert "message them" not in text + assert "settle it over messaging" not in text + assert "You communicate naturally" not in text + if not git_enabled and not messaging_enabled: + assert "Work independently" in text + + +@pytest.mark.parametrize("git_enabled", [False, True]) +def test_messaging_disabled_does_not_inject_peer_messages(git_enabled): + model = MagicMock() + agent = DefaultAgent(model=model, env=MagicMock(), comm=None, system_template="sys", instance_template="task") + agent.extra_template_vars.update(git_enabled=git_enabled, messaging_enabled=False) + agent.query = MagicMock(return_value={"role": "assistant"}) + agent.execute_actions = MagicMock(return_value=[]) + agent.step() + assert agent.messages == [] + model.format_message.assert_not_called() + + +def test_pointer_does_not_fetch_when_shared_git_is_off_even_if_local_pr_exists(): + text = _agent(False, True)._peer_work_pointer("agent2") + assert "git fetch" not in text + assert "not visible" in text + assert "graded submission" in text + + +def test_pointer_fetches_only_when_shared_git_published(): + visible = _agent(True, True)._peer_work_pointer("agent2") + assert "git fetch origin && git diff HEAD...origin/agent2" in visible + hidden = _agent(True, False)._peer_work_pointer("agent2") + assert "could NOT be published" in hidden + assert "git fetch" not in hidden