Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 17 additions & 2 deletions src/cooperbench/agents/mini_swe_agent_v2/agents/default.py
Original file line number Diff line number Diff line change
Expand Up @@ -537,7 +537,7 @@ def _handle_send_message(self, action: dict) -> dict:
if not delivered:
return self._peer_gone_result(recipient)
self.log(f"SENT (blocking) to {recipient}: {content[:80]}...")
self.sent_messages.append({"to": recipient, "content": content})
self._record_sent_message(recipient, content)
output = f"Message sent to {recipient}"
if replies:
for r in replies:
Expand All @@ -552,9 +552,24 @@ def _handle_send_message(self, action: dict) -> dict:
if not self.comm.send(recipient, content):
return self._peer_gone_result(recipient)
self.log(f"SENT to {recipient}: {content[:80]}...")
self.sent_messages.append({"to": recipient, "content": content})
self._record_sent_message(recipient, content)
return {"output": f"Message sent to {recipient}", "returncode": 0, "exception_info": ""}

def _record_sent_message(self, recipient: str, content: str) -> None:
"""Store a send that actually queued, with the enqueue timestamp when we have one.

``send_and_wait`` returns only after the peer replies, so a clock read here
would stamp the reply rather than the send. ``MessagingConnector.send`` writes
the ISO timestamp into the Redis payload at ``rpush`` and copies it to
``last_enqueued_timestamp``. Read that. If a connector has no such stamp,
omit the field; do not fill in ``time.time()``.
"""
record = {"to": recipient, "content": content}
ts = getattr(self.comm, "last_enqueued_timestamp", None)
if isinstance(ts, str) and ts:
record["timestamp"] = ts
self.sent_messages.append(record)

def _peer_gone_result(self, recipient: str) -> dict:
"""Tell the agent its peer is gone, and what to do about it.

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,9 @@ def __init__(self, agent_id: str, agents: list[str], url: str = "redis://localho
self._client = redis.from_url(url)
self._inbox_key = f"{self._prefix}{agent_id}:inbox"
self._seen_alive: set[str] = set()
# Set only after a successful rpush, to the ISO timestamp stored in that payload.
# Callers that record the send must read it before another send overwrites it.
self.last_enqueued_timestamp: str | None = None

# Clear stale messages from previous runs
self._client.delete(self._inbox_key)
Expand Down Expand Up @@ -166,6 +169,7 @@ def send(self, recipient: str, content: str) -> bool:
"timestamp": datetime.now().isoformat(),
}
self._client.rpush(f"{self._prefix}{recipient}:inbox", json.dumps(message))
self.last_enqueued_timestamp = message["timestamp"]
return True

def receive(self) -> list[dict]:
Expand Down
34 changes: 27 additions & 7 deletions src/cooperbench/runner/coop.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
"""Coop mode execution - multiple agents collaborate on separate features."""

import json
import math
import re
import threading
import uuid
Expand All @@ -21,21 +22,40 @@ def _message_timestamp_key(msg: dict) -> float:
"""Return a sortable float for a conversation message.

Timestamps in ``_extract_conversation`` output come from multiple agent
adapters: mini_swe_agent emits floats (``time.time()``), the OpenHands
SDK adapter can emit ISO strings, and some events have no timestamp at
all. Coerce to float on a best-effort basis so the sort doesn't blow up
with ``TypeError: '<' not supported between instances of 'int' and 'str'``
adapters: mini_swe_agent emits floats (``time.time()``) or ISO strings
from Redis (``datetime.now().isoformat()``), the OpenHands SDK adapter
can emit ISO strings, and some events have no timestamp at all. Coerce
to float on a best-effort basis so the sort doesn't blow up with
``TypeError: '<' not supported between instances of 'int' and 'str'``
mid-rollout — which used to crash :func:`execute_coop` before
``agent{fid}_traj.json`` was written, leaving callers with no structured
output to evaluate.

Missing, non-finite, or unparseable values stay ``0.0``. Do not invent a timestamp
for a historical record that never had one. ``Z`` is accepted on
Python 3.10, whose ``fromisoformat`` rejects that suffix.
"""
ts = msg.get("timestamp")
if ts is None:
if ts is None or isinstance(ts, bool):
return 0.0
if isinstance(ts, (int, float)):
value = float(ts)
return value if math.isfinite(value) else 0.0
Comment on lines +41 to +43
if not isinstance(ts, str):
return 0.0
text = ts.strip()
if not text:
return 0.0
try:
value = float(text)
return value if math.isfinite(value) else 0.0
except ValueError:
pass
try:
return float(ts)
except (TypeError, ValueError):
parsed = datetime.fromisoformat(text.replace("Z", "+00:00"))
except ValueError:
return 0.0
return parsed.timestamp()


def execute_coop(
Expand Down
78 changes: 78 additions & 0 deletions tests/agents/mini_swe_agent_v2/test_sent_timestamp.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
"""sent_messages must keep the enqueue timestamp, including after --wait."""

import json
from unittest.mock import MagicMock

from cooperbench.agents.mini_swe_agent_v2.agents.default import DefaultAgent
from cooperbench.agents.mini_swe_agent_v2.connectors.messaging import MessagingConnector

ENQUEUED = "2026-05-13T22:47:00"


class _Comm:
"""Stand-in that stamps at enqueue and only then pretends to wait."""

def __init__(self):
self.last_enqueued_timestamp = None

def send(self, recipient, content):
self.last_enqueued_timestamp = ENQUEUED
return True

def send_and_wait(self, recipient, content, timeout=60):
self.last_enqueued_timestamp = ENQUEUED
return True, [{"from": "agent2", "content": "ack"}]

def has_exited(self, agent_id):
return False

def has_published(self, agent_id):
return False


def _agent(comm):
return DefaultAgent(
model=MagicMock(),
env=MagicMock(),
comm=comm,
system_template="sys",
instance_template="task",
)


def test_plain_send_stores_enqueue_timestamp():
agent = _agent(_Comm())
result = agent._handle_send_message({"recipient": "agent2", "content": "ask", "wait": False})
assert result["returncode"] == 0
assert agent.sent_messages == [{"to": "agent2", "content": "ask", "timestamp": ENQUEUED}]


def test_blocking_send_stores_enqueue_timestamp_not_the_reply_time():
agent = _agent(_Comm())
result = agent._handle_send_message({"recipient": "agent2", "content": "ask", "wait": True})
assert result["returncode"] == 0
assert "ack" in result["output"]
assert agent.sent_messages == [{"to": "agent2", "content": "ask", "timestamp": ENQUEUED}]


def test_undelivered_send_is_not_recorded():
comm = _Comm()
comm.send = lambda recipient, content: False
agent = _agent(comm)
result = agent._handle_send_message({"recipient": "agent2", "content": "late", "wait": False})
assert result["returncode"] == 1
assert agent.sent_messages == []


def test_connector_send_publishes_the_same_timestamp_it_enqueues():
connector = MessagingConnector.__new__(MessagingConnector)
connector.agent_id = "agent1"
connector._prefix = ""
connector._client = MagicMock()
connector.has_exited = lambda recipient: False
connector.last_enqueued_timestamp = None

assert connector.send("agent2", "hello") is True
payload = json.loads(connector._client.rpush.call_args.args[1])
assert connector.last_enqueued_timestamp == payload["timestamp"]
assert payload["timestamp"]
74 changes: 70 additions & 4 deletions tests/runner/test_coop.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
"""Unit tests for cooperbench.runner.coop module."""

import pytest

from cooperbench.runner.coop import _extract_conversation, _message_timestamp_key


Expand Down Expand Up @@ -80,10 +82,74 @@ def test_mixed_int_float_string_timestamps_sort_cleanly(self):
sorted_msgs = self._sort_like_production(conversation)
# All five sent rows preserved (none received-flagged).
assert len(sorted_msgs) == 5
# Numeric timestamps sort in order; unparseable strings + None +
# missing all coerce to 0.0 and end up at the front (stable order).
floats = [float(m["timestamp"]) for m in sorted_msgs if isinstance(m.get("timestamp"), (int, float))]
assert floats == sorted(floats)
# None and missing stay 0 and keep their relative order. Numeric
# values sort numerically. An ISO timestamp sorts by its instant,
# not as 0.
assert [m["message"] for m in sorted_msgs] == ["d", "e", "a", "c", "b"]

def test_iso_reply_sorts_after_the_message_it_answers(self):
"""A reply recorded later must not be exported ahead of the send it answers.

The old key did ``float(ts)`` and mapped ISO strings to 0, so a
sender-grouped stable sort put the reply first.
"""
conversation = [
{
"from": "agent2",
"to": "agent1",
"message": "reply",
"timestamp": "2026-05-13T22:47:05Z",
},
{
"from": "agent1",
"to": "agent2",
"message": "ask",
"timestamp": "2026-05-13T22:47:00+00:00",
},
]
sorted_msgs = self._sort_like_production(conversation)
assert [m["message"] for m in sorted_msgs] == ["ask", "reply"]

def test_garbage_timestamp_still_sorts_as_zero(self):
conversation = [
{"from": "agent1", "to": "agent2", "message": "later", "timestamp": 2.0},
{"from": "agent2", "to": "agent1", "message": "bad", "timestamp": "not-a-time"},
]
sorted_msgs = self._sort_like_production(conversation)
assert [m["message"] for m in sorted_msgs] == ["bad", "later"]

@pytest.mark.parametrize(
"timestamp", [float("nan"), float("inf"), float("-inf"), "nan", "inf", "-inf", " NaN ", "1e999"]
)
def test_nonfinite_timestamp_falls_back_to_zero(self, timestamp):
assert _message_timestamp_key({"timestamp": timestamp}) == 0.0

def test_nonfinite_timestamps_keep_stable_missing_group_in_mixed_sort(self):
conversation = [
{"message": "later", "timestamp": "1970-01-01T00:00:03Z"},
{"message": "nan", "timestamp": float("nan")},
{"message": "earlier", "timestamp": 1.0},
{"message": "missing"},
{"message": "infinity", "timestamp": "inf"},
{"message": "middle", "timestamp": "2"},
{"message": "negative-infinity", "timestamp": float("-inf")},
]

sorted_msgs = self._sort_like_production(conversation)

assert [m["message"] for m in sorted_msgs] == [
"nan",
"missing",
"infinity",
"negative-infinity",
"earlier",
"middle",
"later",
]

def test_iso_offsets_represent_the_same_utc_instant(self):
assert _message_timestamp_key({"timestamp": "1970-01-01T08:00:01+08:00"}) == 1.0
assert _message_timestamp_key({"timestamp": "1970-01-01T00:00:01Z"}) == 1.0

def test_received_messages_excluded_before_sort(self):
"""Received entries must be filtered out before the sort, not after —
Expand Down