From 4ef0f0b1906f1e193e8ed7df0cd6dfae0979c2c2 Mon Sep 17 00:00:00 2001 From: Chojan Shang Date: Wed, 19 Aug 2026 18:15:24 +0800 Subject: [PATCH 1/4] feat(e2e): batch Harbor tasks at runtime --- e2e/bub/README.md | 17 +- .../steps/reset/instruction.md | 1 + .../locomo-multihop-football/task.toml | 5 +- .../{steps/capture => }/tests/test.sh | 6 +- .../steps/reset/instruction.md | 1 + .../locomo-open-pastries/task.toml | 5 +- .../tests/test.sh | 6 +- .../steps/capture/tests/test.sh | 18 - .../steps/recall/tests/test.sh | 18 - .../steps/reset/instruction.md | 1 + .../locomo-support-group/task.toml | 5 +- .../tests/test.sh | 6 +- .../steps/capture/tests/test.sh | 18 - .../steps/recall/tests/test.sh | 18 - .../steps/reset/instruction.md | 1 + .../locomo-temporal-banker/task.toml | 5 +- .../tests/test.sh | 6 +- e2e/bub/src/powercontext_e2e/__main__.py | 7 + e2e/bub/src/powercontext_e2e/evaluation.py | 48 +- e2e/bub/src/powercontext_e2e/harbor_agent.py | 23 + e2e/bub/src/powercontext_e2e/models.py | 2 + e2e/bub/src/powercontext_e2e/report.py | 11 + e2e/bub/src/powercontext_e2e/runner.py | 754 +++++++++++++++++- e2e/bub/tasks/locomo-multihop-football.yaml | 3 +- e2e/bub/tasks/locomo-open-pastries.yaml | 3 +- e2e/bub/tasks/locomo-support-group.yaml | 3 +- e2e/bub/tasks/locomo-temporal-banker.yaml | 3 +- e2e/bub/tests/test_runtime_batch.py | 213 +++++ e2e/bub/tests/test_workload_catalog.py | 12 + 29 files changed, 1073 insertions(+), 146 deletions(-) create mode 100644 e2e/bub/harbor-tasks/locomo-multihop-football/steps/reset/instruction.md rename e2e/bub/harbor-tasks/locomo-multihop-football/{steps/capture => }/tests/test.sh (81%) mode change 100644 => 100755 create mode 100644 e2e/bub/harbor-tasks/locomo-open-pastries/steps/reset/instruction.md rename e2e/bub/harbor-tasks/{locomo-multihop-football/steps/recall => locomo-open-pastries}/tests/test.sh (81%) mode change 100644 => 100755 delete mode 100644 e2e/bub/harbor-tasks/locomo-support-group/steps/capture/tests/test.sh delete mode 100644 e2e/bub/harbor-tasks/locomo-support-group/steps/recall/tests/test.sh create mode 100644 e2e/bub/harbor-tasks/locomo-support-group/steps/reset/instruction.md rename e2e/bub/harbor-tasks/{locomo-open-pastries/steps/capture => locomo-support-group}/tests/test.sh (81%) mode change 100644 => 100755 delete mode 100644 e2e/bub/harbor-tasks/locomo-temporal-banker/steps/capture/tests/test.sh delete mode 100644 e2e/bub/harbor-tasks/locomo-temporal-banker/steps/recall/tests/test.sh create mode 100644 e2e/bub/harbor-tasks/locomo-temporal-banker/steps/reset/instruction.md rename e2e/bub/harbor-tasks/{locomo-open-pastries/steps/recall => locomo-temporal-banker}/tests/test.sh (81%) mode change 100644 => 100755 create mode 100644 e2e/bub/tests/test_runtime_batch.py diff --git a/e2e/bub/README.md b/e2e/bub/README.md index 294c3f50f..b6d7dc444 100644 --- a/e2e/bub/README.md +++ b/e2e/bub/README.md @@ -65,6 +65,8 @@ evaluation: The dataset can be a local Harbor dataset path or a registry dataset name and version. `execution` selects the adapter and its budget. `model` declares only whether the workload requires a model. The runtime selects the model, provider, endpoint, and credentials. `evaluation` declares only externally observable Memory behavior. +Two or more compatible selected tasks with the same `batch:` category share one run-local Harbor task and +container. Their scopes, evidence, and evaluation remain independent; selecting one task uses the normal path. Runtime configuration keeps the native ownership of each component. The harness Client reads `POWERCONTEXT_CLIENT_*`, the Bub adapter forwards native `BUB_*` settings for model-backed workloads, and the @@ -76,10 +78,7 @@ The built-in manifests are: | ID | Dataset | Categories | Purpose | | --- | --- | --- | --- | -| `locomo-multihop-football` | local Harbor multi-step task | `acceptance`, `sample` | Pinned LoCoMo-derived sample (multi-hop) | -| `locomo-open-pastries` | local Harbor multi-step task | `acceptance`, `sample` | Pinned LoCoMo-derived sample (open-domain listing) | -| `locomo-support-group` | local Harbor multi-step task | `acceptance`, `sample` | Pinned LoCoMo-derived sample | -| `locomo-temporal-banker` | local Harbor multi-step task | `acceptance`, `sample` | Pinned LoCoMo-derived sample (temporal) | +| `locomo-*` (four manifests) | one local Harbor task each | `acceptance`, `sample`, `batch:locomo` | Pinned LoCoMo-derived cases | | `project-database-decision` | local Harbor multi-step task | `acceptance`, `sample`, `smoke` | Durable project decision | | `terminal-bench-db-wal-recovery` | `terminal-bench@2.0` | `long-horizon`, `terminal-bench` | Long-running capture and recall | @@ -121,12 +120,10 @@ Each selected workload writes the same layout: harbor-jobs/ ``` -`replay.json` is a self-contained Pydantic observation. Its workload's `execution.type` identifies the `bub` adapter, -and the remaining fields record the pre-execution Memory baseline and the instructions resolved by Harbor's ACP -runner. -`eval-report.json` uses -`powercontext.e2e-evaluation/v1`. `report.md` is rendered from the report model with Marko. Native Harbor and ACP -evidence remains under `harbor-jobs/`. +Shared runs write the same v1 files per source task under `batch-/tasks//`, plus one aggregate +evaluation and report at `batch-/`. `collect-all` reports every failed task; `fail-fast` stops only that shared +Harbor trial at its first failed step. Each source task starts with `steps/reset`; runtime batch steps are flat and +task-prefixed. ## Long-horizon task diff --git a/e2e/bub/harbor-tasks/locomo-multihop-football/steps/reset/instruction.md b/e2e/bub/harbor-tasks/locomo-multihop-football/steps/reset/instruction.md new file mode 100644 index 000000000..78d9bccda --- /dev/null +++ b/e2e/bub/harbor-tasks/locomo-multihop-football/steps/reset/instruction.md @@ -0,0 +1 @@ +,tape.reset diff --git a/e2e/bub/harbor-tasks/locomo-multihop-football/task.toml b/e2e/bub/harbor-tasks/locomo-multihop-football/task.toml index cae761fe3..6c94ff365 100644 --- a/e2e/bub/harbor-tasks/locomo-multihop-football/task.toml +++ b/e2e/bub/harbor-tasks/locomo-multihop-football/task.toml @@ -13,7 +13,7 @@ # limitations under the License. version = "1.3" -multi_step_reward_strategy = "final" +multi_step_reward_strategy = "mean" [metadata] sample = "locomo" @@ -27,6 +27,9 @@ timeout_sec = 60.0 [environment] build_timeout_sec = 300.0 +[[steps]] +name = "reset" + [[steps]] name = "capture" diff --git a/e2e/bub/harbor-tasks/locomo-multihop-football/steps/capture/tests/test.sh b/e2e/bub/harbor-tasks/locomo-multihop-football/tests/test.sh old mode 100644 new mode 100755 similarity index 81% rename from e2e/bub/harbor-tasks/locomo-multihop-football/steps/capture/tests/test.sh rename to e2e/bub/harbor-tasks/locomo-multihop-football/tests/test.sh index ea6268777..fa405bd81 --- a/e2e/bub/harbor-tasks/locomo-multihop-football/steps/capture/tests/test.sh +++ b/e2e/bub/harbor-tasks/locomo-multihop-football/tests/test.sh @@ -15,4 +15,8 @@ set -eu -echo 1 > /logs/verifier/reward.txt +if [ -e /logs/agent/powercontext-step-failed ]; then + echo 0 > /logs/verifier/reward.txt +else + echo 1 > /logs/verifier/reward.txt +fi diff --git a/e2e/bub/harbor-tasks/locomo-open-pastries/steps/reset/instruction.md b/e2e/bub/harbor-tasks/locomo-open-pastries/steps/reset/instruction.md new file mode 100644 index 000000000..78d9bccda --- /dev/null +++ b/e2e/bub/harbor-tasks/locomo-open-pastries/steps/reset/instruction.md @@ -0,0 +1 @@ +,tape.reset diff --git a/e2e/bub/harbor-tasks/locomo-open-pastries/task.toml b/e2e/bub/harbor-tasks/locomo-open-pastries/task.toml index cae761fe3..6c94ff365 100644 --- a/e2e/bub/harbor-tasks/locomo-open-pastries/task.toml +++ b/e2e/bub/harbor-tasks/locomo-open-pastries/task.toml @@ -13,7 +13,7 @@ # limitations under the License. version = "1.3" -multi_step_reward_strategy = "final" +multi_step_reward_strategy = "mean" [metadata] sample = "locomo" @@ -27,6 +27,9 @@ timeout_sec = 60.0 [environment] build_timeout_sec = 300.0 +[[steps]] +name = "reset" + [[steps]] name = "capture" diff --git a/e2e/bub/harbor-tasks/locomo-multihop-football/steps/recall/tests/test.sh b/e2e/bub/harbor-tasks/locomo-open-pastries/tests/test.sh old mode 100644 new mode 100755 similarity index 81% rename from e2e/bub/harbor-tasks/locomo-multihop-football/steps/recall/tests/test.sh rename to e2e/bub/harbor-tasks/locomo-open-pastries/tests/test.sh index ea6268777..fa405bd81 --- a/e2e/bub/harbor-tasks/locomo-multihop-football/steps/recall/tests/test.sh +++ b/e2e/bub/harbor-tasks/locomo-open-pastries/tests/test.sh @@ -15,4 +15,8 @@ set -eu -echo 1 > /logs/verifier/reward.txt +if [ -e /logs/agent/powercontext-step-failed ]; then + echo 0 > /logs/verifier/reward.txt +else + echo 1 > /logs/verifier/reward.txt +fi diff --git a/e2e/bub/harbor-tasks/locomo-support-group/steps/capture/tests/test.sh b/e2e/bub/harbor-tasks/locomo-support-group/steps/capture/tests/test.sh deleted file mode 100644 index ea6268777..000000000 --- a/e2e/bub/harbor-tasks/locomo-support-group/steps/capture/tests/test.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/sh -# Copyright (c) 2026 OceanBase. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -eu - -echo 1 > /logs/verifier/reward.txt diff --git a/e2e/bub/harbor-tasks/locomo-support-group/steps/recall/tests/test.sh b/e2e/bub/harbor-tasks/locomo-support-group/steps/recall/tests/test.sh deleted file mode 100644 index ea6268777..000000000 --- a/e2e/bub/harbor-tasks/locomo-support-group/steps/recall/tests/test.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/sh -# Copyright (c) 2026 OceanBase. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -eu - -echo 1 > /logs/verifier/reward.txt diff --git a/e2e/bub/harbor-tasks/locomo-support-group/steps/reset/instruction.md b/e2e/bub/harbor-tasks/locomo-support-group/steps/reset/instruction.md new file mode 100644 index 000000000..78d9bccda --- /dev/null +++ b/e2e/bub/harbor-tasks/locomo-support-group/steps/reset/instruction.md @@ -0,0 +1 @@ +,tape.reset diff --git a/e2e/bub/harbor-tasks/locomo-support-group/task.toml b/e2e/bub/harbor-tasks/locomo-support-group/task.toml index cae761fe3..6c94ff365 100644 --- a/e2e/bub/harbor-tasks/locomo-support-group/task.toml +++ b/e2e/bub/harbor-tasks/locomo-support-group/task.toml @@ -13,7 +13,7 @@ # limitations under the License. version = "1.3" -multi_step_reward_strategy = "final" +multi_step_reward_strategy = "mean" [metadata] sample = "locomo" @@ -27,6 +27,9 @@ timeout_sec = 60.0 [environment] build_timeout_sec = 300.0 +[[steps]] +name = "reset" + [[steps]] name = "capture" diff --git a/e2e/bub/harbor-tasks/locomo-open-pastries/steps/capture/tests/test.sh b/e2e/bub/harbor-tasks/locomo-support-group/tests/test.sh old mode 100644 new mode 100755 similarity index 81% rename from e2e/bub/harbor-tasks/locomo-open-pastries/steps/capture/tests/test.sh rename to e2e/bub/harbor-tasks/locomo-support-group/tests/test.sh index ea6268777..fa405bd81 --- a/e2e/bub/harbor-tasks/locomo-open-pastries/steps/capture/tests/test.sh +++ b/e2e/bub/harbor-tasks/locomo-support-group/tests/test.sh @@ -15,4 +15,8 @@ set -eu -echo 1 > /logs/verifier/reward.txt +if [ -e /logs/agent/powercontext-step-failed ]; then + echo 0 > /logs/verifier/reward.txt +else + echo 1 > /logs/verifier/reward.txt +fi diff --git a/e2e/bub/harbor-tasks/locomo-temporal-banker/steps/capture/tests/test.sh b/e2e/bub/harbor-tasks/locomo-temporal-banker/steps/capture/tests/test.sh deleted file mode 100644 index ea6268777..000000000 --- a/e2e/bub/harbor-tasks/locomo-temporal-banker/steps/capture/tests/test.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/sh -# Copyright (c) 2026 OceanBase. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -eu - -echo 1 > /logs/verifier/reward.txt diff --git a/e2e/bub/harbor-tasks/locomo-temporal-banker/steps/recall/tests/test.sh b/e2e/bub/harbor-tasks/locomo-temporal-banker/steps/recall/tests/test.sh deleted file mode 100644 index ea6268777..000000000 --- a/e2e/bub/harbor-tasks/locomo-temporal-banker/steps/recall/tests/test.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/sh -# Copyright (c) 2026 OceanBase. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -eu - -echo 1 > /logs/verifier/reward.txt diff --git a/e2e/bub/harbor-tasks/locomo-temporal-banker/steps/reset/instruction.md b/e2e/bub/harbor-tasks/locomo-temporal-banker/steps/reset/instruction.md new file mode 100644 index 000000000..78d9bccda --- /dev/null +++ b/e2e/bub/harbor-tasks/locomo-temporal-banker/steps/reset/instruction.md @@ -0,0 +1 @@ +,tape.reset diff --git a/e2e/bub/harbor-tasks/locomo-temporal-banker/task.toml b/e2e/bub/harbor-tasks/locomo-temporal-banker/task.toml index cae761fe3..6c94ff365 100644 --- a/e2e/bub/harbor-tasks/locomo-temporal-banker/task.toml +++ b/e2e/bub/harbor-tasks/locomo-temporal-banker/task.toml @@ -13,7 +13,7 @@ # limitations under the License. version = "1.3" -multi_step_reward_strategy = "final" +multi_step_reward_strategy = "mean" [metadata] sample = "locomo" @@ -27,6 +27,9 @@ timeout_sec = 60.0 [environment] build_timeout_sec = 300.0 +[[steps]] +name = "reset" + [[steps]] name = "capture" diff --git a/e2e/bub/harbor-tasks/locomo-open-pastries/steps/recall/tests/test.sh b/e2e/bub/harbor-tasks/locomo-temporal-banker/tests/test.sh old mode 100644 new mode 100755 similarity index 81% rename from e2e/bub/harbor-tasks/locomo-open-pastries/steps/recall/tests/test.sh rename to e2e/bub/harbor-tasks/locomo-temporal-banker/tests/test.sh index ea6268777..fa405bd81 --- a/e2e/bub/harbor-tasks/locomo-open-pastries/steps/recall/tests/test.sh +++ b/e2e/bub/harbor-tasks/locomo-temporal-banker/tests/test.sh @@ -15,4 +15,8 @@ set -eu -echo 1 > /logs/verifier/reward.txt +if [ -e /logs/agent/powercontext-step-failed ]; then + echo 0 > /logs/verifier/reward.txt +else + echo 1 > /logs/verifier/reward.txt +fi diff --git a/e2e/bub/src/powercontext_e2e/__main__.py b/e2e/bub/src/powercontext_e2e/__main__.py index a78ef043a..951da301b 100644 --- a/e2e/bub/src/powercontext_e2e/__main__.py +++ b/e2e/bub/src/powercontext_e2e/__main__.py @@ -49,6 +49,12 @@ def main() -> None: help="Select one category; repeat to select more than one.", ) acceptance_parser.add_argument("--output", type=Path, required=True) + acceptance_parser.add_argument( + "--failure-policy", + choices=("collect-all", "fail-fast"), + default="collect-all", + help="Continue through case failures or stop the Harbor trial at the first failed step.", + ) rescore_parser = subparsers.add_parser("rescore") rescore_parser.add_argument("replay", type=Path) @@ -71,6 +77,7 @@ def main() -> None: selected, output_dir=args.output, settings=settings, + failure_policy=args.failure_policy, ) ) raise SystemExit(0 if passed else 1) diff --git a/e2e/bub/src/powercontext_e2e/evaluation.py b/e2e/bub/src/powercontext_e2e/evaluation.py index 6ad1a71d1..ff2208459 100644 --- a/e2e/bub/src/powercontext_e2e/evaluation.py +++ b/e2e/bub/src/powercontext_e2e/evaluation.py @@ -17,8 +17,10 @@ from __future__ import annotations from pathlib import Path +from typing import Literal from .models import ( + SHARED_TRIAL_SKIPPED_ERROR, CaseEvaluation, EvaluationReport, EvaluationValue, @@ -35,6 +37,24 @@ class MemoryEvaluator: @staticmethod def evaluate(observation: TaskObservation, *, experiment: str) -> EvaluationReport: task = observation.task + attributes = _attributes(observation) + if SHARED_TRIAL_SKIPPED_ERROR in observation.errors: + return EvaluationReport( + experiment=experiment, + cases=( + CaseEvaluation( + name=task.id, + assertions={ + "execution_completed": EvaluationValue( + value=False, + reason=SHARED_TRIAL_SKIPPED_ERROR, + ) + }, + labels={"task_outcome": EvaluationValue(value="skipped")}, + attributes=attributes, + ), + ), + ) evaluation = task.evaluation eligible_records = [record for record in observation.capture_records if record.event in CAPTURE_EVENTS] captured_records = [record for record in eligible_records if record.status == "captured"] @@ -89,15 +109,6 @@ def evaluate(observation: TaskObservation, *, experiment: str) -> EvaluationRepo ) thresholds = evaluation.thresholds - attributes = { - "commit": observation.environment.commit, - "database": observation.environment.database, - "dataset": task.dataset.name or str(task.dataset.path), - "execution_adapter": task.execution.type, - "harbor_task_id": task.dataset.task_id, - "run_id": observation.run_id, - "workload_id": task.id, - } metrics = { "capture_events": len(eligible_records), "captured_sources": len(captured_records), @@ -171,7 +182,7 @@ def evaluate(observation: TaskObservation, *, experiment: str) -> EvaluationRepo for name, reward in sorted(observation.harbor.rewards.items()) }, } - labels = {"task_outcome": EvaluationValue(value=_task_outcome(observation.harbor))} + labels = {"task_outcome": EvaluationValue(value=_task_outcome(observation.harbor, observation.status))} return EvaluationReport( experiment=experiment, cases=( @@ -192,9 +203,24 @@ def _contains_fragments(value: str, expected: tuple[str, ...]) -> bool: return all(fragment.casefold() in folded for fragment in expected) -def _task_outcome(harbor: HarborTrialObservation) -> str: +def _attributes(observation: TaskObservation) -> dict[str, str]: + task = observation.task + return { + "commit": observation.environment.commit, + "database": observation.environment.database, + "dataset": task.dataset.name or str(task.dataset.path), + "execution_adapter": task.execution.type, + "harbor_task_id": task.dataset.task_id, + "run_id": observation.run_id, + "workload_id": task.id, + } + + +def _task_outcome(harbor: HarborTrialObservation, status: Literal["completed", "failed"]) -> str: if harbor.exception_type is not None: return f"error:{harbor.exception_type}" + if status == "failed": + return "not_passed" if not harbor.rewards: return "unscored" return "passed" if all(float(reward) >= 1 for reward in harbor.rewards.values()) else "not_passed" diff --git a/e2e/bub/src/powercontext_e2e/harbor_agent.py b/e2e/bub/src/powercontext_e2e/harbor_agent.py index b9bbfaa47..8ca8e4292 100644 --- a/e2e/bub/src/powercontext_e2e/harbor_agent.py +++ b/e2e/bub/src/powercontext_e2e/harbor_agent.py @@ -35,12 +35,15 @@ BUB_VERSION = version("bub") POWERCONTEXT_VERSION = version("powercontext") BUB_ACP_SERVER_VERSION = "0.0.2" +STEP_FAILURE_MARKER = "/logs/agent/powercontext-step-failed" class PowerContextBubAcpAgent(harbor_acp.AcpAgent): """Install Bub through its supported uv tool and plugin commands.""" def __init__(self, **kwargs: Any) -> None: + self._invocation_scopes = tuple(kwargs.pop("invocation_scopes", ())) + self._step_index = 0 super().__init__( registry_entry={ "id": AGENT_ID, @@ -55,6 +58,26 @@ def __init__(self, **kwargs: Any) -> None: **kwargs, ) + async def run(self, instruction: str, environment: BaseEnvironment, context: Any) -> None: + try: + await environment.exec(command=f"rm -f {STEP_FAILURE_MARKER}") + if not self._invocation_scopes: + await super().run(instruction, environment, context) + else: + if self._step_index >= len(self._invocation_scopes): + invocation = self._step_index + 1 + raise RuntimeError( # noqa: TRY003, TRY301 + f"No E2E scope configured for agent invocation {invocation}" + ) + scope_id = self._invocation_scopes[self._step_index] + with environment.scoped_exec_env({"POWERCONTEXT_BUB_SCOPE_ID": scope_id}): + await super().run(instruction, environment, context) + except Exception: + await environment.exec(command=f"touch {STEP_FAILURE_MARKER}") + raise + finally: + self._step_index += 1 + async def install(self, environment: BaseEnvironment) -> None: await self.exec_as_root( environment, diff --git a/e2e/bub/src/powercontext_e2e/models.py b/e2e/bub/src/powercontext_e2e/models.py index df0709ebe..a9f4264b7 100644 --- a/e2e/bub/src/powercontext_e2e/models.py +++ b/e2e/bub/src/powercontext_e2e/models.py @@ -23,6 +23,8 @@ from .catalog import E2ETask +SHARED_TRIAL_SKIPPED_ERROR = "Skipped after an earlier task stopped the shared Harbor trial." + class EvidenceModel(BaseModel): model_config = ConfigDict(extra="forbid") diff --git a/e2e/bub/src/powercontext_e2e/report.py b/e2e/bub/src/powercontext_e2e/report.py index 51c80c0de..b876ac454 100644 --- a/e2e/bub/src/powercontext_e2e/report.py +++ b/e2e/bub/src/powercontext_e2e/report.py @@ -57,6 +57,17 @@ def render_report(observation: TaskObservation, report: EvaluationReport) -> str return markdown.render(document) +def render_evaluation_summary(report: EvaluationReport) -> str: + markdown = Markdown(renderer=MarkdownRenderer) + document = block.Document() + document.children = [ + *_nodes(markdown, "# PowerContext end-to-end Memory evaluation"), + block.BlankLine(0), + *_nodes(markdown, f"```text\n{_evaluation_text(report)}\n```"), + ] + return markdown.render(document) + + def _nodes(markdown: Markdown, source: str) -> list[Element]: return list(markdown.parse(source).children) diff --git a/e2e/bub/src/powercontext_e2e/runner.py b/e2e/bub/src/powercontext_e2e/runner.py index 1f4675947..a414f6c28 100644 --- a/e2e/bub/src/powercontext_e2e/runner.py +++ b/e2e/bub/src/powercontext_e2e/runner.py @@ -16,31 +16,38 @@ from __future__ import annotations +import hashlib +import re +import shutil from contextlib import suppress from datetime import UTC, datetime -from pathlib import Path -from typing import Any +from pathlib import Path, PurePosixPath +from typing import Any, Literal, NamedTuple from urllib.parse import unquote, urlparse from uuid import uuid4 from harbor.job import Job from harbor.models.environment_type import EnvironmentType from harbor.models.job.config import DatasetConfig, JobConfig +from harbor.models.task.config import TaskConfig +from harbor.models.task.paths import TaskPaths +from harbor.models.task.task import Task as HarborTask from harbor.models.trial.config import AgentConfig, EnvironmentConfig, ResourceMode, ServiceVolumeConfig +from harbor.models.trial.paths import TrialPaths from powercontext.client import PowerContextClient from powercontext.client.settings import ClientSettings -from powercontext.http import ( - ListMemoryEntriesRequest, - PrepareContextRequest, -) +from powercontext.http import ListMemoryEntriesRequest, PrepareContextRequest +from pydantic import ValidationError from .artifacts import write_artifacts from .catalog import E2ETask from .evaluation import MemoryEvaluator -from .evidence import fingerprint, load_resolved_instructions, redact +from .evidence import fingerprint, load_resolved_instructions, redact, write_evaluation_report, write_evidence from .harbor_agent import BUB_ACP_SERVER_VERSION, BUB_VERSION from .models import ( + SHARED_TRIAL_SKIPPED_ERROR, CaptureRecord, + EvaluationReport, HarborTrialObservation, MemoryEntrySnapshot, MemorySnapshot, @@ -52,6 +59,7 @@ SourceReferenceSnapshot, TaskObservation, ) +from .report import render_evaluation_summary from .settings import ( HarnessSettings, ModelNotConfiguredError, @@ -60,6 +68,48 @@ powercontext_bub_environment, ) +FailurePolicy = Literal["fail-fast", "collect-all"] +BATCH_CATEGORY_PREFIX = "batch:" +BATCH_NAME_PATTERN = re.compile(r"^[a-z0-9][a-z0-9_-]*$") + + +class TaskArtifacts(NamedTuple): + capture_records: tuple[CaptureRecord, ...] + native_artifacts: tuple[NativeArtifact, ...] + resolved_instructions: tuple[ResolvedInstruction, ...] + errors: tuple[str, ...] = () + + +class ExecutionGroup(NamedTuple): + tasks: tuple[E2ETask, ...] + batch: str | None + + @property + def output_id(self) -> str: + return self.tasks[0].id if self.batch is None else f"batch-{self.batch}" + + +class SourceTask(NamedTuple): + task: E2ETask + harbor_task: HarborTask + source_steps: tuple[str, ...] + + @property + def runtime_steps(self) -> tuple[str, ...]: + return tuple(f"{self.task.id}-{step}" for step in self.source_steps) + + +class PreparedRuntime(NamedTuple): + dataset_config: DatasetConfig + sources: tuple[SourceTask, ...] + runtime_checksum: str + + +class SourceResult(NamedTuple): + status: Literal["completed", "failed", "skipped"] + errors: tuple[str, ...] + steps: tuple[Any, ...] + async def evaluate_task( task: E2ETask, @@ -73,24 +123,102 @@ async def evaluate_task( return report.accepted +async def evaluate_tasks( + tasks: tuple[E2ETask, ...], + *, + output_dir: Path, + settings: HarnessSettings, + failure_policy: FailurePolicy = "collect-all", +) -> bool: + if len(tasks) == 1: + return await evaluate_task(tasks[0], output_dir=output_dir, settings=settings) + observations = await run_task_group( + tasks, + output_dir=output_dir, + settings=settings, + failure_policy=failure_policy, + ) + experiment = f"e2e:batch:{_task_batch(tasks[0])}" + reports = tuple( + MemoryEvaluator.evaluate(observation, experiment=f"e2e:{observation.task.id}") for observation in observations + ) + for observation, report in zip(observations, reports, strict=True): + write_artifacts(observation, report, output_dir / "tasks" / observation.task.id, settings=settings) + aggregate = EvaluationReport( + experiment=experiment, cases=tuple(case for report in reports for case in report.cases) + ) + _write_batch_summary(aggregate, output_dir, settings) + return aggregate.accepted + + +def _write_batch_summary(report: EvaluationReport, output_dir: Path, settings: HarnessSettings) -> None: + output_dir.mkdir(parents=True, exist_ok=True) + write_evaluation_report(output_dir / "eval-report.json", report=report, settings=settings) + write_evidence(output_dir / "report.md", render_evaluation_summary(report), settings) + + async def run_tasks( tasks: tuple[E2ETask, ...], *, output_dir: Path, settings: HarnessSettings, + failure_policy: FailurePolicy = "collect-all", ) -> bool: model_workload_ids = tuple(task.id for task in tasks if task.execution.model) if model_workload_ids and "BUB_MODEL" not in bub_environment(): raise ModelNotConfiguredError(model_workload_ids) accepted = True - for task in tasks: - task_accepted = await evaluate_task(task, output_dir=output_dir / task.id, settings=settings) - accepted = task_accepted and accepted + for group in group_tasks(tasks): + group_accepted = await evaluate_tasks( + group.tasks, + output_dir=output_dir / group.output_id, + settings=settings, + failure_policy=failure_policy, + ) + accepted = group_accepted and accepted return accepted -async def run_task(task: E2ETask, *, output_dir: Path, settings: HarnessSettings) -> TaskObservation: +def group_tasks(tasks: tuple[E2ETask, ...]) -> tuple[ExecutionGroup, ...]: + """Group only selected tasks that share an explicit batch category.""" + + groups: list[ExecutionGroup] = [] + positions: dict[str, int] = {} + for task in tasks: + batch = _task_batch(task) + if batch is None: + groups.append(ExecutionGroup(tasks=(task,), batch=None)) + continue + if batch not in positions: + positions[batch] = len(groups) + groups.append(ExecutionGroup(tasks=(task,), batch=batch)) + continue + index = positions[batch] + group = groups[index] + groups[index] = ExecutionGroup(tasks=(*group.tasks, task), batch=group.batch) + return tuple( + ExecutionGroup(tasks=group.tasks, batch=group.batch if len(group.tasks) > 1 else None) for group in groups + ) + + +def _task_batch(task: E2ETask) -> str | None: + batches = tuple( + category.removeprefix(BATCH_CATEGORY_PREFIX) + for category in task.categories + if category.startswith(BATCH_CATEGORY_PREFIX) + ) + if len(batches) > 1 or any(BATCH_NAME_PATTERN.fullmatch(batch) is None for batch in batches): + raise ValueError(f"Task {task.id!r} must declare at most one valid batch category") # noqa: TRY003 + return batches[0] if batches else None + + +async def run_task( + task: E2ETask, + *, + output_dir: Path, + settings: HarnessSettings, +) -> TaskObservation: started_at = datetime.now(UTC) run_id = f"{task.id}-{uuid4().hex[:12]}" scope_id = f"e2e:{run_id}" @@ -98,7 +226,7 @@ async def run_task(task: E2ETask, *, output_dir: Path, settings: HarnessSettings capture_records: tuple[CaptureRecord, ...] = () native_artifacts: tuple[NativeArtifact, ...] = () resolved_instructions: tuple[ResolvedInstruction, ...] = () - harbor_observation = HarborTrialObservation() + harbor = HarborTrialObservation() memory_before = MemorySnapshot() memory_after = MemorySnapshot() probes: tuple[RecallProbeObservation, ...] = () @@ -113,31 +241,18 @@ async def run_task(task: E2ETask, *, output_dir: Path, settings: HarnessSettings try: await client.get_readiness() memory_before = await memory_snapshot(client, scope_id) - output_dir.mkdir(parents=True, exist_ok=True) job = await Job.create(_job_config(task, run_id, scope_id, output_dir, settings)) result = await job.run() - harbor_observation, trial_dir = _harbor_observation(result, settings) - if harbor_observation.exception_type is not None: - errors.append( - f"{harbor_observation.exception_type}: {harbor_observation.exception_message or ''}".strip() - ) + harbor, _, trial_dir = _harbor_observation(result, settings) + if harbor.exception_type is not None: + errors.append(f"{harbor.exception_type}: {harbor.exception_message or ''}".strip()) if trial_dir is not None: capture_records = _load_capture_records(trial_dir) native_artifacts = _native_artifacts(trial_dir, task.execution.native_artifact_names) resolved_instructions = load_resolved_instructions(trial_dir, settings) - memory_after = await memory_snapshot(client, scope_id) - probe_observations: list[RecallProbeObservation] = [] - for probe in task.evaluation.probes: - probe_observations.append( - RecallProbeObservation( - id=probe.id, - query=probe.query, - prepared_context=await prepared_context(client, scope_id, probe.query), - ) - ) - probes = tuple(probe_observations) + probes = await _prepared_probes(client, task, scope_id) except Exception as exc: errors.append(redact(f"{type(exc).__name__}: {exc}", settings)) with suppress(Exception): @@ -145,19 +260,11 @@ async def run_task(task: E2ETask, *, output_dir: Path, settings: HarnessSettings return TaskObservation( run_id=run_id, - environment=RunEnvironment( - commit=settings.commit_id(), - database=settings.database, - adapter_version=BUB_VERSION, - adapter_protocol_version=BUB_ACP_SERVER_VERSION, - agent_model=bub_environment().get("BUB_MODEL") if task.execution.model else None, - started_at=started_at, - finished_at=datetime.now(UTC), - ), + environment=_run_environment(task, started_at, settings), task=task, status="completed" if not errors else "failed", errors=tuple(errors), - harbor=harbor_observation, + harbor=harbor, capture_records=capture_records, native_artifacts=native_artifacts, resolved_instructions=resolved_instructions, @@ -167,6 +274,271 @@ async def run_task(task: E2ETask, *, output_dir: Path, settings: HarnessSettings ) +async def run_task_group( # noqa: C901 - one shared trial owns the client and evidence lifecycle + tasks: tuple[E2ETask, ...], + *, + output_dir: Path, + settings: HarnessSettings, + failure_policy: FailurePolicy = "collect-all", +) -> tuple[TaskObservation, ...]: + if len(tasks) < 2: + raise ValueError("A runtime batch requires at least two E2E tasks") # noqa: TRY003 + batch = _task_batch(tasks[0]) + run_id = f"batch-{batch}-{uuid4().hex[:12]}" + prepared = prepare_runtime_dataset( + tasks, + output_dir=output_dir, + settings=settings, + failure_policy=failure_policy, + runtime_id=run_id, + ) + started_at = datetime.now(UTC) + task_scopes = {task.id: f"e2e:{run_id}:{task.id}" for task in tasks} + invocation_scopes = tuple(task_scopes[source.task.id] for source in prepared.sources for _ in source.runtime_steps) + memory_before = {task.id: MemorySnapshot() for task in tasks} + memory_after = {task.id: MemorySnapshot() for task in tasks} + harbor = HarborTrialObservation() + step_results: tuple[Any, ...] = () + trial_dir: Path | None = None + execution_errors: list[str] = [] + provenance_error: str | None = None + client_settings = ClientSettings() + client_token = None if client_settings.api_token is None else client_settings.api_token.get_secret_value() + + async with PowerContextClient( + client_settings.server_url, + token=client_token, + timeout=client_settings.timeout, + ) as client: + try: + await client.get_readiness() + for task in tasks: + memory_before[task.id] = await memory_snapshot(client, task_scopes[task.id]) + job = await Job.create( + _batch_job_config( + tasks[0], + run_id, + invocation_scopes, + output_dir, + settings, + dataset_config=prepared.dataset_config, + ) + ) + result = await job.run() + harbor, step_results, trial_dir = _harbor_observation(result, settings) + if harbor.task_checksum != prepared.runtime_checksum: + provenance_error = ( + f"Runtime Harbor task checksum changed: expected {prepared.runtime_checksum}, " + f"observed {harbor.task_checksum}" + ) + if harbor.exception_type is not None: + execution_errors.append(f"{harbor.exception_type}: {harbor.exception_message or ''}".strip()) + except Exception as exc: + execution_errors.append(redact(f"{type(exc).__name__}: {exc}", settings)) + + source_results = _source_results( + prepared.sources, + step_results, + tuple(execution_errors), + provenance_error, + ) + artifacts_by_task = ( + collect_task_artifacts(prepared.sources, trial_dir, tasks[0].execution.native_artifact_names, settings) + if trial_dir is not None + else {task.id: TaskArtifacts((), (), (), ()) for task in tasks} + ) + environment = _run_environment(tasks[0], started_at, settings) + observations: list[TaskObservation] = [] + for source in prepared.sources: + task = source.task + scope_id = task_scopes[task.id] + source_result = source_results[task.id] + artifacts = artifacts_by_task[task.id] + errors = [*source_result.errors, *artifacts.errors] + if source_result.status != "skipped": + try: + memory_after[task.id] = await memory_snapshot(client, scope_id) + except Exception as exc: + errors.append(redact(f"{type(exc).__name__}: {exc}", settings)) + + probes: list[RecallProbeObservation] = [] + if source_result.status != "skipped": + try: + probes.extend(await _prepared_probes(client, task, scope_id)) + except Exception as exc: + errors.append(redact(f"{type(exc).__name__}: {exc}", settings)) + + status: Literal["completed", "failed"] = ( + "completed" if source_result.status == "completed" and not errors else "failed" + ) + observations.append( + TaskObservation( + run_id=run_id, + environment=environment, + task=task, + status=status, + errors=tuple(errors), + harbor=_source_harbor_observation(harbor, source, source_result, settings), + capture_records=artifacts.capture_records, + native_artifacts=artifacts.native_artifacts, + resolved_instructions=artifacts.resolved_instructions, + memory_before=memory_before[task.id], + memory_after=memory_after[task.id], + probes=tuple(probes), + ) + ) + environment = environment.model_copy(update={"finished_at": datetime.now(UTC)}) + return tuple(observation.model_copy(update={"environment": environment}) for observation in observations) + + +def _source_results( + sources: tuple[SourceTask, ...], + step_results: tuple[Any, ...], + execution_errors: tuple[str, ...], + provenance_error: str | None, +) -> dict[str, SourceResult]: + results = {source.task.id: _source_result(source, step_results) for source in sources} + + if provenance_error is not None: + return { + source.task.id: SourceResult( + "failed", + (*results[source.task.id].errors, provenance_error), + results[source.task.id].steps, + ) + for source in sources + } + if not step_results and not execution_errors: + execution_errors = ("Harbor did not execute any steps.",) + if execution_errors: + error_index = next( + ( + index + for index, source in enumerate(sources) + if results[source.task.id].status != "completed" + and all(results[previous.task.id].status == "completed" for previous in sources[:index]) + ), + max((index for index, source in enumerate(sources) if results[source.task.id].steps), default=0), + ) + source = sources[error_index] + result = results[source.task.id] + results[source.task.id] = SourceResult("failed", (*result.errors, *execution_errors), result.steps) + + stopped_index = _stopped_source_index(sources, results) + if stopped_index is None: + stopped_index = next( + (index for index, source in enumerate(sources) if results[source.task.id].status == "skipped"), + None, + ) + if stopped_index is not None: + source = sources[stopped_index] + result = results[source.task.id] + results[source.task.id] = SourceResult( + "failed", + (*result.errors, f"Harbor did not execute steps: {list(source.runtime_steps)!r}"), + (), + ) + if stopped_index is not None: + for source in sources[stopped_index + 1 :]: + result = results[source.task.id] + if result.status == "skipped" and not result.errors: + results[source.task.id] = SourceResult( + "skipped", + (SHARED_TRIAL_SKIPPED_ERROR,), + (), + ) + return results + + +def _source_result(source: SourceTask, step_results: tuple[Any, ...]) -> SourceResult: + owned_steps = tuple(step for step in step_results if step.step_name in source.runtime_steps) + executed_names = {step.step_name for step in owned_steps} + missing_steps = [step for step in source.runtime_steps if step not in executed_names] + errors = (f"Harbor did not execute steps: {missing_steps!r}",) if missing_steps and owned_steps else () + if not owned_steps: + status: Literal["completed", "failed", "skipped"] = "skipped" + elif missing_steps: + status = "failed" + else: + status = "completed" + return SourceResult(status, errors, owned_steps) + + +def _stopped_source_index( + sources: tuple[SourceTask, ...], + results: dict[str, SourceResult], +) -> int | None: + return next( + ( + index + for index, source in enumerate(sources) + if results[source.task.id].status == "failed" + or any(_step_failed(step) for step in results[source.task.id].steps) + ), + None, + ) + + +def _step_failed(step: Any) -> bool: + rewards = step.verifier_result.rewards if step.verifier_result is not None else {} + return step.exception_info is not None or any(float(value) < 1 for value in (rewards or {}).values()) + + +def _source_harbor_observation( + shared: HarborTrialObservation, + source: SourceTask, + result: SourceResult, + settings: HarnessSettings, +) -> HarborTrialObservation: + step_rewards = [ + step.verifier_result.rewards + for step in result.steps + if step.verifier_result is not None and step.verifier_result.rewards + ] + reward_names = {name for rewards in step_rewards for name in rewards} + rewards = { + name: sum(float(item.get(name, 0)) for item in step_rewards) / len(step_rewards) for name in reward_names + } + step_exception = next((step.exception_info for step in result.steps if step.exception_info is not None), None) + exception_type = None if step_exception is None else step_exception.exception_type + exception_message = None if step_exception is None else redact(step_exception.exception_message or "", settings) + if result.status != "completed" and step_exception is None and shared.exception_type is not None: + exception_type = shared.exception_type + exception_message = shared.exception_message + return HarborTrialObservation( + job_id=shared.job_id, + trial_name=shared.trial_name, + trial_uri=shared.trial_uri, + task_checksum=source.harbor_task.checksum, + rewards=rewards, + exception_type=exception_type, + exception_message=exception_message, + started_at=shared.started_at, + finished_at=shared.finished_at, + ) + + +def _batch_job_config( + task: E2ETask, + job_name: str, + invocation_scopes: tuple[str, ...], + output_dir: Path, + settings: HarnessSettings, + *, + dataset_config: DatasetConfig, +) -> JobConfig: + config = _job_config(task, job_name, invocation_scopes[0], output_dir, settings) + agent = config.agents[0] + env = dict(agent.env) + env.pop("POWERCONTEXT_BUB_SCOPE_ID") + return config.model_copy( + update={ + "agents": [agent.model_copy(update={"env": env, "kwargs": {"invocation_scopes": invocation_scopes}})], + "datasets": [dataset_config], + } + ) + + def _job_config( task: E2ETask, run_id: str, @@ -251,6 +623,180 @@ def _dataset_config(task: E2ETask, repository: Path) -> DatasetConfig: return DatasetConfig(name=dataset.name, version=dataset.version, task_names=[dataset.task_id]) +def prepare_runtime_dataset( + tasks: tuple[E2ETask, ...], + *, + output_dir: Path, + settings: HarnessSettings, + failure_policy: FailurePolicy, + runtime_id: str | None = None, +) -> PreparedRuntime: + """Assemble compatible selected source tasks into one run-local Harbor task.""" + + if len(tasks) < 2: + raise ValueError("Runtime aggregation requires at least two tasks") # noqa: TRY003 + + sources = _validate_batch_compatibility(tasks, settings) + batch = _task_batch(tasks[0]) + if batch is None: + raise ValueError("Runtime aggregation requires an explicit batch") # noqa: TRY003 + + runtime_task_id = runtime_id or f"batch-{batch}-{uuid4().hex[:12]}" + runtime_root = output_dir / "harbor-runtime-dataset" + runtime_paths = TaskPaths(runtime_root / runtime_task_id) + runtime_paths.task_dir.mkdir(parents=True) + + first_paths = sources[0].harbor_task.paths + for source_dir, target_dir in ( + (first_paths.environment_dir, runtime_paths.environment_dir), + (first_paths.tests_dir, runtime_paths.tests_dir), + ): + if source_dir.exists(): + shutil.copytree(source_dir, target_dir) + runtime_paths.steps_dir.mkdir() + for source in sources: + for source_step, runtime_step in zip(source.source_steps, source.runtime_steps, strict=True): + shutil.copytree( + source.harbor_task.paths.step_dir(source_step), + runtime_paths.step_dir(runtime_step), + ) + + runtime_config = _runtime_task_config(sources, failure_policy) + runtime_paths.config_path.write_text(runtime_config.model_dump_toml(), encoding="utf-8") + + harbor_task = HarborTask(runtime_paths.task_dir) + runtime_checksum = harbor_task.checksum + expected_steps = tuple(step for source in sources for step in source.runtime_steps) + if tuple(step.name for step in harbor_task.config.steps or ()) != expected_steps: + raise ValueError("Runtime Harbor steps do not match the selected source tasks") # noqa: TRY003 + return PreparedRuntime( + dataset_config=DatasetConfig(path=runtime_root, task_names=[runtime_task_id]), + sources=sources, + runtime_checksum=runtime_checksum, + ) + + +def _validate_batch_compatibility( + tasks: tuple[E2ETask, ...], + settings: HarnessSettings, +) -> tuple[SourceTask, ...]: + batch = _task_batch(tasks[0]) + if batch is None or any(_task_batch(task) != batch for task in tasks): + raise ValueError("Runtime aggregation requires one explicit shared batch") # noqa: TRY003 + if any(task.dataset.path is None for task in tasks): + raise ValueError(f"Batch {batch!r} requires local Harbor datasets") # noqa: TRY003 + + if len({task.id for task in tasks}) != len(tasks): + raise ValueError(f"Batch {batch!r} task IDs must be unique") # noqa: TRY003 + repository = settings.repository_path() + sources = tuple(_load_source_task(task, repository) for task in tasks) + first_profile = _runtime_profile(sources[0]) + for source in sources[1:]: + profile = _runtime_profile(source) + if incompatible := [name for name in first_profile if profile[name] != first_profile[name]]: + raise ValueError( # noqa: TRY003 + f"Source task {source.task.id!r} has incompatible batch settings: {incompatible!r}" + ) + steps = [step for source in sources for step in source.runtime_steps] + if len(steps) != len(set(steps)): + raise ValueError(f"Batch {batch!r} step names must be unique") # noqa: TRY003 + return sources + + +def _load_source_task(task: E2ETask, repository: Path) -> SourceTask: + dataset_path = task.dataset.path + if dataset_path is None: + raise ValueError(f"Source task {task.id!r} does not use a local Harbor dataset") # noqa: TRY003 + task_dir = repository / dataset_path / task.dataset.task_id + try: + harbor_task = HarborTask(task_dir) + except Exception as exc: + raise ValueError(f"Source task {task.id!r} cannot be loaded from {task_dir}") from exc # noqa: TRY003 + unexpected = {item.name for item in task_dir.iterdir()} - {"environment", "steps", "task.toml", "tests"} + if unexpected: + raise ValueError(f"Source task {task.id!r} has unsupported runtime inputs: {sorted(unexpected)!r}") # noqa: TRY003 + if harbor_task.checksum != task.dataset.checksum: + raise ValueError(f"Source task {task.id!r} checksum changed") # noqa: TRY003 + steps = _task_layout(task, harbor_task) + return SourceTask(task, harbor_task, steps) + + +def _runtime_profile(source: SourceTask) -> dict[str, Any]: + task = source.task + paths = source.harbor_task.paths + return { + "dataset": task.dataset.model_dump(mode="json", exclude={"task_id", "checksum"}), + "execution": task.execution.model_dump(mode="json"), + "capture": _capture_profile(task), + "harbor": _shared_harbor_config(source.harbor_task), + "environment": _directory_snapshot(paths.environment_dir), + "tests": _directory_snapshot(paths.tests_dir), + } + + +def _capture_profile(task: E2ETask) -> tuple[bool, int, int]: + evaluation = task.evaluation + return ( + evaluation.capture_events, + evaluation.checkpoint_every_events, + evaluation.max_event_bytes, + ) + + +def _shared_harbor_config(task: HarborTask) -> dict[str, Any]: + return task.config.model_dump(mode="json", exclude={"steps"}) + + +def _directory_snapshot(path: Path) -> tuple[tuple[str, int, str], ...]: + if not path.is_dir(): + return () + snapshot = [] + for item in sorted(path.rglob("*")): + if item.is_symlink(): + raise ValueError(f"Shared Harbor directory cannot contain symlinks: {item}") # noqa: TRY003 + if item.is_file(): + snapshot.append(( + item.relative_to(path).as_posix(), + item.stat().st_mode & 0o777, + hashlib.sha256(item.read_bytes()).hexdigest(), + )) + return tuple(snapshot) + + +def _runtime_task_config(sources: tuple[SourceTask, ...], failure_policy: FailurePolicy) -> TaskConfig: + min_reward = 1.0 if failure_policy == "fail-fast" else None + steps = [ + step.model_copy(update={"name": runtime_name, "min_reward": min_reward}) + for source in sources + for step, runtime_name in zip( + source.harbor_task.config.steps or (), + source.runtime_steps, + strict=True, + ) + ] + return sources[0].harbor_task.config.model_copy(update={"steps": steps}) + + +def _task_layout(task: E2ETask, harbor_task: HarborTask) -> tuple[str, ...]: + steps = tuple(step.name for step in harbor_task.config.steps or ()) + if not steps: + raise ValueError(f"Source task {task.id!r} has no Harbor steps") # noqa: TRY003 + if len(steps) != len(set(steps)): + raise ValueError(f"Source task {task.id!r} step names must be unique") # noqa: TRY003 + paths = tuple(PurePosixPath(step) for step in steps) + if any( + "\\" in step or path.is_absolute() or len(path.parts) != 1 or path.parts[0] == ".." or path.as_posix() != step + for step, path in zip(steps, paths, strict=True) + ): + raise ValueError(f"Source task {task.id!r} step names must be single path components") # noqa: TRY003 + step_entries = {path.name: path.is_dir() for path in harbor_task.paths.steps_dir.iterdir()} + if step_entries != dict.fromkeys(steps, True): + raise ValueError(f"Source task {task.id!r} step directories do not match task.toml") # noqa: TRY003 + if harbor_task.step_instruction(steps[0]).strip() != ",tape.reset": + raise ValueError(f"Task {task.id!r} must start with a tape.reset step") # noqa: TRY003 + return steps + + async def memory_snapshot(client: PowerContextClient, scope_id: str) -> MemorySnapshot: response = await client.list_memory_entries(ListMemoryEntriesRequest(scope_id=scope_id)) return MemorySnapshot( @@ -277,9 +823,49 @@ async def prepared_context(client: PowerContextClient, scope_id: str, query: str return PreparedContextSnapshot(status=prepared.status.value, content=prepared.content or "") -def _harbor_observation(result: Any, settings: HarnessSettings) -> tuple[HarborTrialObservation, Path | None]: +async def _prepared_probes( + client: PowerContextClient, + task: E2ETask, + scope_id: str, +) -> tuple[RecallProbeObservation, ...]: + observations = [] + for probe in task.evaluation.probes: + observations.append( + RecallProbeObservation( + id=probe.id, + query=probe.query, + prepared_context=await prepared_context(client, scope_id, probe.query), + ) + ) + return tuple(observations) + + +def _run_environment(task: E2ETask, started_at: datetime, settings: HarnessSettings) -> RunEnvironment: + return RunEnvironment( + commit=settings.commit_id(), + database=settings.database, + adapter_version=BUB_VERSION, + adapter_protocol_version=BUB_ACP_SERVER_VERSION, + agent_model=bub_environment().get("BUB_MODEL") if task.execution.model else None, + started_at=started_at, + finished_at=datetime.now(UTC), + ) + + +def _harbor_observation( + result: Any, + settings: HarnessSettings, +) -> tuple[HarborTrialObservation, tuple[Any, ...], Path | None]: if not result.trial_results: - return HarborTrialObservation(job_id=str(result.id)), None + return ( + HarborTrialObservation( + job_id=str(result.id), + exception_type="HarborJobError", + exception_message="Harbor job returned no trial results.", + ), + (), + None, + ) trial = result.trial_results[0] rewards = trial.verifier_result.rewards if trial.verifier_result is not None else {} exception = trial.exception_info @@ -295,6 +881,7 @@ def _harbor_observation(result: Any, settings: HarnessSettings) -> tuple[HarborT started_at=trial.started_at, finished_at=trial.finished_at, ), + tuple(trial.step_results or ()), _trial_dir(trial.trial_uri), ) @@ -319,3 +906,88 @@ def _native_artifacts(trial_dir: Path, names: frozenset[str]) -> tuple[NativeArt return tuple( fingerprint(path, relative_to=trial_dir) for path in sorted(trial_dir.rglob("*")) if path.name in names ) + + +def collect_task_artifacts( + sources: tuple[SourceTask, ...], + trial_dir: Path, + native_artifact_names: frozenset[str], + settings: HarnessSettings, +) -> dict[str, TaskArtifacts]: + """Collect shared-trial artifacts below each source task's Harbor steps.""" + + trial_paths = TrialPaths(trial_dir) + instructions = load_resolved_instructions(trial_dir, settings) + return { + source.task.id: _collect_source_artifacts( + source, + trial_paths, + native_artifact_names, + instructions, + settings, + ) + for source in sources + } + + +def _collect_source_artifacts( + source: SourceTask, + trial_paths: TrialPaths, + native_artifact_names: frozenset[str], + instructions: tuple[ResolvedInstruction, ...], + settings: HarnessSettings, +) -> TaskArtifacts: + step_roots = tuple(trial_paths.step_dir(step) for step in source.runtime_steps) + errors: list[str] = [] + capture_records = _source_capture_records(step_roots, trial_paths.trial_dir, settings, errors) + native_artifacts: list[NativeArtifact] = [] + for root in step_roots: + for path in sorted(root.rglob("*")): + if path.name not in native_artifact_names: + continue + try: + native_artifacts.append(fingerprint(path, relative_to=trial_paths.trial_dir)) + except OSError as exc: + errors.append( + redact( + f"Cannot fingerprint {path.relative_to(trial_paths.trial_dir)}: {type(exc).__name__}: {exc}", + settings, + ) + ) + prefixes = tuple(f"steps/{step}/" for step in source.runtime_steps) + resolved_instructions = tuple( + instruction for instruction in instructions if instruction.artifact.startswith(prefixes) + ) + return TaskArtifacts(tuple(capture_records), tuple(native_artifacts), resolved_instructions, tuple(errors)) + + +def _source_capture_records( + step_roots: tuple[Path, ...], + trial_dir: Path, + settings: HarnessSettings, + errors: list[str], +) -> list[CaptureRecord]: + records: list[CaptureRecord] = [] + for root in step_roots: + for path in sorted(root.rglob("powercontext-capture.jsonl")): + try: + lines = path.read_text(encoding="utf-8").splitlines() + except (OSError, UnicodeDecodeError) as exc: + errors.append( + redact(f"Cannot read {path.relative_to(trial_dir)}: {type(exc).__name__}: {exc}", settings) + ) + continue + for line_number, line in enumerate(lines, start=1): + if not line.strip(): + continue + try: + records.append(CaptureRecord.model_validate_json(line)) + except ValidationError as exc: + errors.append( + redact( + f"Cannot parse {path.relative_to(trial_dir)} line {line_number}: " + f"{type(exc).__name__}: {exc}", + settings, + ) + ) + return records diff --git a/e2e/bub/tasks/locomo-multihop-football.yaml b/e2e/bub/tasks/locomo-multihop-football.yaml index 5fb5f4305..862ecee0c 100644 --- a/e2e/bub/tasks/locomo-multihop-football.yaml +++ b/e2e/bub/tasks/locomo-multihop-football.yaml @@ -17,6 +17,7 @@ id: locomo-multihop-football categories: - acceptance - sample + - batch:locomo provenance: source: benchmark/locomo/dataset/locomo10.json revision: 4448275ea2c5cd0af5774d80aea7b05b5a16e1b996caf8554ca3d762a301ae84 @@ -28,7 +29,7 @@ provenance: dataset: path: e2e/bub/harbor-tasks task_id: locomo-multihop-football - checksum: 1b3697582140d2b2adb82adcd00cc33a4203da4284f55da3da124f5ee98320db + checksum: 90a2a2ea97636106f1a44c7880a005304047a9d77825a375dc4f2a8270f114a1 execution: type: bub model: false diff --git a/e2e/bub/tasks/locomo-open-pastries.yaml b/e2e/bub/tasks/locomo-open-pastries.yaml index 256864298..312b037fa 100644 --- a/e2e/bub/tasks/locomo-open-pastries.yaml +++ b/e2e/bub/tasks/locomo-open-pastries.yaml @@ -17,6 +17,7 @@ id: locomo-open-pastries categories: - acceptance - sample + - batch:locomo provenance: source: benchmark/locomo/dataset/locomo10.json revision: 4448275ea2c5cd0af5774d80aea7b05b5a16e1b996caf8554ca3d762a301ae84 @@ -28,7 +29,7 @@ provenance: dataset: path: e2e/bub/harbor-tasks task_id: locomo-open-pastries - checksum: c1c4df25435b4faba60fada92891a3423cd4826236c04daf84e7ee721e6dd094 + checksum: a33560d6baf2729987fe98928c8ae153ace91e6562df007178623542bbf3d48e execution: type: bub model: false diff --git a/e2e/bub/tasks/locomo-support-group.yaml b/e2e/bub/tasks/locomo-support-group.yaml index 8f9dd5e23..fc01ca74d 100644 --- a/e2e/bub/tasks/locomo-support-group.yaml +++ b/e2e/bub/tasks/locomo-support-group.yaml @@ -17,6 +17,7 @@ id: locomo-support-group categories: - acceptance - sample + - batch:locomo provenance: source: benchmark/locomo/dataset/locomo10.json revision: 4448275ea2c5cd0af5774d80aea7b05b5a16e1b996caf8554ca3d762a301ae84 @@ -28,7 +29,7 @@ provenance: dataset: path: e2e/bub/harbor-tasks task_id: locomo-support-group - checksum: 01fec08696a68bf4b6c70529a8d8cf864fe8e7e2a3915a263a653bfe380a5b69 + checksum: d1e9e18ae098210917073d80b46e3da00de850ec20e03ef7e1424a90e1f2fbd2 execution: type: bub model: false diff --git a/e2e/bub/tasks/locomo-temporal-banker.yaml b/e2e/bub/tasks/locomo-temporal-banker.yaml index 73afa602e..c1b66b083 100644 --- a/e2e/bub/tasks/locomo-temporal-banker.yaml +++ b/e2e/bub/tasks/locomo-temporal-banker.yaml @@ -17,6 +17,7 @@ id: locomo-temporal-banker categories: - acceptance - sample + - batch:locomo provenance: source: benchmark/locomo/dataset/locomo10.json revision: 4448275ea2c5cd0af5774d80aea7b05b5a16e1b996caf8554ca3d762a301ae84 @@ -28,7 +29,7 @@ provenance: dataset: path: e2e/bub/harbor-tasks task_id: locomo-temporal-banker - checksum: 6b7577995e466e35964a904b8c6dee83fc54e9b2818a8ee236b079126b6e0afe + checksum: 594e9e0c127f31ec37b62617c9136c1169e81e7b7f604d418e6c4df230549ccd execution: type: bub model: false diff --git a/e2e/bub/tests/test_runtime_batch.py b/e2e/bub/tests/test_runtime_batch.py new file mode 100644 index 000000000..b91aa8f62 --- /dev/null +++ b/e2e/bub/tests/test_runtime_batch.py @@ -0,0 +1,213 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import json +from datetime import UTC, datetime +from pathlib import Path + +from harbor.models.task.task import Task as HarborTask +from harbor.models.trial.result import ExceptionInfo, StepResult +from harbor.models.verifier.result import VerifierResult + +from powercontext_e2e.catalog import E2ETask, load_tasks +from powercontext_e2e.evaluation import MemoryEvaluator +from powercontext_e2e.models import ( + SHARED_TRIAL_SKIPPED_ERROR, + HarborTrialObservation, + MemorySnapshot, + RunEnvironment, + TaskObservation, +) +from powercontext_e2e.runner import _source_results, collect_task_artifacts, prepare_runtime_dataset +from powercontext_e2e.settings import HarnessSettings + + +def _repository() -> Path: + return Path(__file__).resolve().parents[3] + + +def _locomo_tasks() -> tuple[E2ETask, ...]: + return tuple( + task for task in load_tasks(_repository() / "e2e" / "bub" / "tasks") if "batch:locomo" in task.categories + ) + + +def test_selected_locomo_tasks_share_one_runtime_with_requested_failure_policy(tmp_path: Path) -> None: + tasks = _locomo_tasks() + settings = HarnessSettings(repository=_repository()) + + collect_all = prepare_runtime_dataset( + tasks, + output_dir=tmp_path, + settings=settings, + failure_policy="collect-all", + ) + runtime_task_id = collect_all.dataset_config.task_names[0] + runtime_task = HarborTask(collect_all.dataset_config.path / runtime_task_id) + runtime_step_names = tuple(f"{task.id}-{step}" for task in tasks for step in ("reset", "capture", "recall")) + + assert len(collect_all.dataset_config.task_names) == 1 + assert tuple(step.name for step in runtime_task.config.steps or ()) == runtime_step_names + assert all(step.min_reward is None for step in runtime_task.config.steps or ()) + + fail_fast = prepare_runtime_dataset( + tasks, + output_dir=tmp_path, + settings=settings, + failure_policy="fail-fast", + ) + fail_fast_task_id = fail_fast.dataset_config.task_names[0] + fail_fast_task = HarborTask(fail_fast.dataset_config.path / fail_fast_task_id) + assert fail_fast_task_id != runtime_task_id + assert all(step.min_reward == 1.0 for step in fail_fast_task.config.steps or ()) + + +def test_shared_trial_artifacts_stay_with_their_source_task(tmp_path: Path) -> None: + tasks = _locomo_tasks()[:2] + prepared = prepare_runtime_dataset( + tasks, + output_dir=tmp_path / "runtime", + settings=HarnessSettings(repository=_repository()), + failure_policy="collect-all", + ) + trial_dir = tmp_path / "trial" + for source in prepared.sources: + agent_dir = trial_dir / "steps" / f"{source.task.id}-recall" / "agent" + agent_dir.mkdir(parents=True) + (agent_dir / "powercontext-capture.jsonl").write_text( + json.dumps({ + "schema": "powercontext.bub-capture-event/v1", + "recorded_at": "2026-08-19T00:00:00Z", + "event": "user_prompt", + "status": "captured", + "source_id": f"bub-event:{source.task.id}", + }) + + "\n", + encoding="utf-8", + ) + malformed_path = ( + trial_dir / "steps" / f"{prepared.sources[0].task.id}-capture" / "agent" / "powercontext-capture.jsonl" + ) + malformed_path.parent.mkdir(parents=True) + malformed_path.write_text("{truncated\n", encoding="utf-8") + + artifacts = collect_task_artifacts( + prepared.sources, + trial_dir, + tasks[0].execution.native_artifact_names, + HarnessSettings(repository=_repository()), + ) + + assert {task.id: [record.source_id for record in artifacts[task.id].capture_records] for task in tasks} == { + task.id: [f"bub-event:{task.id}"] for task in tasks + } + assert len(artifacts[tasks[0].id].errors) == 1 + assert artifacts[tasks[1].id].errors == () + + +def test_collect_all_keeps_later_tasks_after_an_agent_failure(tmp_path: Path) -> None: + tasks = _locomo_tasks()[:2] + prepared = prepare_runtime_dataset( + tasks, + output_dir=tmp_path, + settings=HarnessSettings(repository=_repository()), + failure_policy="collect-all", + ) + failed_step = prepared.sources[0].runtime_steps[1] + step_results = tuple( + StepResult( + step_name=step, + verifier_result=VerifierResult(rewards={"reward": 0 if step == failed_step else 1}), + exception_info=( + ExceptionInfo( + exception_type="AgentError", + exception_message="agent failed", + exception_traceback="", + occurred_at=datetime(2026, 8, 24, tzinfo=UTC), + ) + if step == failed_step + else None + ), + ) + for source in prepared.sources + for step in source.runtime_steps + ) + + results = _source_results(prepared.sources, step_results, (), None) + + assert results[tasks[0].id].status == "completed" + assert results[tasks[0].id].errors == () + assert results[tasks[1].id].status == "completed" + assert len(results[tasks[1].id].steps) == 3 + + +def test_fail_fast_marks_unexecuted_tasks_as_skipped(tmp_path: Path) -> None: + tasks = _locomo_tasks()[:2] + prepared = prepare_runtime_dataset( + tasks, + output_dir=tmp_path, + settings=HarnessSettings(repository=_repository()), + failure_policy="fail-fast", + ) + failed_step = prepared.sources[0].runtime_steps[0] + step_results = ( + StepResult( + step_name=failed_step, + verifier_result=VerifierResult(rewards={"reward": 0}), + exception_info=ExceptionInfo( + exception_type="AgentError", + exception_message="agent failed", + exception_traceback="", + occurred_at=datetime(2026, 8, 24, tzinfo=UTC), + ), + ), + ) + + results = _source_results(prepared.sources, step_results, (), None) + + assert results[tasks[0].id].status == "failed" + assert "did not execute steps" in results[tasks[0].id].errors[0] + assert results[tasks[1].id].status == "skipped" + assert results[tasks[1].id].errors == (SHARED_TRIAL_SKIPPED_ERROR,) + + +def test_skipped_shared_task_has_only_an_execution_result() -> None: + task = _locomo_tasks()[0] + recorded_at = datetime(2026, 8, 24, tzinfo=UTC) + observation = TaskObservation( + run_id="batch-locomo-test", + environment=RunEnvironment( + commit="abcdef0", + database="sqlite", + adapter_version="test-adapter", + adapter_protocol_version="test-protocol", + started_at=recorded_at, + finished_at=recorded_at, + ), + task=task, + status="failed", + errors=(SHARED_TRIAL_SKIPPED_ERROR,), + harbor=HarborTrialObservation(task_checksum=task.dataset.checksum), + memory_before=MemorySnapshot(), + memory_after=MemorySnapshot(), + ) + + report = MemoryEvaluator.evaluate(observation, experiment="e2e:batch:locomo") + case = report.cases[0] + + assert set(case.assertions) == {"execution_completed"} + assert case.assertions["execution_completed"].value is False + assert case.labels["task_outcome"].value == "skipped" diff --git a/e2e/bub/tests/test_workload_catalog.py b/e2e/bub/tests/test_workload_catalog.py index a355294e5..8ab9dcd77 100644 --- a/e2e/bub/tests/test_workload_catalog.py +++ b/e2e/bub/tests/test_workload_catalog.py @@ -16,7 +16,10 @@ from pathlib import Path +import pytest + from powercontext_e2e.catalog import load_tasks, select_tasks +from powercontext_e2e.runner import group_tasks def test_workloads_can_be_selected_by_multiple_ids_or_category() -> None: @@ -51,3 +54,12 @@ def test_workloads_can_be_selected_by_multiple_ids_or_category() -> None: "locomo-temporal-banker", "project-database-decision", ] + + +def test_batch_category_cannot_escape_the_runtime_dataset() -> None: + repository = Path(__file__).resolve().parents[3] + task = load_tasks(repository / "e2e" / "bub" / "tasks" / "locomo-support-group.yaml")[0] + task = task.model_copy(update={"categories": (*task.categories[:-1], "batch:x/../../evil")}) + + with pytest.raises(ValueError, match="valid batch category"): + group_tasks((task,)) From 25d1ac94adefee31aa8b377c7a95e2394b19bbc7 Mon Sep 17 00:00:00 2001 From: Chojan Shang Date: Mon, 24 Aug 2026 11:28:55 +0800 Subject: [PATCH 2/4] refactor(e2e): simplify Harbor batch execution --- .../steps/reset/instruction.md | 1 - .../locomo-multihop-football/task.toml | 3 - .../steps/reset/instruction.md | 1 - .../locomo-open-pastries/task.toml | 3 - .../steps/reset/instruction.md | 1 - .../locomo-support-group/task.toml | 3 - .../steps/reset/instruction.md | 1 - .../locomo-temporal-banker/task.toml | 3 - e2e/bub/src/powercontext_e2e/runner.py | 526 ++++++------------ e2e/bub/tasks/locomo-multihop-football.yaml | 2 +- e2e/bub/tasks/locomo-open-pastries.yaml | 2 +- e2e/bub/tasks/locomo-support-group.yaml | 2 +- e2e/bub/tasks/locomo-temporal-banker.yaml | 2 +- e2e/bub/tests/test_runtime_batch.py | 191 ++----- 14 files changed, 247 insertions(+), 494 deletions(-) delete mode 100644 e2e/bub/harbor-tasks/locomo-multihop-football/steps/reset/instruction.md delete mode 100644 e2e/bub/harbor-tasks/locomo-open-pastries/steps/reset/instruction.md delete mode 100644 e2e/bub/harbor-tasks/locomo-support-group/steps/reset/instruction.md delete mode 100644 e2e/bub/harbor-tasks/locomo-temporal-banker/steps/reset/instruction.md diff --git a/e2e/bub/harbor-tasks/locomo-multihop-football/steps/reset/instruction.md b/e2e/bub/harbor-tasks/locomo-multihop-football/steps/reset/instruction.md deleted file mode 100644 index 78d9bccda..000000000 --- a/e2e/bub/harbor-tasks/locomo-multihop-football/steps/reset/instruction.md +++ /dev/null @@ -1 +0,0 @@ -,tape.reset diff --git a/e2e/bub/harbor-tasks/locomo-multihop-football/task.toml b/e2e/bub/harbor-tasks/locomo-multihop-football/task.toml index 6c94ff365..6d29fd658 100644 --- a/e2e/bub/harbor-tasks/locomo-multihop-football/task.toml +++ b/e2e/bub/harbor-tasks/locomo-multihop-football/task.toml @@ -27,9 +27,6 @@ timeout_sec = 60.0 [environment] build_timeout_sec = 300.0 -[[steps]] -name = "reset" - [[steps]] name = "capture" diff --git a/e2e/bub/harbor-tasks/locomo-open-pastries/steps/reset/instruction.md b/e2e/bub/harbor-tasks/locomo-open-pastries/steps/reset/instruction.md deleted file mode 100644 index 78d9bccda..000000000 --- a/e2e/bub/harbor-tasks/locomo-open-pastries/steps/reset/instruction.md +++ /dev/null @@ -1 +0,0 @@ -,tape.reset diff --git a/e2e/bub/harbor-tasks/locomo-open-pastries/task.toml b/e2e/bub/harbor-tasks/locomo-open-pastries/task.toml index 6c94ff365..6d29fd658 100644 --- a/e2e/bub/harbor-tasks/locomo-open-pastries/task.toml +++ b/e2e/bub/harbor-tasks/locomo-open-pastries/task.toml @@ -27,9 +27,6 @@ timeout_sec = 60.0 [environment] build_timeout_sec = 300.0 -[[steps]] -name = "reset" - [[steps]] name = "capture" diff --git a/e2e/bub/harbor-tasks/locomo-support-group/steps/reset/instruction.md b/e2e/bub/harbor-tasks/locomo-support-group/steps/reset/instruction.md deleted file mode 100644 index 78d9bccda..000000000 --- a/e2e/bub/harbor-tasks/locomo-support-group/steps/reset/instruction.md +++ /dev/null @@ -1 +0,0 @@ -,tape.reset diff --git a/e2e/bub/harbor-tasks/locomo-support-group/task.toml b/e2e/bub/harbor-tasks/locomo-support-group/task.toml index 6c94ff365..6d29fd658 100644 --- a/e2e/bub/harbor-tasks/locomo-support-group/task.toml +++ b/e2e/bub/harbor-tasks/locomo-support-group/task.toml @@ -27,9 +27,6 @@ timeout_sec = 60.0 [environment] build_timeout_sec = 300.0 -[[steps]] -name = "reset" - [[steps]] name = "capture" diff --git a/e2e/bub/harbor-tasks/locomo-temporal-banker/steps/reset/instruction.md b/e2e/bub/harbor-tasks/locomo-temporal-banker/steps/reset/instruction.md deleted file mode 100644 index 78d9bccda..000000000 --- a/e2e/bub/harbor-tasks/locomo-temporal-banker/steps/reset/instruction.md +++ /dev/null @@ -1 +0,0 @@ -,tape.reset diff --git a/e2e/bub/harbor-tasks/locomo-temporal-banker/task.toml b/e2e/bub/harbor-tasks/locomo-temporal-banker/task.toml index 6c94ff365..6d29fd658 100644 --- a/e2e/bub/harbor-tasks/locomo-temporal-banker/task.toml +++ b/e2e/bub/harbor-tasks/locomo-temporal-banker/task.toml @@ -27,9 +27,6 @@ timeout_sec = 60.0 [environment] build_timeout_sec = 300.0 -[[steps]] -name = "reset" - [[steps]] name = "capture" diff --git a/e2e/bub/src/powercontext_e2e/runner.py b/e2e/bub/src/powercontext_e2e/runner.py index a414f6c28..dcfd07dad 100644 --- a/e2e/bub/src/powercontext_e2e/runner.py +++ b/e2e/bub/src/powercontext_e2e/runner.py @@ -16,28 +16,35 @@ from __future__ import annotations -import hashlib import re import shutil -from contextlib import suppress from datetime import UTC, datetime from pathlib import Path, PurePosixPath from typing import Any, Literal, NamedTuple from urllib.parse import unquote, urlparse from uuid import uuid4 +from dirhash import dirhash from harbor.job import Job from harbor.models.environment_type import EnvironmentType from harbor.models.job.config import DatasetConfig, JobConfig -from harbor.models.task.config import TaskConfig +from harbor.models.task.config import MultiStepRewardStrategy, TaskConfig from harbor.models.task.paths import TaskPaths from harbor.models.task.task import Task as HarborTask -from harbor.models.trial.config import AgentConfig, EnvironmentConfig, ResourceMode, ServiceVolumeConfig +from harbor.models.trial.config import ( + AgentConfig, + EnvironmentConfig, + ResourceMode, + ServiceVolumeConfig, +) +from harbor.models.trial.config import ( + TaskConfig as HarborTrialTaskConfig, +) from harbor.models.trial.paths import TrialPaths +from harbor.models.trial.result import StepResult from powercontext.client import PowerContextClient from powercontext.client.settings import ClientSettings from powercontext.http import ListMemoryEntriesRequest, PrepareContextRequest -from pydantic import ValidationError from .artifacts import write_artifacts from .catalog import E2ETask @@ -74,10 +81,9 @@ class TaskArtifacts(NamedTuple): - capture_records: tuple[CaptureRecord, ...] - native_artifacts: tuple[NativeArtifact, ...] - resolved_instructions: tuple[ResolvedInstruction, ...] - errors: tuple[str, ...] = () + capture_records: tuple[CaptureRecord, ...] = () + native_artifacts: tuple[NativeArtifact, ...] = () + resolved_instructions: tuple[ResolvedInstruction, ...] = () class ExecutionGroup(NamedTuple): @@ -100,15 +106,22 @@ def runtime_steps(self) -> tuple[str, ...]: class PreparedRuntime(NamedTuple): - dataset_config: DatasetConfig + task_config: HarborTrialTaskConfig sources: tuple[SourceTask, ...] - runtime_checksum: str class SourceResult(NamedTuple): status: Literal["completed", "failed", "skipped"] errors: tuple[str, ...] - steps: tuple[Any, ...] + steps: tuple[StepResult, ...] + + +class PreparedTask(NamedTuple): + task: E2ETask + run_id: str + scope_id: str + started_at: datetime + memory_before: MemorySnapshot async def evaluate_task( @@ -213,68 +226,46 @@ def _task_batch(task: E2ETask) -> str | None: return batches[0] if batches else None +def _powercontext_client() -> PowerContextClient: + settings = ClientSettings() + token = None if settings.api_token is None else settings.api_token.get_secret_value() + return PowerContextClient(settings.server_url, token=token, timeout=settings.timeout) + + async def run_task( task: E2ETask, *, output_dir: Path, settings: HarnessSettings, ) -> TaskObservation: - started_at = datetime.now(UTC) run_id = f"{task.id}-{uuid4().hex[:12]}" scope_id = f"e2e:{run_id}" + prepared = PreparedTask(task, run_id, scope_id, datetime.now(UTC), MemorySnapshot()) errors: list[str] = [] - capture_records: tuple[CaptureRecord, ...] = () - native_artifacts: tuple[NativeArtifact, ...] = () - resolved_instructions: tuple[ResolvedInstruction, ...] = () + artifacts = TaskArtifacts() harbor = HarborTrialObservation() - memory_before = MemorySnapshot() - memory_after = MemorySnapshot() - probes: tuple[RecallProbeObservation, ...] = () - client_settings = ClientSettings() - client_token = None if client_settings.api_token is None else client_settings.api_token.get_secret_value() - - async with PowerContextClient( - client_settings.server_url, - token=client_token, - timeout=client_settings.timeout, - ) as client: + execution_status: Literal["completed", "failed"] = "failed" + + async with _powercontext_client() as client: try: await client.get_readiness() - memory_before = await memory_snapshot(client, scope_id) + prepared = prepared._replace(memory_before=await memory_snapshot(client, scope_id)) output_dir.mkdir(parents=True, exist_ok=True) job = await Job.create(_job_config(task, run_id, scope_id, output_dir, settings)) result = await job.run() harbor, _, trial_dir = _harbor_observation(result, settings) if harbor.exception_type is not None: errors.append(f"{harbor.exception_type}: {harbor.exception_message or ''}".strip()) + else: + execution_status = "completed" if trial_dir is not None: - capture_records = _load_capture_records(trial_dir) - native_artifacts = _native_artifacts(trial_dir, task.execution.native_artifact_names) - resolved_instructions = load_resolved_instructions(trial_dir, settings) - memory_after = await memory_snapshot(client, scope_id) - probes = await _prepared_probes(client, task, scope_id) + artifacts = _load_task_artifacts(trial_dir, task.execution.native_artifact_names, settings) except Exception as exc: errors.append(redact(f"{type(exc).__name__}: {exc}", settings)) - with suppress(Exception): - memory_after = await memory_snapshot(client, scope_id) - - return TaskObservation( - run_id=run_id, - environment=_run_environment(task, started_at, settings), - task=task, - status="completed" if not errors else "failed", - errors=tuple(errors), - harbor=harbor, - capture_records=capture_records, - native_artifacts=native_artifacts, - resolved_instructions=resolved_instructions, - memory_before=memory_before, - memory_after=memory_after, - probes=probes, - ) + return await _finalize_task(client, prepared, execution_status, tuple(errors), harbor, artifacts, settings) -async def run_task_group( # noqa: C901 - one shared trial owns the client and evidence lifecycle +async def run_task_group( tasks: tuple[E2ETask, ...], *, output_dir: Path, @@ -285,35 +276,31 @@ async def run_task_group( # noqa: C901 - one shared trial owns the client and e raise ValueError("A runtime batch requires at least two E2E tasks") # noqa: TRY003 batch = _task_batch(tasks[0]) run_id = f"batch-{batch}-{uuid4().hex[:12]}" - prepared = prepare_runtime_dataset( + prepared = prepare_runtime_task( tasks, output_dir=output_dir, settings=settings, failure_policy=failure_policy, runtime_id=run_id, ) - started_at = datetime.now(UTC) task_scopes = {task.id: f"e2e:{run_id}:{task.id}" for task in tasks} + prepared_tasks = { + task.id: PreparedTask(task, run_id, task_scopes[task.id], datetime.now(UTC), MemorySnapshot()) for task in tasks + } invocation_scopes = tuple(task_scopes[source.task.id] for source in prepared.sources for _ in source.runtime_steps) - memory_before = {task.id: MemorySnapshot() for task in tasks} - memory_after = {task.id: MemorySnapshot() for task in tasks} harbor = HarborTrialObservation() step_results: tuple[Any, ...] = () trial_dir: Path | None = None execution_errors: list[str] = [] - provenance_error: str | None = None - client_settings = ClientSettings() - client_token = None if client_settings.api_token is None else client_settings.api_token.get_secret_value() - - async with PowerContextClient( - client_settings.server_url, - token=client_token, - timeout=client_settings.timeout, - ) as client: + + async with _powercontext_client() as client: try: await client.get_readiness() for task in tasks: - memory_before[task.id] = await memory_snapshot(client, task_scopes[task.id]) + prepared_task = prepared_tasks[task.id] + prepared_tasks[task.id] = prepared_task._replace( + memory_before=await memory_snapshot(client, prepared_task.scope_id) + ) job = await Job.create( _batch_job_config( tasks[0], @@ -321,188 +308,129 @@ async def run_task_group( # noqa: C901 - one shared trial owns the client and e invocation_scopes, output_dir, settings, - dataset_config=prepared.dataset_config, + task_config=prepared.task_config, ) ) result = await job.run() harbor, step_results, trial_dir = _harbor_observation(result, settings) - if harbor.task_checksum != prepared.runtime_checksum: - provenance_error = ( - f"Runtime Harbor task checksum changed: expected {prepared.runtime_checksum}, " - f"observed {harbor.task_checksum}" - ) if harbor.exception_type is not None: execution_errors.append(f"{harbor.exception_type}: {harbor.exception_message or ''}".strip()) except Exception as exc: execution_errors.append(redact(f"{type(exc).__name__}: {exc}", settings)) - source_results = _source_results( - prepared.sources, - step_results, - tuple(execution_errors), - provenance_error, - ) - artifacts_by_task = ( - collect_task_artifacts(prepared.sources, trial_dir, tasks[0].execution.native_artifact_names, settings) - if trial_dir is not None - else {task.id: TaskArtifacts((), (), (), ()) for task in tasks} - ) - environment = _run_environment(tasks[0], started_at, settings) + source_results = _source_results(prepared.sources, step_results, tuple(execution_errors)) observations: list[TaskObservation] = [] for source in prepared.sources: task = source.task - scope_id = task_scopes[task.id] source_result = source_results[task.id] - artifacts = artifacts_by_task[task.id] - errors = [*source_result.errors, *artifacts.errors] - if source_result.status != "skipped": + artifacts = TaskArtifacts() + if trial_dir is not None and source_result.status != "skipped": try: - memory_after[task.id] = await memory_snapshot(client, scope_id) - except Exception as exc: - errors.append(redact(f"{type(exc).__name__}: {exc}", settings)) - - probes: list[RecallProbeObservation] = [] - if source_result.status != "skipped": - try: - probes.extend(await _prepared_probes(client, task, scope_id)) + artifacts = _load_task_artifacts( + trial_dir, + task.execution.native_artifact_names, + settings, + step_names=source.runtime_steps, + ) except Exception as exc: - errors.append(redact(f"{type(exc).__name__}: {exc}", settings)) - - status: Literal["completed", "failed"] = ( - "completed" if source_result.status == "completed" and not errors else "failed" - ) + source_result = source_result._replace( + status="failed", + errors=(*source_result.errors, redact(f"{type(exc).__name__}: {exc}", settings)), + ) + source_harbor = _source_harbor_observation(harbor, source, source_result, settings) observations.append( - TaskObservation( - run_id=run_id, - environment=environment, - task=task, - status=status, - errors=tuple(errors), - harbor=_source_harbor_observation(harbor, source, source_result, settings), - capture_records=artifacts.capture_records, - native_artifacts=artifacts.native_artifacts, - resolved_instructions=artifacts.resolved_instructions, - memory_before=memory_before[task.id], - memory_after=memory_after[task.id], - probes=tuple(probes), + await _finalize_task( + client, + prepared_tasks[task.id], + source_result.status, + source_result.errors, + source_harbor, + artifacts, + settings, ) ) - environment = environment.model_copy(update={"finished_at": datetime.now(UTC)}) - return tuple(observation.model_copy(update={"environment": environment}) for observation in observations) + return tuple(observations) + + +async def _finalize_task( + client: PowerContextClient, + prepared: PreparedTask, + execution_status: Literal["completed", "failed", "skipped"], + errors: tuple[str, ...], + harbor: HarborTrialObservation, + artifacts: TaskArtifacts, + settings: HarnessSettings, +) -> TaskObservation: + final_errors = list(errors) + memory_after = MemorySnapshot() + probes: tuple[RecallProbeObservation, ...] = () + if execution_status != "skipped": + try: + memory_after = await memory_snapshot(client, prepared.scope_id) + except Exception as exc: + final_errors.append(redact(f"{type(exc).__name__}: {exc}", settings)) + try: + probes = await _prepared_probes(client, prepared.task, prepared.scope_id) + except Exception as exc: + final_errors.append(redact(f"{type(exc).__name__}: {exc}", settings)) + + return TaskObservation( + run_id=prepared.run_id, + environment=_run_environment(prepared.task, prepared.started_at, settings), + task=prepared.task, + status="completed" if execution_status == "completed" and not final_errors else "failed", + errors=tuple(final_errors), + harbor=harbor, + capture_records=artifacts.capture_records, + native_artifacts=artifacts.native_artifacts, + resolved_instructions=artifacts.resolved_instructions, + memory_before=prepared.memory_before, + memory_after=memory_after, + probes=probes, + ) def _source_results( sources: tuple[SourceTask, ...], - step_results: tuple[Any, ...], + step_results: tuple[StepResult, ...], execution_errors: tuple[str, ...], - provenance_error: str | None, ) -> dict[str, SourceResult]: - results = {source.task.id: _source_result(source, step_results) for source in sources} - - if provenance_error is not None: - return { - source.task.id: SourceResult( - "failed", - (*results[source.task.id].errors, provenance_error), - results[source.task.id].steps, - ) - for source in sources - } - if not step_results and not execution_errors: - execution_errors = ("Harbor did not execute any steps.",) + results: dict[str, SourceResult] = {} + for index, source in enumerate(sources): + owned = tuple(step for step in step_results if step.step_name in source.runtime_steps) + executed = {step.step_name for step in owned} + missing = [name for name in source.runtime_steps if name not in executed] + if not owned: + status: Literal["completed", "failed", "skipped"] = "failed" if index == 0 else "skipped" + errors = ("Harbor did not execute any steps.",) if index == 0 else (SHARED_TRIAL_SKIPPED_ERROR,) + elif missing or any(step.exception_info is not None for step in owned): + status = "failed" + errors = (f"Harbor did not execute steps: {missing!r}",) if missing else () + else: + status = "completed" + errors = () + results[source.task.id] = SourceResult(status, errors, owned) + if execution_errors: - error_index = next( - ( - index - for index, source in enumerate(sources) - if results[source.task.id].status != "completed" - and all(results[previous.task.id].status == "completed" for previous in sources[:index]) - ), - max((index for index, source in enumerate(sources) if results[source.task.id].steps), default=0), + source = next( + (source for source in reversed(sources) if results[source.task.id].status != "skipped"), + sources[0], ) - source = sources[error_index] result = results[source.task.id] results[source.task.id] = SourceResult("failed", (*result.errors, *execution_errors), result.steps) - - stopped_index = _stopped_source_index(sources, results) - if stopped_index is None: - stopped_index = next( - (index for index, source in enumerate(sources) if results[source.task.id].status == "skipped"), - None, - ) - if stopped_index is not None: - source = sources[stopped_index] - result = results[source.task.id] - results[source.task.id] = SourceResult( - "failed", - (*result.errors, f"Harbor did not execute steps: {list(source.runtime_steps)!r}"), - (), - ) - if stopped_index is not None: - for source in sources[stopped_index + 1 :]: - result = results[source.task.id] - if result.status == "skipped" and not result.errors: - results[source.task.id] = SourceResult( - "skipped", - (SHARED_TRIAL_SKIPPED_ERROR,), - (), - ) return results -def _source_result(source: SourceTask, step_results: tuple[Any, ...]) -> SourceResult: - owned_steps = tuple(step for step in step_results if step.step_name in source.runtime_steps) - executed_names = {step.step_name for step in owned_steps} - missing_steps = [step for step in source.runtime_steps if step not in executed_names] - errors = (f"Harbor did not execute steps: {missing_steps!r}",) if missing_steps and owned_steps else () - if not owned_steps: - status: Literal["completed", "failed", "skipped"] = "skipped" - elif missing_steps: - status = "failed" - else: - status = "completed" - return SourceResult(status, errors, owned_steps) - - -def _stopped_source_index( - sources: tuple[SourceTask, ...], - results: dict[str, SourceResult], -) -> int | None: - return next( - ( - index - for index, source in enumerate(sources) - if results[source.task.id].status == "failed" - or any(_step_failed(step) for step in results[source.task.id].steps) - ), - None, - ) - - -def _step_failed(step: Any) -> bool: - rewards = step.verifier_result.rewards if step.verifier_result is not None else {} - return step.exception_info is not None or any(float(value) < 1 for value in (rewards or {}).values()) - - def _source_harbor_observation( shared: HarborTrialObservation, source: SourceTask, result: SourceResult, settings: HarnessSettings, ) -> HarborTrialObservation: - step_rewards = [ - step.verifier_result.rewards - for step in result.steps - if step.verifier_result is not None and step.verifier_result.rewards - ] - reward_names = {name for rewards in step_rewards for name in rewards} - rewards = { - name: sum(float(item.get(name, 0)) for item in step_rewards) / len(step_rewards) for name in reward_names - } step_exception = next((step.exception_info for step in result.steps if step.exception_info is not None), None) exception_type = None if step_exception is None else step_exception.exception_type exception_message = None if step_exception is None else redact(step_exception.exception_message or "", settings) - if result.status != "completed" and step_exception is None and shared.exception_type is not None: + if result.status == "failed" and step_exception is None and shared.exception_type is not None: exception_type = shared.exception_type exception_message = shared.exception_message return HarborTrialObservation( @@ -510,7 +438,7 @@ def _source_harbor_observation( trial_name=shared.trial_name, trial_uri=shared.trial_uri, task_checksum=source.harbor_task.checksum, - rewards=rewards, + rewards=_source_rewards(source, result.steps), exception_type=exception_type, exception_message=exception_message, started_at=shared.started_at, @@ -518,6 +446,19 @@ def _source_harbor_observation( ) +def _source_rewards(source: SourceTask, steps: tuple[StepResult, ...]) -> dict[str, float | int]: + strategy = source.harbor_task.config.multi_step_reward_strategy + if strategy is MultiStepRewardStrategy.FINAL: + verifier = steps[-1].verifier_result if steps else None + return dict(verifier.rewards or {}) if verifier is not None else {} + + rewards = [step.verifier_result.rewards or {} for step in steps if step.verifier_result is not None] + keys = {key for result in rewards for key in result} + if not rewards or not keys: + return {} + return {key: sum(result.get(key, 0) for result in rewards) / len(rewards) for key in keys} + + def _batch_job_config( task: E2ETask, job_name: str, @@ -525,7 +466,7 @@ def _batch_job_config( output_dir: Path, settings: HarnessSettings, *, - dataset_config: DatasetConfig, + task_config: HarborTrialTaskConfig, ) -> JobConfig: config = _job_config(task, job_name, invocation_scopes[0], output_dir, settings) agent = config.agents[0] @@ -534,7 +475,8 @@ def _batch_job_config( return config.model_copy( update={ "agents": [agent.model_copy(update={"env": env, "kwargs": {"invocation_scopes": invocation_scopes}})], - "datasets": [dataset_config], + "datasets": [], + "tasks": [task_config], } ) @@ -623,7 +565,7 @@ def _dataset_config(task: E2ETask, repository: Path) -> DatasetConfig: return DatasetConfig(name=dataset.name, version=dataset.version, task_names=[dataset.task_id]) -def prepare_runtime_dataset( +def prepare_runtime_task( tasks: tuple[E2ETask, ...], *, output_dir: Path, @@ -664,16 +606,8 @@ def prepare_runtime_dataset( runtime_config = _runtime_task_config(sources, failure_policy) runtime_paths.config_path.write_text(runtime_config.model_dump_toml(), encoding="utf-8") - harbor_task = HarborTask(runtime_paths.task_dir) - runtime_checksum = harbor_task.checksum - expected_steps = tuple(step for source in sources for step in source.runtime_steps) - if tuple(step.name for step in harbor_task.config.steps or ()) != expected_steps: - raise ValueError("Runtime Harbor steps do not match the selected source tasks") # noqa: TRY003 - return PreparedRuntime( - dataset_config=DatasetConfig(path=runtime_root, task_names=[runtime_task_id]), - sources=sources, - runtime_checksum=runtime_checksum, - ) + HarborTask(runtime_paths.task_dir) + return PreparedRuntime(HarborTrialTaskConfig(path=runtime_paths.task_dir), sources) def _validate_batch_compatibility( @@ -697,9 +631,6 @@ def _validate_batch_compatibility( raise ValueError( # noqa: TRY003 f"Source task {source.task.id!r} has incompatible batch settings: {incompatible!r}" ) - steps = [step for source in sources for step in source.runtime_steps] - if len(steps) != len(set(steps)): - raise ValueError(f"Batch {batch!r} step names must be unique") # noqa: TRY003 return sources @@ -712,9 +643,6 @@ def _load_source_task(task: E2ETask, repository: Path) -> SourceTask: harbor_task = HarborTask(task_dir) except Exception as exc: raise ValueError(f"Source task {task.id!r} cannot be loaded from {task_dir}") from exc # noqa: TRY003 - unexpected = {item.name for item in task_dir.iterdir()} - {"environment", "steps", "task.toml", "tests"} - if unexpected: - raise ValueError(f"Source task {task.id!r} has unsupported runtime inputs: {sorted(unexpected)!r}") # noqa: TRY003 if harbor_task.checksum != task.dataset.checksum: raise ValueError(f"Source task {task.id!r} checksum changed") # noqa: TRY003 steps = _task_layout(task, harbor_task) @@ -727,40 +655,19 @@ def _runtime_profile(source: SourceTask) -> dict[str, Any]: return { "dataset": task.dataset.model_dump(mode="json", exclude={"task_id", "checksum"}), "execution": task.execution.model_dump(mode="json"), - "capture": _capture_profile(task), - "harbor": _shared_harbor_config(source.harbor_task), - "environment": _directory_snapshot(paths.environment_dir), - "tests": _directory_snapshot(paths.tests_dir), + "capture": ( + task.evaluation.capture_events, + task.evaluation.checkpoint_every_events, + task.evaluation.max_event_bytes, + ), + "harbor": source.harbor_task.config.model_dump(mode="json", exclude={"steps"}), + "environment": _directory_checksum(paths.environment_dir), + "tests": _directory_checksum(paths.tests_dir), } -def _capture_profile(task: E2ETask) -> tuple[bool, int, int]: - evaluation = task.evaluation - return ( - evaluation.capture_events, - evaluation.checkpoint_every_events, - evaluation.max_event_bytes, - ) - - -def _shared_harbor_config(task: HarborTask) -> dict[str, Any]: - return task.config.model_dump(mode="json", exclude={"steps"}) - - -def _directory_snapshot(path: Path) -> tuple[tuple[str, int, str], ...]: - if not path.is_dir(): - return () - snapshot = [] - for item in sorted(path.rglob("*")): - if item.is_symlink(): - raise ValueError(f"Shared Harbor directory cannot contain symlinks: {item}") # noqa: TRY003 - if item.is_file(): - snapshot.append(( - item.relative_to(path).as_posix(), - item.stat().st_mode & 0o777, - hashlib.sha256(item.read_bytes()).hexdigest(), - )) - return tuple(snapshot) +def _directory_checksum(path: Path) -> str | None: + return dirhash(path, "sha256") if path.is_dir() else None def _runtime_task_config(sources: tuple[SourceTask, ...], failure_policy: FailurePolicy) -> TaskConfig: @@ -789,11 +696,6 @@ def _task_layout(task: E2ETask, harbor_task: HarborTask) -> tuple[str, ...]: for step, path in zip(steps, paths, strict=True) ): raise ValueError(f"Source task {task.id!r} step names must be single path components") # noqa: TRY003 - step_entries = {path.name: path.is_dir() for path in harbor_task.paths.steps_dir.iterdir()} - if step_entries != dict.fromkeys(steps, True): - raise ValueError(f"Source task {task.id!r} step directories do not match task.toml") # noqa: TRY003 - if harbor_task.step_instruction(steps[0]).strip() != ",tape.reset": - raise ValueError(f"Task {task.id!r} must start with a tape.reset step") # noqa: TRY003 return steps @@ -902,92 +804,30 @@ def _load_capture_records(trial_dir: Path) -> tuple[CaptureRecord, ...]: return tuple(records) -def _native_artifacts(trial_dir: Path, names: frozenset[str]) -> tuple[NativeArtifact, ...]: +def _native_artifacts( + root: Path, names: frozenset[str], *, relative_to: Path | None = None +) -> tuple[NativeArtifact, ...]: return tuple( - fingerprint(path, relative_to=trial_dir) for path in sorted(trial_dir.rglob("*")) if path.name in names + fingerprint(path, relative_to=relative_to or root) for path in sorted(root.rglob("*")) if path.name in names ) -def collect_task_artifacts( - sources: tuple[SourceTask, ...], +def _load_task_artifacts( trial_dir: Path, native_artifact_names: frozenset[str], settings: HarnessSettings, -) -> dict[str, TaskArtifacts]: - """Collect shared-trial artifacts below each source task's Harbor steps.""" - - trial_paths = TrialPaths(trial_dir) - instructions = load_resolved_instructions(trial_dir, settings) - return { - source.task.id: _collect_source_artifacts( - source, - trial_paths, - native_artifact_names, - instructions, - settings, - ) - for source in sources - } - - -def _collect_source_artifacts( - source: SourceTask, - trial_paths: TrialPaths, - native_artifact_names: frozenset[str], - instructions: tuple[ResolvedInstruction, ...], - settings: HarnessSettings, + *, + step_names: tuple[str, ...] = (), ) -> TaskArtifacts: - step_roots = tuple(trial_paths.step_dir(step) for step in source.runtime_steps) - errors: list[str] = [] - capture_records = _source_capture_records(step_roots, trial_paths.trial_dir, settings, errors) - native_artifacts: list[NativeArtifact] = [] - for root in step_roots: - for path in sorted(root.rglob("*")): - if path.name not in native_artifact_names: - continue - try: - native_artifacts.append(fingerprint(path, relative_to=trial_paths.trial_dir)) - except OSError as exc: - errors.append( - redact( - f"Cannot fingerprint {path.relative_to(trial_paths.trial_dir)}: {type(exc).__name__}: {exc}", - settings, - ) - ) - prefixes = tuple(f"steps/{step}/" for step in source.runtime_steps) - resolved_instructions = tuple( - instruction for instruction in instructions if instruction.artifact.startswith(prefixes) + roots = tuple(TrialPaths(trial_dir).step_dir(name) for name in step_names) or (trial_dir,) + instructions = load_resolved_instructions(trial_dir, settings) + prefixes = tuple(f"steps/{name}/" for name in step_names) + return TaskArtifacts( + tuple(record for root in roots for record in _load_capture_records(root)), + tuple( + artifact + for root in roots + for artifact in _native_artifacts(root, native_artifact_names, relative_to=trial_dir) + ), + tuple(instruction for instruction in instructions if not prefixes or instruction.artifact.startswith(prefixes)), ) - return TaskArtifacts(tuple(capture_records), tuple(native_artifacts), resolved_instructions, tuple(errors)) - - -def _source_capture_records( - step_roots: tuple[Path, ...], - trial_dir: Path, - settings: HarnessSettings, - errors: list[str], -) -> list[CaptureRecord]: - records: list[CaptureRecord] = [] - for root in step_roots: - for path in sorted(root.rglob("powercontext-capture.jsonl")): - try: - lines = path.read_text(encoding="utf-8").splitlines() - except (OSError, UnicodeDecodeError) as exc: - errors.append( - redact(f"Cannot read {path.relative_to(trial_dir)}: {type(exc).__name__}: {exc}", settings) - ) - continue - for line_number, line in enumerate(lines, start=1): - if not line.strip(): - continue - try: - records.append(CaptureRecord.model_validate_json(line)) - except ValidationError as exc: - errors.append( - redact( - f"Cannot parse {path.relative_to(trial_dir)} line {line_number}: " - f"{type(exc).__name__}: {exc}", - settings, - ) - ) - return records diff --git a/e2e/bub/tasks/locomo-multihop-football.yaml b/e2e/bub/tasks/locomo-multihop-football.yaml index 862ecee0c..a28751a44 100644 --- a/e2e/bub/tasks/locomo-multihop-football.yaml +++ b/e2e/bub/tasks/locomo-multihop-football.yaml @@ -29,7 +29,7 @@ provenance: dataset: path: e2e/bub/harbor-tasks task_id: locomo-multihop-football - checksum: 90a2a2ea97636106f1a44c7880a005304047a9d77825a375dc4f2a8270f114a1 + checksum: 773e967b07c26ab3b6e8e0de6f1f0d1c752e6d10c3227b5788796ee967aeff6f execution: type: bub model: false diff --git a/e2e/bub/tasks/locomo-open-pastries.yaml b/e2e/bub/tasks/locomo-open-pastries.yaml index 312b037fa..34bf96e34 100644 --- a/e2e/bub/tasks/locomo-open-pastries.yaml +++ b/e2e/bub/tasks/locomo-open-pastries.yaml @@ -29,7 +29,7 @@ provenance: dataset: path: e2e/bub/harbor-tasks task_id: locomo-open-pastries - checksum: a33560d6baf2729987fe98928c8ae153ace91e6562df007178623542bbf3d48e + checksum: 92991fc1ff3edb187b420c943e88e504c2b0565d9c6c757e7e13f8c77ed3d81a execution: type: bub model: false diff --git a/e2e/bub/tasks/locomo-support-group.yaml b/e2e/bub/tasks/locomo-support-group.yaml index fc01ca74d..f1926b979 100644 --- a/e2e/bub/tasks/locomo-support-group.yaml +++ b/e2e/bub/tasks/locomo-support-group.yaml @@ -29,7 +29,7 @@ provenance: dataset: path: e2e/bub/harbor-tasks task_id: locomo-support-group - checksum: d1e9e18ae098210917073d80b46e3da00de850ec20e03ef7e1424a90e1f2fbd2 + checksum: babb50723aff96edba700a80159aa2fbf8333a4111ad8339f54e81d138f9ff11 execution: type: bub model: false diff --git a/e2e/bub/tasks/locomo-temporal-banker.yaml b/e2e/bub/tasks/locomo-temporal-banker.yaml index c1b66b083..6e03b60ba 100644 --- a/e2e/bub/tasks/locomo-temporal-banker.yaml +++ b/e2e/bub/tasks/locomo-temporal-banker.yaml @@ -29,7 +29,7 @@ provenance: dataset: path: e2e/bub/harbor-tasks task_id: locomo-temporal-banker - checksum: 594e9e0c127f31ec37b62617c9136c1169e81e7b7f604d418e6c4df230549ccd + checksum: 1afc402fffa2ea2509fb1007ec218eda81e09d86401304a5570af39eca28fb58 execution: type: bub model: false diff --git a/e2e/bub/tests/test_runtime_batch.py b/e2e/bub/tests/test_runtime_batch.py index b91aa8f62..c7e073ecb 100644 --- a/e2e/bub/tests/test_runtime_batch.py +++ b/e2e/bub/tests/test_runtime_batch.py @@ -15,72 +15,66 @@ from __future__ import annotations import json -from datetime import UTC, datetime from pathlib import Path +import pytest from harbor.models.task.task import Task as HarborTask -from harbor.models.trial.result import ExceptionInfo, StepResult +from harbor.models.trial.result import StepResult from harbor.models.verifier.result import VerifierResult from powercontext_e2e.catalog import E2ETask, load_tasks -from powercontext_e2e.evaluation import MemoryEvaluator -from powercontext_e2e.models import ( - SHARED_TRIAL_SKIPPED_ERROR, - HarborTrialObservation, - MemorySnapshot, - RunEnvironment, - TaskObservation, +from powercontext_e2e.models import HarborTrialObservation +from powercontext_e2e.runner import ( + SourceResult, + SourceTask, + _load_task_artifacts, + _source_harbor_observation, + prepare_runtime_task, ) -from powercontext_e2e.runner import _source_results, collect_task_artifacts, prepare_runtime_dataset from powercontext_e2e.settings import HarnessSettings - -def _repository() -> Path: - return Path(__file__).resolve().parents[3] +REPOSITORY = Path(__file__).resolve().parents[3] def _locomo_tasks() -> tuple[E2ETask, ...]: - return tuple( - task for task in load_tasks(_repository() / "e2e" / "bub" / "tasks") if "batch:locomo" in task.categories - ) + return tuple(task for task in load_tasks(REPOSITORY / "e2e" / "bub" / "tasks") if "batch:locomo" in task.categories) def test_selected_locomo_tasks_share_one_runtime_with_requested_failure_policy(tmp_path: Path) -> None: tasks = _locomo_tasks() - settings = HarnessSettings(repository=_repository()) + settings = HarnessSettings(repository=REPOSITORY) - collect_all = prepare_runtime_dataset( + collect_all = prepare_runtime_task( tasks, output_dir=tmp_path, settings=settings, failure_policy="collect-all", ) - runtime_task_id = collect_all.dataset_config.task_names[0] - runtime_task = HarborTask(collect_all.dataset_config.path / runtime_task_id) - runtime_step_names = tuple(f"{task.id}-{step}" for task in tasks for step in ("reset", "capture", "recall")) + assert collect_all.task_config.path is not None + runtime_task = HarborTask(collect_all.task_config.path) + runtime_step_names = tuple(f"{task.id}-{step}" for task in tasks for step in ("capture", "recall")) - assert len(collect_all.dataset_config.task_names) == 1 assert tuple(step.name for step in runtime_task.config.steps or ()) == runtime_step_names assert all(step.min_reward is None for step in runtime_task.config.steps or ()) - fail_fast = prepare_runtime_dataset( + fail_fast = prepare_runtime_task( tasks, output_dir=tmp_path, settings=settings, failure_policy="fail-fast", ) - fail_fast_task_id = fail_fast.dataset_config.task_names[0] - fail_fast_task = HarborTask(fail_fast.dataset_config.path / fail_fast_task_id) - assert fail_fast_task_id != runtime_task_id + assert fail_fast.task_config.path is not None + fail_fast_task = HarborTask(fail_fast.task_config.path) + assert fail_fast.task_config.path != collect_all.task_config.path assert all(step.min_reward == 1.0 for step in fail_fast_task.config.steps or ()) def test_shared_trial_artifacts_stay_with_their_source_task(tmp_path: Path) -> None: tasks = _locomo_tasks()[:2] - prepared = prepare_runtime_dataset( + prepared = prepare_runtime_task( tasks, output_dir=tmp_path / "runtime", - settings=HarnessSettings(repository=_repository()), + settings=HarnessSettings(repository=REPOSITORY), failure_policy="collect-all", ) trial_dir = tmp_path / "trial" @@ -98,116 +92,51 @@ def test_shared_trial_artifacts_stay_with_their_source_task(tmp_path: Path) -> N + "\n", encoding="utf-8", ) - malformed_path = ( - trial_dir / "steps" / f"{prepared.sources[0].task.id}-capture" / "agent" / "powercontext-capture.jsonl" - ) - malformed_path.parent.mkdir(parents=True) - malformed_path.write_text("{truncated\n", encoding="utf-8") - - artifacts = collect_task_artifacts( - prepared.sources, - trial_dir, - tasks[0].execution.native_artifact_names, - HarnessSettings(repository=_repository()), - ) + settings = HarnessSettings(repository=REPOSITORY) + artifacts = { + source.task.id: _load_task_artifacts( + trial_dir, + source.task.execution.native_artifact_names, + settings, + step_names=source.runtime_steps, + ) + for source in prepared.sources + } assert {task.id: [record.source_id for record in artifacts[task.id].capture_records] for task in tasks} == { task.id: [f"bub-event:{task.id}"] for task in tasks } - assert len(artifacts[tasks[0].id].errors) == 1 - assert artifacts[tasks[1].id].errors == () - - -def test_collect_all_keeps_later_tasks_after_an_agent_failure(tmp_path: Path) -> None: - tasks = _locomo_tasks()[:2] - prepared = prepare_runtime_dataset( - tasks, - output_dir=tmp_path, - settings=HarnessSettings(repository=_repository()), - failure_policy="collect-all", - ) - failed_step = prepared.sources[0].runtime_steps[1] - step_results = tuple( - StepResult( - step_name=step, - verifier_result=VerifierResult(rewards={"reward": 0 if step == failed_step else 1}), - exception_info=( - ExceptionInfo( - exception_type="AgentError", - exception_message="agent failed", - exception_traceback="", - occurred_at=datetime(2026, 8, 24, tzinfo=UTC), - ) - if step == failed_step - else None - ), - ) - for source in prepared.sources - for step in source.runtime_steps - ) - - results = _source_results(prepared.sources, step_results, (), None) - assert results[tasks[0].id].status == "completed" - assert results[tasks[0].id].errors == () - assert results[tasks[1].id].status == "completed" - assert len(results[tasks[1].id].steps) == 3 - -def test_fail_fast_marks_unexecuted_tasks_as_skipped(tmp_path: Path) -> None: - tasks = _locomo_tasks()[:2] - prepared = prepare_runtime_dataset( - tasks, - output_dir=tmp_path, - settings=HarnessSettings(repository=_repository()), - failure_policy="fail-fast", - ) - failed_step = prepared.sources[0].runtime_steps[0] - step_results = ( - StepResult( - step_name=failed_step, - verifier_result=VerifierResult(rewards={"reward": 0}), - exception_info=ExceptionInfo( - exception_type="AgentError", - exception_message="agent failed", - exception_traceback="", - occurred_at=datetime(2026, 8, 24, tzinfo=UTC), - ), +@pytest.mark.parametrize( + ("task_id", "step_rewards", "expected"), + [ + ( + "locomo-support-group", + ({"reward": 1, "detail": 1}, {"reward": 0}), + {"reward": 0.5, "detail": 0.5}, ), + ("project-database-decision", ({"reward": 1}, {"reward": 0}), {"reward": 0}), + ], +) +def test_batch_source_evidence_preserves_harbor_reward_strategy( + task_id: str, + step_rewards: tuple[dict[str, int], ...], + expected: dict[str, float | int], +) -> None: + task = load_tasks(REPOSITORY / "e2e" / "bub" / "tasks" / f"{task_id}.yaml")[0] + assert task.dataset.path is not None + harbor_task = HarborTask(REPOSITORY / task.dataset.path / task.dataset.task_id) + source = SourceTask(task, harbor_task, tuple(step.name for step in harbor_task.config.steps or ())) + steps = tuple( + StepResult(step_name=name, verifier_result=VerifierResult(rewards=rewards)) + for name, rewards in zip(source.runtime_steps, step_rewards, strict=True) ) - - results = _source_results(prepared.sources, step_results, (), None) - - assert results[tasks[0].id].status == "failed" - assert "did not execute steps" in results[tasks[0].id].errors[0] - assert results[tasks[1].id].status == "skipped" - assert results[tasks[1].id].errors == (SHARED_TRIAL_SKIPPED_ERROR,) - - -def test_skipped_shared_task_has_only_an_execution_result() -> None: - task = _locomo_tasks()[0] - recorded_at = datetime(2026, 8, 24, tzinfo=UTC) - observation = TaskObservation( - run_id="batch-locomo-test", - environment=RunEnvironment( - commit="abcdef0", - database="sqlite", - adapter_version="test-adapter", - adapter_protocol_version="test-protocol", - started_at=recorded_at, - finished_at=recorded_at, - ), - task=task, - status="failed", - errors=(SHARED_TRIAL_SKIPPED_ERROR,), - harbor=HarborTrialObservation(task_checksum=task.dataset.checksum), - memory_before=MemorySnapshot(), - memory_after=MemorySnapshot(), + source_harbor = _source_harbor_observation( + HarborTrialObservation(), + source, + SourceResult("completed", (), steps), + HarnessSettings(repository=REPOSITORY), ) - report = MemoryEvaluator.evaluate(observation, experiment="e2e:batch:locomo") - case = report.cases[0] - - assert set(case.assertions) == {"execution_completed"} - assert case.assertions["execution_completed"].value is False - assert case.labels["task_outcome"].value == "skipped" + assert source_harbor.rewards == expected From 9a455a14a187254010c971059055941c365dd4fd Mon Sep 17 00:00:00 2001 From: Chojan Shang Date: Mon, 24 Aug 2026 11:29:04 +0800 Subject: [PATCH 3/4] test(e2e): add Harbor failure policy acceptance --- e2e/bub/README.md | 8 +- .../environment/Dockerfile | 17 +++ .../steps/run/instruction.md | 1 + .../steps/run/solution/solve.sh | 18 +++ .../steps/run/tests/test.sh | 22 +++ .../failure-policy-followup/task.toml | 18 +++ .../environment/Dockerfile | 17 +++ .../steps/run/instruction.md | 1 + .../steps/run/solution/solve.sh | 18 +++ .../steps/run/tests/test.sh | 18 +++ .../failure-policy-timeout/task.toml | 21 +++ e2e/bub/src/powercontext_e2e/__main__.py | 46 +++++- .../powercontext_e2e/failure_acceptance.py | 141 ++++++++++++++++++ e2e/bub/src/powercontext_e2e/report.py | 8 +- 14 files changed, 344 insertions(+), 10 deletions(-) create mode 100644 e2e/bub/harbor-tasks/failure-policy-followup/environment/Dockerfile create mode 100644 e2e/bub/harbor-tasks/failure-policy-followup/steps/run/instruction.md create mode 100644 e2e/bub/harbor-tasks/failure-policy-followup/steps/run/solution/solve.sh create mode 100644 e2e/bub/harbor-tasks/failure-policy-followup/steps/run/tests/test.sh create mode 100644 e2e/bub/harbor-tasks/failure-policy-followup/task.toml create mode 100644 e2e/bub/harbor-tasks/failure-policy-timeout/environment/Dockerfile create mode 100644 e2e/bub/harbor-tasks/failure-policy-timeout/steps/run/instruction.md create mode 100644 e2e/bub/harbor-tasks/failure-policy-timeout/steps/run/solution/solve.sh create mode 100644 e2e/bub/harbor-tasks/failure-policy-timeout/steps/run/tests/test.sh create mode 100644 e2e/bub/harbor-tasks/failure-policy-timeout/task.toml create mode 100644 e2e/bub/src/powercontext_e2e/failure_acceptance.py diff --git a/e2e/bub/README.md b/e2e/bub/README.md index b6d7dc444..224351f2e 100644 --- a/e2e/bub/README.md +++ b/e2e/bub/README.md @@ -92,6 +92,10 @@ export POWERCONTEXT_BUB_BASE_URL=http://host-gateway:8000 make harness-acceptance ``` +The default run also exercises Harbor runtime batching under collect-all and fail-fast with native Oracle tasks. Its +report and Harbor artifacts are written under `/failure-policy/`. Explicit ID or category selection runs only +the selected PowerContext workloads. + Selection uses the `acceptance` command's repeatable `--id` and `--category` options: ```bash @@ -122,8 +126,8 @@ Each selected workload writes the same layout: Shared runs write the same v1 files per source task under `batch-/tasks//`, plus one aggregate evaluation and report at `batch-/`. `collect-all` reports every failed task; `fail-fast` stops only that shared -Harbor trial at its first failed step. Each source task starts with `steps/reset`; runtime batch steps are flat and -task-prefixed. +Harbor trial at its first failed step. Runtime batch steps are flat and task-prefixed. Each agent invocation starts an +independent ACP session and Bub tape. ## Long-horizon task diff --git a/e2e/bub/harbor-tasks/failure-policy-followup/environment/Dockerfile b/e2e/bub/harbor-tasks/failure-policy-followup/environment/Dockerfile new file mode 100644 index 000000000..6da75edc5 --- /dev/null +++ b/e2e/bub/harbor-tasks/failure-policy-followup/environment/Dockerfile @@ -0,0 +1,17 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM python:3.12-slim-bookworm + +WORKDIR /workspace diff --git a/e2e/bub/harbor-tasks/failure-policy-followup/steps/run/instruction.md b/e2e/bub/harbor-tasks/failure-policy-followup/steps/run/instruction.md new file mode 100644 index 000000000..aa75293d1 --- /dev/null +++ b/e2e/bub/harbor-tasks/failure-policy-followup/steps/run/instruction.md @@ -0,0 +1 @@ +Run the follow-up failure-policy fixture. diff --git a/e2e/bub/harbor-tasks/failure-policy-followup/steps/run/solution/solve.sh b/e2e/bub/harbor-tasks/failure-policy-followup/steps/run/solution/solve.sh new file mode 100644 index 000000000..2016def17 --- /dev/null +++ b/e2e/bub/harbor-tasks/failure-policy-followup/steps/run/solution/solve.sh @@ -0,0 +1,18 @@ +#!/bin/sh +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eu + +touch /workspace/failure-policy-followup-ran diff --git a/e2e/bub/harbor-tasks/failure-policy-followup/steps/run/tests/test.sh b/e2e/bub/harbor-tasks/failure-policy-followup/steps/run/tests/test.sh new file mode 100644 index 000000000..1e78473bd --- /dev/null +++ b/e2e/bub/harbor-tasks/failure-policy-followup/steps/run/tests/test.sh @@ -0,0 +1,22 @@ +#!/bin/sh +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eu + +if [ -e /workspace/failure-policy-followup-ran ]; then + echo 1 > /logs/verifier/reward.txt +else + echo 0 > /logs/verifier/reward.txt +fi diff --git a/e2e/bub/harbor-tasks/failure-policy-followup/task.toml b/e2e/bub/harbor-tasks/failure-policy-followup/task.toml new file mode 100644 index 000000000..28173512e --- /dev/null +++ b/e2e/bub/harbor-tasks/failure-policy-followup/task.toml @@ -0,0 +1,18 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version = "1.3" + +[[steps]] +name = "run" diff --git a/e2e/bub/harbor-tasks/failure-policy-timeout/environment/Dockerfile b/e2e/bub/harbor-tasks/failure-policy-timeout/environment/Dockerfile new file mode 100644 index 000000000..6da75edc5 --- /dev/null +++ b/e2e/bub/harbor-tasks/failure-policy-timeout/environment/Dockerfile @@ -0,0 +1,17 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM python:3.12-slim-bookworm + +WORKDIR /workspace diff --git a/e2e/bub/harbor-tasks/failure-policy-timeout/steps/run/instruction.md b/e2e/bub/harbor-tasks/failure-policy-timeout/steps/run/instruction.md new file mode 100644 index 000000000..26f577d0d --- /dev/null +++ b/e2e/bub/harbor-tasks/failure-policy-timeout/steps/run/instruction.md @@ -0,0 +1 @@ +Run the timeout failure-policy fixture. diff --git a/e2e/bub/harbor-tasks/failure-policy-timeout/steps/run/solution/solve.sh b/e2e/bub/harbor-tasks/failure-policy-timeout/steps/run/solution/solve.sh new file mode 100644 index 000000000..ff7c127d5 --- /dev/null +++ b/e2e/bub/harbor-tasks/failure-policy-timeout/steps/run/solution/solve.sh @@ -0,0 +1,18 @@ +#!/bin/sh +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eu + +sleep 5 diff --git a/e2e/bub/harbor-tasks/failure-policy-timeout/steps/run/tests/test.sh b/e2e/bub/harbor-tasks/failure-policy-timeout/steps/run/tests/test.sh new file mode 100644 index 000000000..27384a095 --- /dev/null +++ b/e2e/bub/harbor-tasks/failure-policy-timeout/steps/run/tests/test.sh @@ -0,0 +1,18 @@ +#!/bin/sh +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eu + +echo 0 > /logs/verifier/reward.txt diff --git a/e2e/bub/harbor-tasks/failure-policy-timeout/task.toml b/e2e/bub/harbor-tasks/failure-policy-timeout/task.toml new file mode 100644 index 000000000..51f8a2961 --- /dev/null +++ b/e2e/bub/harbor-tasks/failure-policy-timeout/task.toml @@ -0,0 +1,21 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version = "1.3" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1.0 diff --git a/e2e/bub/src/powercontext_e2e/__main__.py b/e2e/bub/src/powercontext_e2e/__main__.py index 951da301b..4fefe51fd 100644 --- a/e2e/bub/src/powercontext_e2e/__main__.py +++ b/e2e/bub/src/powercontext_e2e/__main__.py @@ -19,6 +19,7 @@ import argparse import asyncio import sys +from collections.abc import Awaitable from pathlib import Path from loguru import logger @@ -26,6 +27,30 @@ from .settings import HarnessSettings +async def _run_default_acceptance( + workloads: Awaitable[bool], + *, + output_dir: Path, + settings: HarnessSettings, +) -> bool: + from .failure_acceptance import run_failure_acceptance + + workloads_passed = await _accepted(workloads, "Memory workload acceptance") + policy_passed = await _accepted( + run_failure_acceptance(output_dir / "failure-policy", settings), + "Harbor failure-policy acceptance", + ) + return workloads_passed and policy_passed + + +async def _accepted(result: Awaitable[bool], name: str) -> bool: + try: + return await result + except Exception: + logger.exception(f"{name} failed.") + return False + + def main() -> None: logger.remove() logger.add(sys.stderr, backtrace=False, diagnose=False) @@ -72,12 +97,21 @@ def main() -> None: tasks = load_tasks(args.manifest) selected = select_tasks(tasks, ids=tuple(args.id), categories=tuple(args.category)) - passed = asyncio.run( - run_tasks( - selected, - output_dir=args.output, - settings=settings, - failure_policy=args.failure_policy, + workloads = run_tasks( + selected, + output_dir=args.output, + settings=settings, + failure_policy=args.failure_policy, + ) + passed = ( + asyncio.run(workloads) + if args.id or args.category + else asyncio.run( + _run_default_acceptance( + workloads, + output_dir=args.output, + settings=settings, + ) ) ) raise SystemExit(0 if passed else 1) diff --git a/e2e/bub/src/powercontext_e2e/failure_acceptance.py b/e2e/bub/src/powercontext_e2e/failure_acceptance.py new file mode 100644 index 000000000..dced6e5a0 --- /dev/null +++ b/e2e/bub/src/powercontext_e2e/failure_acceptance.py @@ -0,0 +1,141 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Verify Harbor collect-all and fail-fast behavior with real trials.""" + +from __future__ import annotations + +from pathlib import Path +from uuid import uuid4 + +from harbor.job import Job +from harbor.models.environment_type import EnvironmentType +from harbor.models.job.config import JobConfig +from harbor.models.task.task import Task as HarborTask +from harbor.models.trial.config import AgentConfig, EnvironmentConfig, ResourceMode, TaskConfig +from harbor.models.trial.result import StepResult + +from .catalog import E2ETask +from .evidence import redact, write_evaluation_report, write_evidence +from .models import CaseEvaluation, EvaluationReport, EvaluationValue +from .report import render_evaluation_summary +from .runner import FailurePolicy, prepare_runtime_task +from .settings import HarnessSettings + +DATASET_PATH = Path("e2e/bub/harbor-tasks") +TASK_IDS = ("failure-policy-timeout", "failure-policy-followup") + + +async def run_failure_acceptance(output_dir: Path, settings: HarnessSettings) -> bool: + """Run both policies and write one acceptance report.""" + + cases = [] + for policy in ("collect-all", "fail-fast"): + try: + steps = await _run_policy(policy, output_dir / policy, settings) + failed = steps.get("failure-policy-timeout-run") + followup = steps.get("failure-policy-followup-run") + failure_ok = failed is not None and failed.exception_info is not None and _reward(failed) == 0 + followup_ok = ( + followup is not None and followup.exception_info is None and _reward(followup) == 1 + if policy == "collect-all" + else followup is None + ) + failure_reason = _step_summary(failed) + followup_reason = _step_summary(followup) + except Exception as exc: + failure_ok = followup_ok = False + failure_reason = followup_reason = redact(f"{type(exc).__name__}: {exc}", settings) + + followup_name = "followup_executed" if policy == "collect-all" else "followup_skipped" + accepted = failure_ok and followup_ok + cases.append( + CaseEvaluation( + name=policy, + assertions={ + "failure_observed": EvaluationValue(value=failure_ok, reason=failure_reason), + followup_name: EvaluationValue(value=followup_ok, reason=followup_reason), + }, + labels={"task_outcome": EvaluationValue(value="passed" if accepted else "not_passed")}, + ) + ) + + report = EvaluationReport(experiment="e2e:failure-policy", cases=tuple(cases)) + output_dir.mkdir(parents=True, exist_ok=True) + write_evaluation_report(output_dir / "eval-report.json", report=report, settings=settings) + write_evidence( + output_dir / "report.md", + render_evaluation_summary(report, title="PowerContext Harbor failure-policy acceptance"), + settings, + ) + return report.accepted + + +async def _run_policy( + policy: FailurePolicy, + output_dir: Path, + settings: HarnessSettings, +) -> dict[str, StepResult]: + runtime_id = f"failure-policy-{policy}-{uuid4().hex[:12]}" + runtime = prepare_runtime_task( + _source_tasks(settings), + output_dir=output_dir, + settings=settings, + failure_policy=policy, + runtime_id=runtime_id, + ) + result = await (await Job.create(_job_config(runtime_id, output_dir, runtime.task_config))).run() + if len(result.trial_results) != 1: + return {} + return {step.step_name: step for step in result.trial_results[0].step_results or ()} + + +def _source_tasks(settings: HarnessSettings) -> tuple[E2ETask, ...]: + root = settings.repository_path() / DATASET_PATH + return tuple( + E2ETask.model_validate({ + "schema": "powercontext.e2e-task/v1", + "id": task_id, + "categories": ["batch:failure-policy"], + "dataset": {"path": str(DATASET_PATH), "task_id": task_id, "checksum": HarborTask(root / task_id).checksum}, + "execution": {"type": "bub", "model": False}, + "evaluation": {"probes": [{"id": "policy-fixture", "query": "Run the policy fixture."}]}, + }) + for task_id in TASK_IDS + ) + + +def _job_config(runtime_id: str, output_dir: Path, task: TaskConfig) -> JobConfig: + return JobConfig( + job_name=runtime_id, + jobs_dir=output_dir / "harbor-jobs", + quiet=True, + environment=EnvironmentConfig( + type=EnvironmentType.DOCKER, + cpu_enforcement_policy=ResourceMode.IGNORE, + memory_enforcement_policy=ResourceMode.IGNORE, + ), + agents=[AgentConfig(name="oracle")], + tasks=[task], + ) + + +def _reward(step: StepResult) -> float | int | None: + return None if step.verifier_result is None else (step.verifier_result.rewards or {}).get("reward") + + +def _step_summary(step: StepResult | None) -> str: + if step is None: + return "The step was not executed." + return f"Agent exception: {step.exception_info is not None}; verifier reward: {_reward(step)!r}." diff --git a/e2e/bub/src/powercontext_e2e/report.py b/e2e/bub/src/powercontext_e2e/report.py index b876ac454..efe14cdff 100644 --- a/e2e/bub/src/powercontext_e2e/report.py +++ b/e2e/bub/src/powercontext_e2e/report.py @@ -57,11 +57,15 @@ def render_report(observation: TaskObservation, report: EvaluationReport) -> str return markdown.render(document) -def render_evaluation_summary(report: EvaluationReport) -> str: +def render_evaluation_summary( + report: EvaluationReport, + *, + title: str = "PowerContext end-to-end Memory evaluation", +) -> str: markdown = Markdown(renderer=MarkdownRenderer) document = block.Document() document.children = [ - *_nodes(markdown, "# PowerContext end-to-end Memory evaluation"), + *_nodes(markdown, f"# {title}"), block.BlankLine(0), *_nodes(markdown, f"```text\n{_evaluation_text(report)}\n```"), ] From 43227bd00d0fc76328abe92521217879cdb52e5d Mon Sep 17 00:00:00 2001 From: Chojan Shang Date: Mon, 24 Aug 2026 15:03:28 +0800 Subject: [PATCH 4/4] refactor(e2e): simplify shared Harbor batches --- e2e/bub/README.md | 9 +- .../steps/run/instruction.md | 2 +- .../failure-policy-followup/task.toml | 10 + .../steps/run/instruction.md | 2 +- .../failure-policy-timeout/task.toml | 10 + .../locomo-multihop-football/task.toml | 2 +- .../locomo-open-pastries/task.toml | 2 +- .../locomo-support-group/task.toml | 2 +- .../locomo-temporal-banker/task.toml | 2 +- .../steps/capture/tests/test.sh | 6 +- .../steps/recall/tests/test.sh | 6 +- e2e/bub/src/powercontext_e2e/__main__.py | 54 +- e2e/bub/src/powercontext_e2e/catalog.py | 11 +- e2e/bub/src/powercontext_e2e/evaluation.py | 76 ++- .../powercontext_e2e/failure_acceptance.py | 141 ----- e2e/bub/src/powercontext_e2e/harbor_agent.py | 8 +- e2e/bub/src/powercontext_e2e/models.py | 5 +- e2e/bub/src/powercontext_e2e/report.py | 12 +- e2e/bub/src/powercontext_e2e/runner.py | 595 ++++++++---------- ...eptance-01-project-database-decision.yaml} | 3 +- ...acceptance-02-failure-policy-timeout.yaml} | 23 +- ...cceptance-03-failure-policy-followup.yaml} | 23 +- e2e/bub/tasks/locomo-multihop-football.yaml | 2 +- e2e/bub/tasks/locomo-open-pastries.yaml | 2 +- e2e/bub/tasks/locomo-support-group.yaml | 2 +- e2e/bub/tasks/locomo-temporal-banker.yaml | 2 +- e2e/bub/tests/test_runtime_batch.py | 142 ----- e2e/bub/tests/test_workload_catalog.py | 12 +- 28 files changed, 478 insertions(+), 688 deletions(-) delete mode 100644 e2e/bub/src/powercontext_e2e/failure_acceptance.py rename e2e/bub/tasks/{project-database-decision.yaml => acceptance-01-project-database-decision.yaml} (92%) rename e2e/bub/{harbor-tasks/failure-policy-followup/steps/run/solution/solve.sh => tasks/acceptance-02-failure-policy-timeout.yaml} (57%) rename e2e/bub/{harbor-tasks/failure-policy-timeout/steps/run/solution/solve.sh => tasks/acceptance-03-failure-policy-followup.yaml} (57%) delete mode 100644 e2e/bub/tests/test_runtime_batch.py diff --git a/e2e/bub/README.md b/e2e/bub/README.md index 224351f2e..610084344 100644 --- a/e2e/bub/README.md +++ b/e2e/bub/README.md @@ -79,8 +79,9 @@ The built-in manifests are: | ID | Dataset | Categories | Purpose | | --- | --- | --- | --- | | `locomo-*` (four manifests) | one local Harbor task each | `acceptance`, `sample`, `batch:locomo` | Pinned LoCoMo-derived cases | -| `project-database-decision` | local Harbor multi-step task | `acceptance`, `sample`, `smoke` | Durable project decision | +| `project-database-decision` | local Harbor multi-step task | `acceptance`, `sample`, `smoke`, `batch:acceptance` | Durable project decision | | `terminal-bench-db-wal-recovery` | `terminal-bench@2.0` | `long-horizon`, `terminal-bench` | Long-running capture and recall | +| `failure-policy-*` | local Harbor tasks | `acceptance`, `fixture`, `batch:acceptance` | Collect-all and fail-fast behavior | ## Run acceptance workloads @@ -92,9 +93,9 @@ export POWERCONTEXT_BUB_BASE_URL=http://host-gateway:8000 make harness-acceptance ``` -The default run also exercises Harbor runtime batching under collect-all and fail-fast with native Oracle tasks. Its -report and Harbor artifacts are written under `/failure-policy/`. Explicit ID or category selection runs only -the selected PowerContext workloads. +The default run executes all acceptance tasks with collect-all, then reruns `batch:acceptance` with fail-fast. LoCoMo +stays in its own `batch:locomo`; the database decision and failure-policy tasks share `batch:acceptance`. Explicit +selection runs only the selected workloads with the requested failure policy. Selection uses the `acceptance` command's repeatable `--id` and `--category` options: diff --git a/e2e/bub/harbor-tasks/failure-policy-followup/steps/run/instruction.md b/e2e/bub/harbor-tasks/failure-policy-followup/steps/run/instruction.md index aa75293d1..ea0891f86 100644 --- a/e2e/bub/harbor-tasks/failure-policy-followup/steps/run/instruction.md +++ b/e2e/bub/harbor-tasks/failure-policy-followup/steps/run/instruction.md @@ -1 +1 @@ -Run the follow-up failure-policy fixture. +,touch /workspace/failure-policy-followup-ran diff --git a/e2e/bub/harbor-tasks/failure-policy-followup/task.toml b/e2e/bub/harbor-tasks/failure-policy-followup/task.toml index 28173512e..d92b5fae6 100644 --- a/e2e/bub/harbor-tasks/failure-policy-followup/task.toml +++ b/e2e/bub/harbor-tasks/failure-policy-followup/task.toml @@ -13,6 +13,16 @@ # limitations under the License. version = "1.3" +multi_step_reward_strategy = "final" + +[agent] +timeout_sec = 300.0 + +[verifier] +timeout_sec = 60.0 + +[environment] +build_timeout_sec = 300.0 [[steps]] name = "run" diff --git a/e2e/bub/harbor-tasks/failure-policy-timeout/steps/run/instruction.md b/e2e/bub/harbor-tasks/failure-policy-timeout/steps/run/instruction.md index 26f577d0d..618e7092f 100644 --- a/e2e/bub/harbor-tasks/failure-policy-timeout/steps/run/instruction.md +++ b/e2e/bub/harbor-tasks/failure-policy-timeout/steps/run/instruction.md @@ -1 +1 @@ -Run the timeout failure-policy fixture. +,sleep 5 diff --git a/e2e/bub/harbor-tasks/failure-policy-timeout/task.toml b/e2e/bub/harbor-tasks/failure-policy-timeout/task.toml index 51f8a2961..ec9c46a60 100644 --- a/e2e/bub/harbor-tasks/failure-policy-timeout/task.toml +++ b/e2e/bub/harbor-tasks/failure-policy-timeout/task.toml @@ -13,6 +13,16 @@ # limitations under the License. version = "1.3" +multi_step_reward_strategy = "final" + +[agent] +timeout_sec = 300.0 + +[verifier] +timeout_sec = 60.0 + +[environment] +build_timeout_sec = 300.0 [[steps]] name = "run" diff --git a/e2e/bub/harbor-tasks/locomo-multihop-football/task.toml b/e2e/bub/harbor-tasks/locomo-multihop-football/task.toml index 6d29fd658..cae761fe3 100644 --- a/e2e/bub/harbor-tasks/locomo-multihop-football/task.toml +++ b/e2e/bub/harbor-tasks/locomo-multihop-football/task.toml @@ -13,7 +13,7 @@ # limitations under the License. version = "1.3" -multi_step_reward_strategy = "mean" +multi_step_reward_strategy = "final" [metadata] sample = "locomo" diff --git a/e2e/bub/harbor-tasks/locomo-open-pastries/task.toml b/e2e/bub/harbor-tasks/locomo-open-pastries/task.toml index 6d29fd658..cae761fe3 100644 --- a/e2e/bub/harbor-tasks/locomo-open-pastries/task.toml +++ b/e2e/bub/harbor-tasks/locomo-open-pastries/task.toml @@ -13,7 +13,7 @@ # limitations under the License. version = "1.3" -multi_step_reward_strategy = "mean" +multi_step_reward_strategy = "final" [metadata] sample = "locomo" diff --git a/e2e/bub/harbor-tasks/locomo-support-group/task.toml b/e2e/bub/harbor-tasks/locomo-support-group/task.toml index 6d29fd658..cae761fe3 100644 --- a/e2e/bub/harbor-tasks/locomo-support-group/task.toml +++ b/e2e/bub/harbor-tasks/locomo-support-group/task.toml @@ -13,7 +13,7 @@ # limitations under the License. version = "1.3" -multi_step_reward_strategy = "mean" +multi_step_reward_strategy = "final" [metadata] sample = "locomo" diff --git a/e2e/bub/harbor-tasks/locomo-temporal-banker/task.toml b/e2e/bub/harbor-tasks/locomo-temporal-banker/task.toml index 6d29fd658..cae761fe3 100644 --- a/e2e/bub/harbor-tasks/locomo-temporal-banker/task.toml +++ b/e2e/bub/harbor-tasks/locomo-temporal-banker/task.toml @@ -13,7 +13,7 @@ # limitations under the License. version = "1.3" -multi_step_reward_strategy = "mean" +multi_step_reward_strategy = "final" [metadata] sample = "locomo" diff --git a/e2e/bub/harbor-tasks/project-database-decision/steps/capture/tests/test.sh b/e2e/bub/harbor-tasks/project-database-decision/steps/capture/tests/test.sh index ea6268777..fa405bd81 100644 --- a/e2e/bub/harbor-tasks/project-database-decision/steps/capture/tests/test.sh +++ b/e2e/bub/harbor-tasks/project-database-decision/steps/capture/tests/test.sh @@ -15,4 +15,8 @@ set -eu -echo 1 > /logs/verifier/reward.txt +if [ -e /logs/agent/powercontext-step-failed ]; then + echo 0 > /logs/verifier/reward.txt +else + echo 1 > /logs/verifier/reward.txt +fi diff --git a/e2e/bub/harbor-tasks/project-database-decision/steps/recall/tests/test.sh b/e2e/bub/harbor-tasks/project-database-decision/steps/recall/tests/test.sh index ea6268777..fa405bd81 100644 --- a/e2e/bub/harbor-tasks/project-database-decision/steps/recall/tests/test.sh +++ b/e2e/bub/harbor-tasks/project-database-decision/steps/recall/tests/test.sh @@ -15,4 +15,8 @@ set -eu -echo 1 > /logs/verifier/reward.txt +if [ -e /logs/agent/powercontext-step-failed ]; then + echo 0 > /logs/verifier/reward.txt +else + echo 1 > /logs/verifier/reward.txt +fi diff --git a/e2e/bub/src/powercontext_e2e/__main__.py b/e2e/bub/src/powercontext_e2e/__main__.py index 4fefe51fd..529683f95 100644 --- a/e2e/bub/src/powercontext_e2e/__main__.py +++ b/e2e/bub/src/powercontext_e2e/__main__.py @@ -19,36 +19,39 @@ import argparse import asyncio import sys -from collections.abc import Awaitable from pathlib import Path +from typing import TYPE_CHECKING from loguru import logger from .settings import HarnessSettings +if TYPE_CHECKING: + from .catalog import E2ETask + async def _run_default_acceptance( - workloads: Awaitable[bool], + tasks: tuple[E2ETask, ...], *, output_dir: Path, settings: HarnessSettings, ) -> bool: - from .failure_acceptance import run_failure_acceptance - - workloads_passed = await _accepted(workloads, "Memory workload acceptance") - policy_passed = await _accepted( - run_failure_acceptance(output_dir / "failure-policy", settings), - "Harbor failure-policy acceptance", + from .catalog import select_tasks + from .runner import run_tasks + + collect_all_passed = await run_tasks( + select_tasks(tasks, categories=("acceptance",)), + output_dir=output_dir, + settings=settings, + failure_policy="collect-all", ) - return workloads_passed and policy_passed - - -async def _accepted(result: Awaitable[bool], name: str) -> bool: - try: - return await result - except Exception: - logger.exception(f"{name} failed.") - return False + fail_fast_passed = await run_tasks( + select_tasks(tasks, categories=("batch:acceptance",)), + output_dir=output_dir / "fail-fast", + settings=settings, + failure_policy="fail-fast", + ) + return collect_all_passed and fail_fast_passed def main() -> None: @@ -97,18 +100,19 @@ def main() -> None: tasks = load_tasks(args.manifest) selected = select_tasks(tasks, ids=tuple(args.id), categories=tuple(args.category)) - workloads = run_tasks( - selected, - output_dir=args.output, - settings=settings, - failure_policy=args.failure_policy, - ) passed = ( - asyncio.run(workloads) + asyncio.run( + run_tasks( + selected, + output_dir=args.output, + settings=settings, + failure_policy=args.failure_policy, + ) + ) if args.id or args.category else asyncio.run( _run_default_acceptance( - workloads, + tasks, output_dir=args.output, settings=settings, ) diff --git a/e2e/bub/src/powercontext_e2e/catalog.py b/e2e/bub/src/powercontext_e2e/catalog.py index 613277304..8236f6be4 100644 --- a/e2e/bub/src/powercontext_e2e/catalog.py +++ b/e2e/bub/src/powercontext_e2e/catalog.py @@ -94,6 +94,15 @@ def require_unique_probe_ids(self) -> MemoryEvaluationSpec: return self +class ExpectedExecutionSpec(CatalogModel): + collect_all: Literal["completed", "failed", "skipped"] = Field(alias="collect-all") + fail_fast: Literal["completed", "failed", "skipped"] = Field(alias="fail-fast") + + +class OutcomeEvaluationSpec(CatalogModel): + expected_execution: ExpectedExecutionSpec + + class E2ETask(CatalogModel): schema_: Literal["powercontext.e2e-task/v1"] = Field(alias="schema") id: str = Field(pattern=r"^[a-z0-9][a-z0-9_-]*$") @@ -101,7 +110,7 @@ class E2ETask(CatalogModel): provenance: Provenance | None = None dataset: HarborDatasetSpec execution: BubExecutionSpec - evaluation: MemoryEvaluationSpec + evaluation: MemoryEvaluationSpec | OutcomeEvaluationSpec class TaskSelectionError(ValueError): diff --git a/e2e/bub/src/powercontext_e2e/evaluation.py b/e2e/bub/src/powercontext_e2e/evaluation.py index ff2208459..9a285969a 100644 --- a/e2e/bub/src/powercontext_e2e/evaluation.py +++ b/e2e/bub/src/powercontext_e2e/evaluation.py @@ -19,8 +19,8 @@ from pathlib import Path from typing import Literal +from .catalog import MemoryEvaluationSpec, OutcomeEvaluationSpec from .models import ( - SHARED_TRIAL_SKIPPED_ERROR, CaseEvaluation, EvaluationReport, EvaluationValue, @@ -29,6 +29,65 @@ ) CAPTURE_EVENTS = frozenset({"user_prompt", "llm_result", "tool_result"}) +FailurePolicy = Literal["fail-fast", "collect-all"] + + +def evaluate_observation( + observation: TaskObservation, + *, + experiment: str, + failure_policy: FailurePolicy, +) -> EvaluationReport: + evaluation = observation.task.evaluation + if isinstance(evaluation, OutcomeEvaluationSpec): + return _evaluate_execution( + observation, + evaluation, + experiment=experiment, + failure_policy=failure_policy, + ) + return MemoryEvaluator.evaluate(observation, experiment=experiment) + + +def _evaluate_execution( + observation: TaskObservation, + evaluation: OutcomeEvaluationSpec, + *, + experiment: str, + failure_policy: FailurePolicy, +) -> EvaluationReport: + expected = ( + evaluation.expected_execution.collect_all + if failure_policy == "collect-all" + else evaluation.expected_execution.fail_fast + ) + actual = observation.status + expected_reward = {"completed": 1, "failed": 0}.get(expected) + actual_reward = observation.harbor.rewards.get("reward") + reward_matches = expected == "skipped" or actual_reward == expected_reward + return EvaluationReport( + experiment=experiment, + cases=( + CaseEvaluation( + name=observation.task.id, + assertions={ + "execution_outcome": EvaluationValue( + value=actual == expected and reward_matches, + reason=( + f"Expected {expected!r}; observed {actual!r}." + if expected == "skipped" + else ( + f"Expected {expected!r} with Harbor reward {expected_reward!r}; " + f"observed {actual!r} with Harbor reward {actual_reward!r}." + ) + ), + ) + }, + labels={"task_outcome": EvaluationValue(value=actual)}, + attributes=_attributes(observation), + ), + ), + ) class MemoryEvaluator: @@ -38,7 +97,7 @@ class MemoryEvaluator: def evaluate(observation: TaskObservation, *, experiment: str) -> EvaluationReport: task = observation.task attributes = _attributes(observation) - if SHARED_TRIAL_SKIPPED_ERROR in observation.errors: + if observation.status == "skipped": return EvaluationReport( experiment=experiment, cases=( @@ -47,7 +106,7 @@ def evaluate(observation: TaskObservation, *, experiment: str) -> EvaluationRepo assertions={ "execution_completed": EvaluationValue( value=False, - reason=SHARED_TRIAL_SKIPPED_ERROR, + reason="Skipped after an earlier task stopped the shared Harbor trial.", ) }, labels={"task_outcome": EvaluationValue(value="skipped")}, @@ -56,6 +115,8 @@ def evaluate(observation: TaskObservation, *, experiment: str) -> EvaluationRepo ), ) evaluation = task.evaluation + if not isinstance(evaluation, MemoryEvaluationSpec): + raise TypeError("MemoryEvaluator requires a Memory evaluation task") # noqa: TRY003 eligible_records = [record for record in observation.capture_records if record.event in CAPTURE_EVENTS] captured_records = [record for record in eligible_records if record.status == "captured"] capture_coverage = len(captured_records) / len(eligible_records) if eligible_records else 0.0 @@ -107,6 +168,7 @@ def evaluate(observation: TaskObservation, *, experiment: str) -> EvaluationRepo any(fragment.casefold() in entry.text.casefold() for entry in observation.memory_after.entries) for fragment in evaluation.expected_memory ) + observed_checksum = observation.harbor.source_task_checksum or observation.harbor.task_checksum thresholds = evaluation.thresholds metrics = { @@ -126,8 +188,8 @@ def evaluate(observation: TaskObservation, *, experiment: str) -> EvaluationRepo reason=None if observation.status == "completed" else "; ".join(observation.errors), ), "task_provenance_matches": EvaluationValue( - value=observation.harbor.task_checksum == task.dataset.checksum, - reason=f"Expected {task.dataset.checksum!r}; observed {observation.harbor.task_checksum!r}.", + value=observed_checksum == task.dataset.checksum, + reason=(f"Expected {task.dataset.checksum!r}; observed {observed_checksum!r}."), ), "native_acp_evidence_recorded": EvaluationValue( value=not missing_native_artifacts, @@ -216,7 +278,9 @@ def _attributes(observation: TaskObservation) -> dict[str, str]: } -def _task_outcome(harbor: HarborTrialObservation, status: Literal["completed", "failed"]) -> str: +def _task_outcome(harbor: HarborTrialObservation, status: Literal["completed", "failed", "skipped"]) -> str: + if status == "skipped": + return "skipped" if harbor.exception_type is not None: return f"error:{harbor.exception_type}" if status == "failed": diff --git a/e2e/bub/src/powercontext_e2e/failure_acceptance.py b/e2e/bub/src/powercontext_e2e/failure_acceptance.py deleted file mode 100644 index dced6e5a0..000000000 --- a/e2e/bub/src/powercontext_e2e/failure_acceptance.py +++ /dev/null @@ -1,141 +0,0 @@ -# Copyright (c) 2026 OceanBase. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Verify Harbor collect-all and fail-fast behavior with real trials.""" - -from __future__ import annotations - -from pathlib import Path -from uuid import uuid4 - -from harbor.job import Job -from harbor.models.environment_type import EnvironmentType -from harbor.models.job.config import JobConfig -from harbor.models.task.task import Task as HarborTask -from harbor.models.trial.config import AgentConfig, EnvironmentConfig, ResourceMode, TaskConfig -from harbor.models.trial.result import StepResult - -from .catalog import E2ETask -from .evidence import redact, write_evaluation_report, write_evidence -from .models import CaseEvaluation, EvaluationReport, EvaluationValue -from .report import render_evaluation_summary -from .runner import FailurePolicy, prepare_runtime_task -from .settings import HarnessSettings - -DATASET_PATH = Path("e2e/bub/harbor-tasks") -TASK_IDS = ("failure-policy-timeout", "failure-policy-followup") - - -async def run_failure_acceptance(output_dir: Path, settings: HarnessSettings) -> bool: - """Run both policies and write one acceptance report.""" - - cases = [] - for policy in ("collect-all", "fail-fast"): - try: - steps = await _run_policy(policy, output_dir / policy, settings) - failed = steps.get("failure-policy-timeout-run") - followup = steps.get("failure-policy-followup-run") - failure_ok = failed is not None and failed.exception_info is not None and _reward(failed) == 0 - followup_ok = ( - followup is not None and followup.exception_info is None and _reward(followup) == 1 - if policy == "collect-all" - else followup is None - ) - failure_reason = _step_summary(failed) - followup_reason = _step_summary(followup) - except Exception as exc: - failure_ok = followup_ok = False - failure_reason = followup_reason = redact(f"{type(exc).__name__}: {exc}", settings) - - followup_name = "followup_executed" if policy == "collect-all" else "followup_skipped" - accepted = failure_ok and followup_ok - cases.append( - CaseEvaluation( - name=policy, - assertions={ - "failure_observed": EvaluationValue(value=failure_ok, reason=failure_reason), - followup_name: EvaluationValue(value=followup_ok, reason=followup_reason), - }, - labels={"task_outcome": EvaluationValue(value="passed" if accepted else "not_passed")}, - ) - ) - - report = EvaluationReport(experiment="e2e:failure-policy", cases=tuple(cases)) - output_dir.mkdir(parents=True, exist_ok=True) - write_evaluation_report(output_dir / "eval-report.json", report=report, settings=settings) - write_evidence( - output_dir / "report.md", - render_evaluation_summary(report, title="PowerContext Harbor failure-policy acceptance"), - settings, - ) - return report.accepted - - -async def _run_policy( - policy: FailurePolicy, - output_dir: Path, - settings: HarnessSettings, -) -> dict[str, StepResult]: - runtime_id = f"failure-policy-{policy}-{uuid4().hex[:12]}" - runtime = prepare_runtime_task( - _source_tasks(settings), - output_dir=output_dir, - settings=settings, - failure_policy=policy, - runtime_id=runtime_id, - ) - result = await (await Job.create(_job_config(runtime_id, output_dir, runtime.task_config))).run() - if len(result.trial_results) != 1: - return {} - return {step.step_name: step for step in result.trial_results[0].step_results or ()} - - -def _source_tasks(settings: HarnessSettings) -> tuple[E2ETask, ...]: - root = settings.repository_path() / DATASET_PATH - return tuple( - E2ETask.model_validate({ - "schema": "powercontext.e2e-task/v1", - "id": task_id, - "categories": ["batch:failure-policy"], - "dataset": {"path": str(DATASET_PATH), "task_id": task_id, "checksum": HarborTask(root / task_id).checksum}, - "execution": {"type": "bub", "model": False}, - "evaluation": {"probes": [{"id": "policy-fixture", "query": "Run the policy fixture."}]}, - }) - for task_id in TASK_IDS - ) - - -def _job_config(runtime_id: str, output_dir: Path, task: TaskConfig) -> JobConfig: - return JobConfig( - job_name=runtime_id, - jobs_dir=output_dir / "harbor-jobs", - quiet=True, - environment=EnvironmentConfig( - type=EnvironmentType.DOCKER, - cpu_enforcement_policy=ResourceMode.IGNORE, - memory_enforcement_policy=ResourceMode.IGNORE, - ), - agents=[AgentConfig(name="oracle")], - tasks=[task], - ) - - -def _reward(step: StepResult) -> float | int | None: - return None if step.verifier_result is None else (step.verifier_result.rewards or {}).get("reward") - - -def _step_summary(step: StepResult | None) -> str: - if step is None: - return "The step was not executed." - return f"Agent exception: {step.exception_info is not None}; verifier reward: {_reward(step)!r}." diff --git a/e2e/bub/src/powercontext_e2e/harbor_agent.py b/e2e/bub/src/powercontext_e2e/harbor_agent.py index 6934c77ac..4453aac09 100644 --- a/e2e/bub/src/powercontext_e2e/harbor_agent.py +++ b/e2e/bub/src/powercontext_e2e/harbor_agent.py @@ -62,21 +62,19 @@ def __init__(self, **kwargs: Any) -> None: @override async def run(self, instruction: str, environment: BaseEnvironment, context: AgentContext) -> None: try: - await environment.exec(command=f"rm -f {STEP_FAILURE_MARKER}") + await environment.exec(command=f"touch {STEP_FAILURE_MARKER}") if not self._invocation_scopes: await super().run(instruction, environment, context) else: if self._step_index >= len(self._invocation_scopes): invocation = self._step_index + 1 - raise RuntimeError( # noqa: TRY003, TRY301 + raise RuntimeError( # noqa: TRY003 f"No E2E scope configured for agent invocation {invocation}" ) scope_id = self._invocation_scopes[self._step_index] with environment.scoped_exec_env({"POWERCONTEXT_BUB_SCOPE_ID": scope_id}): await super().run(instruction, environment, context) - except Exception: - await environment.exec(command=f"touch {STEP_FAILURE_MARKER}") - raise + await environment.exec(command=f"rm -f {STEP_FAILURE_MARKER}") finally: self._step_index += 1 diff --git a/e2e/bub/src/powercontext_e2e/models.py b/e2e/bub/src/powercontext_e2e/models.py index a9f4264b7..dc18f8bbc 100644 --- a/e2e/bub/src/powercontext_e2e/models.py +++ b/e2e/bub/src/powercontext_e2e/models.py @@ -23,8 +23,6 @@ from .catalog import E2ETask -SHARED_TRIAL_SKIPPED_ERROR = "Skipped after an earlier task stopped the shared Harbor trial." - class EvidenceModel(BaseModel): model_config = ConfigDict(extra="forbid") @@ -96,6 +94,7 @@ class HarborTrialObservation(EvidenceModel): trial_name: str | None = None trial_uri: str | None = None task_checksum: str | None = None + source_task_checksum: str | None = None rewards: dict[str, float | int] = Field(default_factory=dict) exception_type: str | None = None exception_message: str | None = None @@ -124,7 +123,7 @@ class TaskObservation(EvidenceModel): run_id: str environment: RunEnvironment task: E2ETask - status: Literal["completed", "failed"] + status: Literal["completed", "failed", "skipped"] errors: tuple[str, ...] = () harbor: HarborTrialObservation capture_records: tuple[CaptureRecord, ...] = () diff --git a/e2e/bub/src/powercontext_e2e/report.py b/e2e/bub/src/powercontext_e2e/report.py index efe14cdff..bb2b3a3a5 100644 --- a/e2e/bub/src/powercontext_e2e/report.py +++ b/e2e/bub/src/powercontext_e2e/report.py @@ -27,7 +27,7 @@ def render_report(observation: TaskObservation, report: EvaluationReport) -> str markdown = Markdown(renderer=MarkdownRenderer) document = block.Document() children: list[Element] = [] - children.extend(_nodes(markdown, "# PowerContext end-to-end Memory evaluation")) + children.extend(_nodes(markdown, "# PowerContext end-to-end evaluation")) children.append(block.BlankLine(0)) children.extend( _nodes( @@ -50,22 +50,18 @@ def render_report(observation: TaskObservation, report: EvaluationReport) -> str ) children.extend(_nodes(markdown, reward_lines)) children.append(block.BlankLine(0)) - children.extend(_nodes(markdown, "## Memory evaluation")) + children.extend(_nodes(markdown, "## Evaluation")) children.append(block.BlankLine(0)) children.extend(_nodes(markdown, f"```text\n{_evaluation_text(report)}\n```")) document.children = children return markdown.render(document) -def render_evaluation_summary( - report: EvaluationReport, - *, - title: str = "PowerContext end-to-end Memory evaluation", -) -> str: +def render_evaluation_summary(report: EvaluationReport) -> str: markdown = Markdown(renderer=MarkdownRenderer) document = block.Document() document.children = [ - *_nodes(markdown, f"# {title}"), + *_nodes(markdown, "# PowerContext end-to-end evaluation"), block.BlankLine(0), *_nodes(markdown, f"```text\n{_evaluation_text(report)}\n```"), ] diff --git a/e2e/bub/src/powercontext_e2e/runner.py b/e2e/bub/src/powercontext_e2e/runner.py index dcfd07dad..c4d930de3 100644 --- a/e2e/bub/src/powercontext_e2e/runner.py +++ b/e2e/bub/src/powercontext_e2e/runner.py @@ -18,6 +18,7 @@ import re import shutil +from collections.abc import Mapping from datetime import UTC, datetime from pathlib import Path, PurePosixPath from typing import Any, Literal, NamedTuple @@ -25,21 +26,15 @@ from uuid import uuid4 from dirhash import dirhash +from harbor.environments.definition import environment_content_hash from harbor.job import Job from harbor.models.environment_type import EnvironmentType from harbor.models.job.config import DatasetConfig, JobConfig from harbor.models.task.config import MultiStepRewardStrategy, TaskConfig from harbor.models.task.paths import TaskPaths from harbor.models.task.task import Task as HarborTask -from harbor.models.trial.config import ( - AgentConfig, - EnvironmentConfig, - ResourceMode, - ServiceVolumeConfig, -) -from harbor.models.trial.config import ( - TaskConfig as HarborTrialTaskConfig, -) +from harbor.models.trial.config import AgentConfig, EnvironmentConfig, ResourceMode, ServiceVolumeConfig +from harbor.models.trial.config import TaskConfig as HarborTrialTaskConfig from harbor.models.trial.paths import TrialPaths from harbor.models.trial.result import StepResult from powercontext.client import PowerContextClient @@ -47,12 +42,11 @@ from powercontext.http import ListMemoryEntriesRequest, PrepareContextRequest from .artifacts import write_artifacts -from .catalog import E2ETask -from .evaluation import MemoryEvaluator +from .catalog import E2ETask, MemoryEvaluationSpec, OutcomeEvaluationSpec +from .evaluation import evaluate_observation from .evidence import fingerprint, load_resolved_instructions, redact, write_evaluation_report, write_evidence from .harbor_agent import BUB_ACP_SERVER_VERSION, BUB_VERSION from .models import ( - SHARED_TRIAL_SKIPPED_ERROR, CaptureRecord, EvaluationReport, HarborTrialObservation, @@ -76,6 +70,7 @@ ) FailurePolicy = Literal["fail-fast", "collect-all"] +TaskStatus = Literal["completed", "failed", "skipped"] BATCH_CATEGORY_PREFIX = "batch:" BATCH_NAME_PATTERN = re.compile(r"^[a-z0-9][a-z0-9_-]*$") @@ -110,13 +105,7 @@ class PreparedRuntime(NamedTuple): sources: tuple[SourceTask, ...] -class SourceResult(NamedTuple): - status: Literal["completed", "failed", "skipped"] - errors: tuple[str, ...] - steps: tuple[StepResult, ...] - - -class PreparedTask(NamedTuple): +class TaskRun(NamedTuple): task: E2ETask run_id: str scope_id: str @@ -124,18 +113,6 @@ class PreparedTask(NamedTuple): memory_before: MemorySnapshot -async def evaluate_task( - task: E2ETask, - *, - output_dir: Path, - settings: HarnessSettings, -) -> bool: - observation = await run_task(task, output_dir=output_dir, settings=settings) - report = MemoryEvaluator.evaluate(observation, experiment=f"e2e:{task.id}") - write_artifacts(observation, report, output_dir, settings=settings) - return report.accepted - - async def evaluate_tasks( tasks: tuple[E2ETask, ...], *, @@ -143,31 +120,34 @@ async def evaluate_tasks( settings: HarnessSettings, failure_policy: FailurePolicy = "collect-all", ) -> bool: - if len(tasks) == 1: - return await evaluate_task(tasks[0], output_dir=output_dir, settings=settings) observations = await run_task_group( tasks, output_dir=output_dir, settings=settings, failure_policy=failure_policy, ) - experiment = f"e2e:batch:{_task_batch(tasks[0])}" reports = tuple( - MemoryEvaluator.evaluate(observation, experiment=f"e2e:{observation.task.id}") for observation in observations + evaluate_observation( + observation, + experiment=f"e2e:{observation.task.id}", + failure_policy=failure_policy, + ) + for observation in observations ) + if len(observations) == 1: + write_artifacts(observations[0], reports[0], output_dir, settings=settings) + return reports[0].accepted + for observation, report in zip(observations, reports, strict=True): write_artifacts(observation, report, output_dir / "tasks" / observation.task.id, settings=settings) aggregate = EvaluationReport( - experiment=experiment, cases=tuple(case for report in reports for case in report.cases) + experiment=f"e2e:batch:{_task_batch(tasks[0])}", + cases=tuple(case for report in reports for case in report.cases), ) - _write_batch_summary(aggregate, output_dir, settings) - return aggregate.accepted - - -def _write_batch_summary(report: EvaluationReport, output_dir: Path, settings: HarnessSettings) -> None: output_dir.mkdir(parents=True, exist_ok=True) - write_evaluation_report(output_dir / "eval-report.json", report=report, settings=settings) - write_evidence(output_dir / "report.md", render_evaluation_summary(report), settings) + write_evaluation_report(output_dir / "eval-report.json", report=aggregate, settings=settings) + write_evidence(output_dir / "report.md", render_evaluation_summary(aggregate), settings) + return aggregate.accepted async def run_tasks( @@ -183,36 +163,31 @@ async def run_tasks( accepted = True for group in group_tasks(tasks): - group_accepted = await evaluate_tasks( + accepted &= await evaluate_tasks( group.tasks, output_dir=output_dir / group.output_id, settings=settings, failure_policy=failure_policy, ) - accepted = group_accepted and accepted return accepted def group_tasks(tasks: tuple[E2ETask, ...]) -> tuple[ExecutionGroup, ...]: - """Group only selected tasks that share an explicit batch category.""" + """Group selected tasks only when they share an explicit batch category.""" groups: list[ExecutionGroup] = [] positions: dict[str, int] = {} for task in tasks: batch = _task_batch(task) if batch is None: - groups.append(ExecutionGroup(tasks=(task,), batch=None)) - continue - if batch not in positions: + groups.append(ExecutionGroup((task,), None)) + elif batch not in positions: positions[batch] = len(groups) - groups.append(ExecutionGroup(tasks=(task,), batch=batch)) - continue - index = positions[batch] - group = groups[index] - groups[index] = ExecutionGroup(tasks=(*group.tasks, task), batch=group.batch) - return tuple( - ExecutionGroup(tasks=group.tasks, batch=group.batch if len(group.tasks) > 1 else None) for group in groups - ) + groups.append(ExecutionGroup((task,), batch)) + else: + index = positions[batch] + groups[index] = ExecutionGroup((*groups[index].tasks, task), batch) + return tuple(ExecutionGroup(group.tasks, group.batch if len(group.tasks) > 1 else None) for group in groups) def _task_batch(task: E2ETask) -> str | None: @@ -226,45 +201,6 @@ def _task_batch(task: E2ETask) -> str | None: return batches[0] if batches else None -def _powercontext_client() -> PowerContextClient: - settings = ClientSettings() - token = None if settings.api_token is None else settings.api_token.get_secret_value() - return PowerContextClient(settings.server_url, token=token, timeout=settings.timeout) - - -async def run_task( - task: E2ETask, - *, - output_dir: Path, - settings: HarnessSettings, -) -> TaskObservation: - run_id = f"{task.id}-{uuid4().hex[:12]}" - scope_id = f"e2e:{run_id}" - prepared = PreparedTask(task, run_id, scope_id, datetime.now(UTC), MemorySnapshot()) - errors: list[str] = [] - artifacts = TaskArtifacts() - harbor = HarborTrialObservation() - execution_status: Literal["completed", "failed"] = "failed" - - async with _powercontext_client() as client: - try: - await client.get_readiness() - prepared = prepared._replace(memory_before=await memory_snapshot(client, scope_id)) - output_dir.mkdir(parents=True, exist_ok=True) - job = await Job.create(_job_config(task, run_id, scope_id, output_dir, settings)) - result = await job.run() - harbor, _, trial_dir = _harbor_observation(result, settings) - if harbor.exception_type is not None: - errors.append(f"{harbor.exception_type}: {harbor.exception_message or ''}".strip()) - else: - execution_status = "completed" - if trial_dir is not None: - artifacts = _load_task_artifacts(trial_dir, task.execution.native_artifact_names, settings) - except Exception as exc: - errors.append(redact(f"{type(exc).__name__}: {exc}", settings)) - return await _finalize_task(client, prepared, execution_status, tuple(errors), harbor, artifacts, settings) - - async def run_task_group( tasks: tuple[E2ETask, ...], *, @@ -272,79 +208,113 @@ async def run_task_group( settings: HarnessSettings, failure_policy: FailurePolicy = "collect-all", ) -> tuple[TaskObservation, ...]: - if len(tasks) < 2: - raise ValueError("A runtime batch requires at least two E2E tasks") # noqa: TRY003 - batch = _task_batch(tasks[0]) - run_id = f"batch-{batch}-{uuid4().hex[:12]}" - prepared = prepare_runtime_task( - tasks, - output_dir=output_dir, - settings=settings, - failure_policy=failure_policy, - runtime_id=run_id, + if not tasks: + raise ValueError("At least one E2E task is required") # noqa: TRY003 + + runtime = ( + prepare_runtime_task(tasks, output_dir=output_dir, settings=settings, failure_policy=failure_policy) + if len(tasks) > 1 + else None + ) + run_id = _run_id(tasks) + scopes = {task.id: f"e2e:{run_id}:{task.id}" if runtime else f"e2e:{run_id}" for task in tasks} + runs = {task.id: TaskRun(task, run_id, scopes[task.id], datetime.now(UTC), MemorySnapshot()) for task in tasks} + invocation_scopes = ( + tuple(scopes[source.task.id] for source in runtime.sources for _ in source.runtime_steps) if runtime else () + ) + job_config = _job_config( + tasks[0], + run_id, + scopes[tasks[0].id], + output_dir, + settings, + runtime=runtime, + invocation_scopes=invocation_scopes, ) - task_scopes = {task.id: f"e2e:{run_id}:{task.id}" for task in tasks} - prepared_tasks = { - task.id: PreparedTask(task, run_id, task_scopes[task.id], datetime.now(UTC), MemorySnapshot()) for task in tasks - } - invocation_scopes = tuple(task_scopes[source.task.id] for source in prepared.sources for _ in source.runtime_steps) harbor = HarborTrialObservation() - step_results: tuple[Any, ...] = () + step_results: tuple[StepResult, ...] = () trial_dir: Path | None = None execution_errors: list[str] = [] async with _powercontext_client() as client: try: - await client.get_readiness() - for task in tasks: - prepared_task = prepared_tasks[task.id] - prepared_tasks[task.id] = prepared_task._replace( - memory_before=await memory_snapshot(client, prepared_task.scope_id) - ) - job = await Job.create( - _batch_job_config( - tasks[0], - run_id, - invocation_scopes, - output_dir, - settings, - task_config=prepared.task_config, - ) - ) - result = await job.run() - harbor, step_results, trial_dir = _harbor_observation(result, settings) + memory_tasks = tuple(task for task in tasks if isinstance(task.evaluation, MemoryEvaluationSpec)) + if memory_tasks: + await client.get_readiness() + for task in memory_tasks: + run = runs[task.id] + runs[task.id] = run._replace(memory_before=await memory_snapshot(client, run.scope_id)) + + output_dir.mkdir(parents=True, exist_ok=True) + harbor, step_results, trial_dir = _harbor_observation(await (await Job.create(job_config)).run(), settings) if harbor.exception_type is not None: execution_errors.append(f"{harbor.exception_type}: {harbor.exception_message or ''}".strip()) except Exception as exc: execution_errors.append(redact(f"{type(exc).__name__}: {exc}", settings)) - source_results = _source_results(prepared.sources, step_results, tuple(execution_errors)) + if runtime is None: + artifacts = _collect_task_artifacts(trial_dir, tasks[0], settings, errors=execution_errors) + reward_failed = _outcome_reward_failed(tasks[0], harbor.rewards) + status: TaskStatus = "failed" if execution_errors or reward_failed else "completed" + return ( + await _finalize_task( + client, + runs[tasks[0].id], + status, + tuple(execution_errors), + harbor, + artifacts, + settings, + ), + ) + + error_owner = next( + ( + source.task.id + for source in reversed(runtime.sources) + if any(step.step_name in source.runtime_steps for step in step_results) + ), + runtime.sources[0].task.id, + ) observations: list[TaskObservation] = [] - for source in prepared.sources: - task = source.task - source_result = source_results[task.id] - artifacts = TaskArtifacts() - if trial_dir is not None and source_result.status != "skipped": - try: - artifacts = _load_task_artifacts( - trial_dir, - task.execution.native_artifact_names, - settings, - step_names=source.runtime_steps, - ) - except Exception as exc: - source_result = source_result._replace( - status="failed", - errors=(*source_result.errors, redact(f"{type(exc).__name__}: {exc}", settings)), - ) - source_harbor = _source_harbor_observation(harbor, source, source_result, settings) + for source in runtime.sources: + owned = tuple(step for step in step_results if step.step_name in source.runtime_steps) + missing = [step for step in source.runtime_steps if step not in {result.step_name for result in owned}] + verifier = owned[-1].verifier_result if owned else None + reward_failed = _outcome_reward_failed( + source.task, + None if verifier is None else verifier.rewards, + ) + status: TaskStatus = ( + "skipped" + if not owned and source.task.id != error_owner + else "failed" + if missing + or any(step.exception_info is not None for step in owned) + or reward_failed + or (source.task.id == error_owner and bool(execution_errors)) + else "completed" + ) + errors = list(execution_errors) if source.task.id == error_owner else [] + if missing and owned: + errors.append(f"Harbor did not execute steps: {missing!r}") + artifacts = _collect_task_artifacts( + trial_dir, + source.task, + settings, + errors=errors, + step_names=source.runtime_steps if status != "skipped" else (), + whole_trial=False, + ) + if errors: + status = "failed" observations.append( await _finalize_task( client, - prepared_tasks[task.id], - source_result.status, - source_result.errors, - source_harbor, + runs[source.task.id], + status, + tuple(errors), + _source_harbor_observation(harbor, source, status, owned, settings), artifacts, settings, ) @@ -352,10 +322,26 @@ async def run_task_group( return tuple(observations) +def _run_id(tasks: tuple[E2ETask, ...]) -> str: + name = tasks[0].id if len(tasks) == 1 else f"batch-{_task_batch(tasks[0])}" + return f"{name}-{uuid4().hex[:12]}" + + +def _outcome_reward_failed(task: E2ETask, rewards: Mapping[str, float | int] | None) -> bool: + reward = None if rewards is None else rewards.get("reward") + return isinstance(task.evaluation, OutcomeEvaluationSpec) and reward is not None and float(reward) < 1 + + +def _powercontext_client() -> PowerContextClient: + settings = ClientSettings() + token = None if settings.api_token is None else settings.api_token.get_secret_value() + return PowerContextClient(settings.server_url, token=token, timeout=settings.timeout) + + async def _finalize_task( client: PowerContextClient, - prepared: PreparedTask, - execution_status: Literal["completed", "failed", "skipped"], + run: TaskRun, + status: TaskStatus, errors: tuple[str, ...], harbor: HarborTrialObservation, artifacts: TaskArtifacts, @@ -364,121 +350,51 @@ async def _finalize_task( final_errors = list(errors) memory_after = MemorySnapshot() probes: tuple[RecallProbeObservation, ...] = () - if execution_status != "skipped": - try: - memory_after = await memory_snapshot(client, prepared.scope_id) - except Exception as exc: - final_errors.append(redact(f"{type(exc).__name__}: {exc}", settings)) + if status != "skipped" and isinstance(run.task.evaluation, MemoryEvaluationSpec): try: - probes = await _prepared_probes(client, prepared.task, prepared.scope_id) + memory_after = await memory_snapshot(client, run.scope_id) + probes = await _prepared_probes(client, run.task.evaluation, run.scope_id) except Exception as exc: final_errors.append(redact(f"{type(exc).__name__}: {exc}", settings)) + if final_errors and status == "completed": + status = "failed" return TaskObservation( - run_id=prepared.run_id, - environment=_run_environment(prepared.task, prepared.started_at, settings), - task=prepared.task, - status="completed" if execution_status == "completed" and not final_errors else "failed", + run_id=run.run_id, + environment=_run_environment(run.task, run.started_at, settings), + task=run.task, + status=status, errors=tuple(final_errors), harbor=harbor, capture_records=artifacts.capture_records, native_artifacts=artifacts.native_artifacts, resolved_instructions=artifacts.resolved_instructions, - memory_before=prepared.memory_before, + memory_before=run.memory_before, memory_after=memory_after, probes=probes, ) -def _source_results( - sources: tuple[SourceTask, ...], - step_results: tuple[StepResult, ...], - execution_errors: tuple[str, ...], -) -> dict[str, SourceResult]: - results: dict[str, SourceResult] = {} - for index, source in enumerate(sources): - owned = tuple(step for step in step_results if step.step_name in source.runtime_steps) - executed = {step.step_name for step in owned} - missing = [name for name in source.runtime_steps if name not in executed] - if not owned: - status: Literal["completed", "failed", "skipped"] = "failed" if index == 0 else "skipped" - errors = ("Harbor did not execute any steps.",) if index == 0 else (SHARED_TRIAL_SKIPPED_ERROR,) - elif missing or any(step.exception_info is not None for step in owned): - status = "failed" - errors = (f"Harbor did not execute steps: {missing!r}",) if missing else () - else: - status = "completed" - errors = () - results[source.task.id] = SourceResult(status, errors, owned) - - if execution_errors: - source = next( - (source for source in reversed(sources) if results[source.task.id].status != "skipped"), - sources[0], - ) - result = results[source.task.id] - results[source.task.id] = SourceResult("failed", (*result.errors, *execution_errors), result.steps) - return results - - def _source_harbor_observation( shared: HarborTrialObservation, source: SourceTask, - result: SourceResult, + status: TaskStatus, + steps: tuple[StepResult, ...], settings: HarnessSettings, ) -> HarborTrialObservation: - step_exception = next((step.exception_info for step in result.steps if step.exception_info is not None), None) - exception_type = None if step_exception is None else step_exception.exception_type - exception_message = None if step_exception is None else redact(step_exception.exception_message or "", settings) - if result.status == "failed" and step_exception is None and shared.exception_type is not None: - exception_type = shared.exception_type - exception_message = shared.exception_message - return HarborTrialObservation( - job_id=shared.job_id, - trial_name=shared.trial_name, - trial_uri=shared.trial_uri, - task_checksum=source.harbor_task.checksum, - rewards=_source_rewards(source, result.steps), - exception_type=exception_type, - exception_message=exception_message, - started_at=shared.started_at, - finished_at=shared.finished_at, - ) - - -def _source_rewards(source: SourceTask, steps: tuple[StepResult, ...]) -> dict[str, float | int]: - strategy = source.harbor_task.config.multi_step_reward_strategy - if strategy is MultiStepRewardStrategy.FINAL: - verifier = steps[-1].verifier_result if steps else None - return dict(verifier.rewards or {}) if verifier is not None else {} - - rewards = [step.verifier_result.rewards or {} for step in steps if step.verifier_result is not None] - keys = {key for result in rewards for key in result} - if not rewards or not keys: - return {} - return {key: sum(result.get(key, 0) for result in rewards) / len(rewards) for key in keys} - - -def _batch_job_config( - task: E2ETask, - job_name: str, - invocation_scopes: tuple[str, ...], - output_dir: Path, - settings: HarnessSettings, - *, - task_config: HarborTrialTaskConfig, -) -> JobConfig: - config = _job_config(task, job_name, invocation_scopes[0], output_dir, settings) - agent = config.agents[0] - env = dict(agent.env) - env.pop("POWERCONTEXT_BUB_SCOPE_ID") - return config.model_copy( - update={ - "agents": [agent.model_copy(update={"env": env, "kwargs": {"invocation_scopes": invocation_scopes}})], - "datasets": [], - "tasks": [task_config], - } - ) + step_exception = next((step.exception_info for step in steps if step.exception_info is not None), None) + verifier = steps[-1].verifier_result if steps else None + updates: dict[str, Any] = { + "source_task_checksum": source.harbor_task.checksum, + "rewards": dict(verifier.rewards or {}) if verifier is not None else {}, + "exception_type": None if step_exception is None else step_exception.exception_type, + "exception_message": ( + None if step_exception is None else redact(step_exception.exception_message or "", settings) + ), + } + if status == "failed" and step_exception is None and shared.exception_type is not None: + updates.update(exception_type=shared.exception_type, exception_message=shared.exception_message) + return shared.model_copy(update=updates) def _job_config( @@ -487,6 +403,9 @@ def _job_config( scope_id: str, output_dir: Path, settings: HarnessSettings, + *, + runtime: PreparedRuntime | None = None, + invocation_scopes: tuple[str, ...] = (), ) -> JobConfig: repository = settings.repository_path() mounts: list[ServiceVolumeConfig] = [ @@ -507,6 +426,7 @@ def _job_config( "bind": {"create_host_path": False}, }) + evaluation = task.evaluation agent_env = powercontext_bub_environment() if task.execution.model: agent_env.update(bub_environment()) @@ -517,10 +437,16 @@ def _job_config( "BUB_MAX_STEPS": str(task.execution.max_steps), "BUB_MAX_TOKENS": str(task.execution.max_tokens), "CODEX_HOME": "/installed-agent/codex", - "POWERCONTEXT_BUB_CAPTURE_CHECKPOINT_EVERY": str(task.evaluation.checkpoint_every_events), - "POWERCONTEXT_BUB_CAPTURE_EVENTS": str(task.evaluation.capture_events).lower(), + "POWERCONTEXT_BUB_CAPTURE_CHECKPOINT_EVERY": str( + evaluation.checkpoint_every_events if isinstance(evaluation, MemoryEvaluationSpec) else 5 + ), + "POWERCONTEXT_BUB_CAPTURE_EVENTS": str( + evaluation.capture_events if isinstance(evaluation, MemoryEvaluationSpec) else False + ).lower(), "POWERCONTEXT_BUB_CAPTURE_LOG": "/logs/agent/powercontext-capture.jsonl", - "POWERCONTEXT_BUB_CAPTURE_MAX_BYTES": str(task.evaluation.max_event_bytes), + "POWERCONTEXT_BUB_CAPTURE_MAX_BYTES": str( + evaluation.max_event_bytes if isinstance(evaluation, MemoryEvaluationSpec) else 8192 + ), "POWERCONTEXT_BUB_SCOPE_ID": scope_id, }) if settings.agent_proxy_url is not None: @@ -533,6 +459,10 @@ def _job_config( "https_proxy": proxy_url, "no_proxy": "127.0.0.1,localhost,host-gateway,powercontext", }) + agent_kwargs: dict[str, Any] = {} + if runtime is not None: + agent_env.pop("POWERCONTEXT_BUB_SCOPE_ID") + agent_kwargs["invocation_scopes"] = invocation_scopes return JobConfig( job_name=run_id, @@ -552,9 +482,11 @@ def _job_config( AgentConfig( import_path="powercontext_e2e.harbor_agent:PowerContextBubAcpAgent", env=agent_env, + kwargs=agent_kwargs, ) ], - datasets=[_dataset_config(task, repository)], + datasets=[] if runtime else [_dataset_config(task, repository)], + tasks=[runtime.task_config] if runtime else [], ) @@ -571,23 +503,16 @@ def prepare_runtime_task( output_dir: Path, settings: HarnessSettings, failure_policy: FailurePolicy, - runtime_id: str | None = None, ) -> PreparedRuntime: - """Assemble compatible selected source tasks into one run-local Harbor task.""" - - if len(tasks) < 2: - raise ValueError("Runtime aggregation requires at least two tasks") # noqa: TRY003 + """Assemble compatible source tasks into one run-local Harbor task.""" sources = _validate_batch_compatibility(tasks, settings) batch = _task_batch(tasks[0]) - if batch is None: - raise ValueError("Runtime aggregation requires an explicit batch") # noqa: TRY003 + if len(tasks) < 2 or batch is None: + raise ValueError("Runtime aggregation requires at least two tasks in one explicit batch") # noqa: TRY003 - runtime_task_id = runtime_id or f"batch-{batch}-{uuid4().hex[:12]}" - runtime_root = output_dir / "harbor-runtime-dataset" - runtime_paths = TaskPaths(runtime_root / runtime_task_id) + runtime_paths = TaskPaths(output_dir / "harbor-runtime-dataset" / f"batch-{batch}-{uuid4().hex[:12]}") runtime_paths.task_dir.mkdir(parents=True) - first_paths = sources[0].harbor_task.paths for source_dir, target_dir in ( (first_paths.environment_dir, runtime_paths.environment_dir), @@ -598,36 +523,33 @@ def prepare_runtime_task( runtime_paths.steps_dir.mkdir() for source in sources: for source_step, runtime_step in zip(source.source_steps, source.runtime_steps, strict=True): - shutil.copytree( - source.harbor_task.paths.step_dir(source_step), - runtime_paths.step_dir(runtime_step), - ) - - runtime_config = _runtime_task_config(sources, failure_policy) - runtime_paths.config_path.write_text(runtime_config.model_dump_toml(), encoding="utf-8") + shutil.copytree(source.harbor_task.paths.step_dir(source_step), runtime_paths.step_dir(runtime_step)) + config = _runtime_task_config(sources, failure_policy) + runtime_paths.config_path.write_text(config.model_dump_toml(), encoding="utf-8") HarborTask(runtime_paths.task_dir) return PreparedRuntime(HarborTrialTaskConfig(path=runtime_paths.task_dir), sources) -def _validate_batch_compatibility( - tasks: tuple[E2ETask, ...], - settings: HarnessSettings, -) -> tuple[SourceTask, ...]: +def _validate_batch_compatibility(tasks: tuple[E2ETask, ...], settings: HarnessSettings) -> tuple[SourceTask, ...]: batch = _task_batch(tasks[0]) if batch is None or any(_task_batch(task) != batch for task in tasks): raise ValueError("Runtime aggregation requires one explicit shared batch") # noqa: TRY003 if any(task.dataset.path is None for task in tasks): raise ValueError(f"Batch {batch!r} requires local Harbor datasets") # noqa: TRY003 - if len({task.id for task in tasks}) != len(tasks): raise ValueError(f"Batch {batch!r} task IDs must be unique") # noqa: TRY003 + repository = settings.repository_path() sources = tuple(_load_source_task(task, repository) for task in tasks) + if any( + source.harbor_task.config.multi_step_reward_strategy is not MultiStepRewardStrategy.FINAL for source in sources + ): + raise ValueError(f"Batch {batch!r} requires Harbor's final multi-step reward strategy") # noqa: TRY003 first_profile = _runtime_profile(sources[0]) for source in sources[1:]: profile = _runtime_profile(source) - if incompatible := [name for name in first_profile if profile[name] != first_profile[name]]: + if incompatible := [name for name, value in first_profile.items() if profile[name] != value]: raise ValueError( # noqa: TRY003 f"Source task {source.task.id!r} has incompatible batch settings: {incompatible!r}" ) @@ -645,52 +567,47 @@ def _load_source_task(task: E2ETask, repository: Path) -> SourceTask: raise ValueError(f"Source task {task.id!r} cannot be loaded from {task_dir}") from exc # noqa: TRY003 if harbor_task.checksum != task.dataset.checksum: raise ValueError(f"Source task {task.id!r} checksum changed") # noqa: TRY003 - steps = _task_layout(task, harbor_task) - return SourceTask(task, harbor_task, steps) + return SourceTask(task, harbor_task, _task_layout(task, harbor_task)) def _runtime_profile(source: SourceTask) -> dict[str, Any]: task = source.task paths = source.harbor_task.paths + evaluation = task.evaluation return { "dataset": task.dataset.model_dump(mode="json", exclude={"task_id", "checksum"}), "execution": task.execution.model_dump(mode="json"), "capture": ( - task.evaluation.capture_events, - task.evaluation.checkpoint_every_events, - task.evaluation.max_event_bytes, - ), + evaluation.capture_events, + evaluation.checkpoint_every_events, + evaluation.max_event_bytes, + ) + if isinstance(evaluation, MemoryEvaluationSpec) + else (False, 5, 8192), "harbor": source.harbor_task.config.model_dump(mode="json", exclude={"steps"}), - "environment": _directory_checksum(paths.environment_dir), - "tests": _directory_checksum(paths.tests_dir), + "environment": environment_content_hash( + paths.environment_dir, + docker_image=source.harbor_task.config.environment.docker_image, + ), + "tests": dirhash(paths.tests_dir, "sha256") if paths.tests_dir.is_dir() else None, } -def _directory_checksum(path: Path) -> str | None: - return dirhash(path, "sha256") if path.is_dir() else None - - def _runtime_task_config(sources: tuple[SourceTask, ...], failure_policy: FailurePolicy) -> TaskConfig: min_reward = 1.0 if failure_policy == "fail-fast" else None steps = [ step.model_copy(update={"name": runtime_name, "min_reward": min_reward}) for source in sources - for step, runtime_name in zip( - source.harbor_task.config.steps or (), - source.runtime_steps, - strict=True, - ) + for step, runtime_name in zip(source.harbor_task.config.steps or (), source.runtime_steps, strict=True) ] return sources[0].harbor_task.config.model_copy(update={"steps": steps}) def _task_layout(task: E2ETask, harbor_task: HarborTask) -> tuple[str, ...]: steps = tuple(step.name for step in harbor_task.config.steps or ()) - if not steps: - raise ValueError(f"Source task {task.id!r} has no Harbor steps") # noqa: TRY003 - if len(steps) != len(set(steps)): - raise ValueError(f"Source task {task.id!r} step names must be unique") # noqa: TRY003 paths = tuple(PurePosixPath(step) for step in steps) + if not steps or len(steps) != len(set(steps)): + raise ValueError(f"Source task {task.id!r} must have unique Harbor steps") # noqa: TRY003 if any( "\\" in step or path.is_absolute() or len(path.parts) != 1 or path.parts[0] == ".." or path.as_posix() != step for step, path in zip(steps, paths, strict=True) @@ -727,19 +644,19 @@ async def prepared_context(client: PowerContextClient, scope_id: str, query: str async def _prepared_probes( client: PowerContextClient, - task: E2ETask, + evaluation: MemoryEvaluationSpec, scope_id: str, ) -> tuple[RecallProbeObservation, ...]: - observations = [] - for probe in task.evaluation.probes: - observations.append( + probes = [] + for probe in evaluation.probes: + probes.append( RecallProbeObservation( id=probe.id, query=probe.query, prepared_context=await prepared_context(client, scope_id, probe.query), ) ) - return tuple(observations) + return tuple(probes) def _run_environment(task: E2ETask, started_at: datetime, settings: HarnessSettings) -> RunEnvironment: @@ -755,9 +672,8 @@ def _run_environment(task: E2ETask, started_at: datetime, settings: HarnessSetti def _harbor_observation( - result: Any, - settings: HarnessSettings, -) -> tuple[HarborTrialObservation, tuple[Any, ...], Path | None]: + result: Any, settings: HarnessSettings +) -> tuple[HarborTrialObservation, tuple[StepResult, ...], Path | None]: if not result.trial_results: return ( HarborTrialObservation( @@ -793,9 +709,49 @@ def _trial_dir(trial_uri: str) -> Path | None: return Path(unquote(parsed.path)) if parsed.scheme == "file" else None -def _load_capture_records(trial_dir: Path) -> tuple[CaptureRecord, ...]: +def _task_artifacts( + trial_dir: Path | None, + task: E2ETask, + settings: HarnessSettings, + *, + step_names: tuple[str, ...] = (), + whole_trial: bool = True, +) -> TaskArtifacts: + if trial_dir is None: + return TaskArtifacts() + roots = (trial_dir,) if whole_trial else tuple(TrialPaths(trial_dir).step_dir(name) for name in step_names) + prefixes = tuple(f"steps/{name}/" for name in step_names) + instructions = load_resolved_instructions(trial_dir, settings) + return TaskArtifacts( + tuple(record for root in roots for record in _load_capture_records(root)), + tuple( + artifact + for root in roots + for artifact in _native_artifacts(root, task.execution.native_artifact_names, relative_to=trial_dir) + ), + tuple(instruction for instruction in instructions if whole_trial or instruction.artifact.startswith(prefixes)), + ) + + +def _collect_task_artifacts( + trial_dir: Path | None, + task: E2ETask, + settings: HarnessSettings, + *, + errors: list[str], + step_names: tuple[str, ...] = (), + whole_trial: bool = True, +) -> TaskArtifacts: + try: + return _task_artifacts(trial_dir, task, settings, step_names=step_names, whole_trial=whole_trial) + except Exception as exc: + errors.append(redact(f"{type(exc).__name__}: {exc}", settings)) + return TaskArtifacts() + + +def _load_capture_records(root: Path) -> tuple[CaptureRecord, ...]: records: list[CaptureRecord] = [] - for path in sorted(trial_dir.rglob("powercontext-capture.jsonl")): + for path in sorted(root.rglob("powercontext-capture.jsonl")): records.extend( CaptureRecord.model_validate_json(line) for line in path.read_text(encoding="utf-8").splitlines() @@ -810,24 +766,3 @@ def _native_artifacts( return tuple( fingerprint(path, relative_to=relative_to or root) for path in sorted(root.rglob("*")) if path.name in names ) - - -def _load_task_artifacts( - trial_dir: Path, - native_artifact_names: frozenset[str], - settings: HarnessSettings, - *, - step_names: tuple[str, ...] = (), -) -> TaskArtifacts: - roots = tuple(TrialPaths(trial_dir).step_dir(name) for name in step_names) or (trial_dir,) - instructions = load_resolved_instructions(trial_dir, settings) - prefixes = tuple(f"steps/{name}/" for name in step_names) - return TaskArtifacts( - tuple(record for root in roots for record in _load_capture_records(root)), - tuple( - artifact - for root in roots - for artifact in _native_artifacts(root, native_artifact_names, relative_to=trial_dir) - ), - tuple(instruction for instruction in instructions if not prefixes or instruction.artifact.startswith(prefixes)), - ) diff --git a/e2e/bub/tasks/project-database-decision.yaml b/e2e/bub/tasks/acceptance-01-project-database-decision.yaml similarity index 92% rename from e2e/bub/tasks/project-database-decision.yaml rename to e2e/bub/tasks/acceptance-01-project-database-decision.yaml index 6c1adacdf..d8d729fbc 100644 --- a/e2e/bub/tasks/project-database-decision.yaml +++ b/e2e/bub/tasks/acceptance-01-project-database-decision.yaml @@ -18,10 +18,11 @@ categories: - acceptance - sample - smoke + - batch:acceptance dataset: path: e2e/bub/harbor-tasks task_id: project-database-decision - checksum: 8c1f9d11581c4cea3f7df56fd0ddfc03ce07f6ab29a86cde6a14c52a349264f9 + checksum: 4d0e45489408dbece683ab91b0d77419bb90a202d1c66c1209e69315ad97e799 execution: type: bub model: false diff --git a/e2e/bub/harbor-tasks/failure-policy-followup/steps/run/solution/solve.sh b/e2e/bub/tasks/acceptance-02-failure-policy-timeout.yaml similarity index 57% rename from e2e/bub/harbor-tasks/failure-policy-followup/steps/run/solution/solve.sh rename to e2e/bub/tasks/acceptance-02-failure-policy-timeout.yaml index 2016def17..bb59a0876 100644 --- a/e2e/bub/harbor-tasks/failure-policy-followup/steps/run/solution/solve.sh +++ b/e2e/bub/tasks/acceptance-02-failure-policy-timeout.yaml @@ -1,4 +1,3 @@ -#!/bin/sh # Copyright (c) 2026 OceanBase. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,6 +12,22 @@ # See the License for the specific language governing permissions and # limitations under the License. -set -eu - -touch /workspace/failure-policy-followup-ran +schema: powercontext.e2e-task/v1 +id: failure-policy-timeout +categories: + - acceptance + - fixture + - batch:acceptance +dataset: + path: e2e/bub/harbor-tasks + task_id: failure-policy-timeout + checksum: d0f01c9a9f1b331ff3904ddc864c4aa5ec673fa2179d9947b853827fb54714d4 +execution: + type: bub + model: false + max_steps: 10 + max_tokens: 4096 +evaluation: + expected_execution: + collect-all: failed + fail-fast: failed diff --git a/e2e/bub/harbor-tasks/failure-policy-timeout/steps/run/solution/solve.sh b/e2e/bub/tasks/acceptance-03-failure-policy-followup.yaml similarity index 57% rename from e2e/bub/harbor-tasks/failure-policy-timeout/steps/run/solution/solve.sh rename to e2e/bub/tasks/acceptance-03-failure-policy-followup.yaml index ff7c127d5..2a3f78f62 100644 --- a/e2e/bub/harbor-tasks/failure-policy-timeout/steps/run/solution/solve.sh +++ b/e2e/bub/tasks/acceptance-03-failure-policy-followup.yaml @@ -1,4 +1,3 @@ -#!/bin/sh # Copyright (c) 2026 OceanBase. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,6 +12,22 @@ # See the License for the specific language governing permissions and # limitations under the License. -set -eu - -sleep 5 +schema: powercontext.e2e-task/v1 +id: failure-policy-followup +categories: + - acceptance + - fixture + - batch:acceptance +dataset: + path: e2e/bub/harbor-tasks + task_id: failure-policy-followup + checksum: 350d4719cac43cd99e93ea7d92ca6bcb651e1e5f1805a041a1abbf58af5216fc +execution: + type: bub + model: false + max_steps: 10 + max_tokens: 4096 +evaluation: + expected_execution: + collect-all: completed + fail-fast: skipped diff --git a/e2e/bub/tasks/locomo-multihop-football.yaml b/e2e/bub/tasks/locomo-multihop-football.yaml index a28751a44..0e54cce07 100644 --- a/e2e/bub/tasks/locomo-multihop-football.yaml +++ b/e2e/bub/tasks/locomo-multihop-football.yaml @@ -29,7 +29,7 @@ provenance: dataset: path: e2e/bub/harbor-tasks task_id: locomo-multihop-football - checksum: 773e967b07c26ab3b6e8e0de6f1f0d1c752e6d10c3227b5788796ee967aeff6f + checksum: d2700bd79b6f5e7d96bb0ab802c414a4a90fd652eb2612036a99aebd2b25f9e4 execution: type: bub model: false diff --git a/e2e/bub/tasks/locomo-open-pastries.yaml b/e2e/bub/tasks/locomo-open-pastries.yaml index 34bf96e34..1c48cab1a 100644 --- a/e2e/bub/tasks/locomo-open-pastries.yaml +++ b/e2e/bub/tasks/locomo-open-pastries.yaml @@ -29,7 +29,7 @@ provenance: dataset: path: e2e/bub/harbor-tasks task_id: locomo-open-pastries - checksum: 92991fc1ff3edb187b420c943e88e504c2b0565d9c6c757e7e13f8c77ed3d81a + checksum: e5578c5f860c439230a9e67399ee61f9303ddf95831b08d78a40385c7f964831 execution: type: bub model: false diff --git a/e2e/bub/tasks/locomo-support-group.yaml b/e2e/bub/tasks/locomo-support-group.yaml index f1926b979..3ad7f23f0 100644 --- a/e2e/bub/tasks/locomo-support-group.yaml +++ b/e2e/bub/tasks/locomo-support-group.yaml @@ -29,7 +29,7 @@ provenance: dataset: path: e2e/bub/harbor-tasks task_id: locomo-support-group - checksum: babb50723aff96edba700a80159aa2fbf8333a4111ad8339f54e81d138f9ff11 + checksum: f1bdfa62f960e3a749b84c19e3cb7661bb3ab0d4ba049547efdbcfb870db18fa execution: type: bub model: false diff --git a/e2e/bub/tasks/locomo-temporal-banker.yaml b/e2e/bub/tasks/locomo-temporal-banker.yaml index 6e03b60ba..98b46dd69 100644 --- a/e2e/bub/tasks/locomo-temporal-banker.yaml +++ b/e2e/bub/tasks/locomo-temporal-banker.yaml @@ -29,7 +29,7 @@ provenance: dataset: path: e2e/bub/harbor-tasks task_id: locomo-temporal-banker - checksum: 1afc402fffa2ea2509fb1007ec218eda81e09d86401304a5570af39eca28fb58 + checksum: f9982061c303d10de5b44f02152e6a3c9abab5e5d76f60fed76f561a603cc363 execution: type: bub model: false diff --git a/e2e/bub/tests/test_runtime_batch.py b/e2e/bub/tests/test_runtime_batch.py deleted file mode 100644 index c7e073ecb..000000000 --- a/e2e/bub/tests/test_runtime_batch.py +++ /dev/null @@ -1,142 +0,0 @@ -# Copyright (c) 2026 OceanBase. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import annotations - -import json -from pathlib import Path - -import pytest -from harbor.models.task.task import Task as HarborTask -from harbor.models.trial.result import StepResult -from harbor.models.verifier.result import VerifierResult - -from powercontext_e2e.catalog import E2ETask, load_tasks -from powercontext_e2e.models import HarborTrialObservation -from powercontext_e2e.runner import ( - SourceResult, - SourceTask, - _load_task_artifacts, - _source_harbor_observation, - prepare_runtime_task, -) -from powercontext_e2e.settings import HarnessSettings - -REPOSITORY = Path(__file__).resolve().parents[3] - - -def _locomo_tasks() -> tuple[E2ETask, ...]: - return tuple(task for task in load_tasks(REPOSITORY / "e2e" / "bub" / "tasks") if "batch:locomo" in task.categories) - - -def test_selected_locomo_tasks_share_one_runtime_with_requested_failure_policy(tmp_path: Path) -> None: - tasks = _locomo_tasks() - settings = HarnessSettings(repository=REPOSITORY) - - collect_all = prepare_runtime_task( - tasks, - output_dir=tmp_path, - settings=settings, - failure_policy="collect-all", - ) - assert collect_all.task_config.path is not None - runtime_task = HarborTask(collect_all.task_config.path) - runtime_step_names = tuple(f"{task.id}-{step}" for task in tasks for step in ("capture", "recall")) - - assert tuple(step.name for step in runtime_task.config.steps or ()) == runtime_step_names - assert all(step.min_reward is None for step in runtime_task.config.steps or ()) - - fail_fast = prepare_runtime_task( - tasks, - output_dir=tmp_path, - settings=settings, - failure_policy="fail-fast", - ) - assert fail_fast.task_config.path is not None - fail_fast_task = HarborTask(fail_fast.task_config.path) - assert fail_fast.task_config.path != collect_all.task_config.path - assert all(step.min_reward == 1.0 for step in fail_fast_task.config.steps or ()) - - -def test_shared_trial_artifacts_stay_with_their_source_task(tmp_path: Path) -> None: - tasks = _locomo_tasks()[:2] - prepared = prepare_runtime_task( - tasks, - output_dir=tmp_path / "runtime", - settings=HarnessSettings(repository=REPOSITORY), - failure_policy="collect-all", - ) - trial_dir = tmp_path / "trial" - for source in prepared.sources: - agent_dir = trial_dir / "steps" / f"{source.task.id}-recall" / "agent" - agent_dir.mkdir(parents=True) - (agent_dir / "powercontext-capture.jsonl").write_text( - json.dumps({ - "schema": "powercontext.bub-capture-event/v1", - "recorded_at": "2026-08-19T00:00:00Z", - "event": "user_prompt", - "status": "captured", - "source_id": f"bub-event:{source.task.id}", - }) - + "\n", - encoding="utf-8", - ) - settings = HarnessSettings(repository=REPOSITORY) - artifacts = { - source.task.id: _load_task_artifacts( - trial_dir, - source.task.execution.native_artifact_names, - settings, - step_names=source.runtime_steps, - ) - for source in prepared.sources - } - - assert {task.id: [record.source_id for record in artifacts[task.id].capture_records] for task in tasks} == { - task.id: [f"bub-event:{task.id}"] for task in tasks - } - - -@pytest.mark.parametrize( - ("task_id", "step_rewards", "expected"), - [ - ( - "locomo-support-group", - ({"reward": 1, "detail": 1}, {"reward": 0}), - {"reward": 0.5, "detail": 0.5}, - ), - ("project-database-decision", ({"reward": 1}, {"reward": 0}), {"reward": 0}), - ], -) -def test_batch_source_evidence_preserves_harbor_reward_strategy( - task_id: str, - step_rewards: tuple[dict[str, int], ...], - expected: dict[str, float | int], -) -> None: - task = load_tasks(REPOSITORY / "e2e" / "bub" / "tasks" / f"{task_id}.yaml")[0] - assert task.dataset.path is not None - harbor_task = HarborTask(REPOSITORY / task.dataset.path / task.dataset.task_id) - source = SourceTask(task, harbor_task, tuple(step.name for step in harbor_task.config.steps or ())) - steps = tuple( - StepResult(step_name=name, verifier_result=VerifierResult(rewards=rewards)) - for name, rewards in zip(source.runtime_steps, step_rewards, strict=True) - ) - source_harbor = _source_harbor_observation( - HarborTrialObservation(), - source, - SourceResult("completed", (), steps), - HarnessSettings(repository=REPOSITORY), - ) - - assert source_harbor.rewards == expected diff --git a/e2e/bub/tests/test_workload_catalog.py b/e2e/bub/tests/test_workload_catalog.py index 8ab9dcd77..3ec4ffdc2 100644 --- a/e2e/bub/tests/test_workload_catalog.py +++ b/e2e/bub/tests/test_workload_catalog.py @@ -27,11 +27,13 @@ def test_workloads_can_be_selected_by_multiple_ids_or_category() -> None: tasks = load_tasks(repository / "e2e" / "bub" / "tasks") assert [task.id for task in tasks] == [ + "project-database-decision", + "failure-policy-timeout", + "failure-policy-followup", "locomo-multihop-football", "locomo-open-pastries", "locomo-support-group", "locomo-temporal-banker", - "project-database-decision", "terminal-bench-db-wal-recovery", ] assert {task.execution.type for task in tasks} == {"bub"} @@ -48,11 +50,17 @@ def test_workloads_can_be_selected_by_multiple_ids_or_category() -> None: "terminal-bench-db-wal-recovery", ] assert [task.id for task in acceptance] == [ + "project-database-decision", + "failure-policy-timeout", + "failure-policy-followup", "locomo-multihop-football", "locomo-open-pastries", "locomo-support-group", "locomo-temporal-banker", - "project-database-decision", + ] + assert [group.tasks for group in group_tasks(acceptance)] == [ + tuple(task for task in acceptance if "batch:acceptance" in task.categories), + tuple(task for task in acceptance if "batch:locomo" in task.categories), ]