diff --git a/evaluators/contrib/galileo/README.md b/evaluators/contrib/galileo/README.md index 8191bb73..e0db3d6a 100644 --- a/evaluators/contrib/galileo/README.md +++ b/evaluators/contrib/galileo/README.md @@ -2,6 +2,14 @@ Integration package for Galileo Luna evaluator. +## Context-limit errors + +If the model server rejects an input for exceeding its maximum context length, +the evaluator returns `metadata.error_code: context_limit`. The server still +redacts the error message. Clients can use this safe code to report an unsupported +input separately from transient evaluator failures without exposing internal error +details or retrying the same oversized input. + ## Migrating from Luna2 The `galileo.luna2` evaluator ID has been removed. Existing controls that use diff --git a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/evaluator.py b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/evaluator.py index 14b58d88..265c194b 100644 --- a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/evaluator.py +++ b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/evaluator.py @@ -359,6 +359,12 @@ def _handle_error( **self._base_metadata(), "error_type": type(error).__name__, } + # The server redacts error text, so retain this deterministic failure as + # a safe code that clients can distinguish from an evaluator outage. + if isinstance(error, RuntimeError) and error_detail.startswith( + "Input exceeds maximum length for Galileo model server:" + ): + metadata["error_code"] = "context_limit" if isinstance(error, httpx.HTTPStatusError): metadata.update(_http_status_error_metadata(error))