From 4e2490a25004891ab4f2c798f3434552c39cfabb Mon Sep 17 00:00:00 2001 From: krisztianfekete Date: Fri, 9 Oct 2026 15:53:20 +0200 Subject: [PATCH] fix: keep GenAI content out of the backend recipe, isolate SDK TLS, set a valid content capture mode --- README.md | 4 +-- docs/opentelemetry-pipeline.md | 24 +++++++++++--- examples/langchain_agent/README.md | 2 +- src/agentevals/otel/sdk_export.py | 53 +++++++++++++++--------------- src/agentevals/sdk.py | 15 ++++++++- tests/test_sdk.py | 29 +++++++++++++--- 6 files changed, 87 insertions(+), 40 deletions(-) diff --git a/README.md b/README.md index 413bf346..706f22c1 100644 --- a/README.md +++ b/README.md @@ -6,10 +6,10 @@

-

Agent evaluation, OpenTelemetry native

+

OpenTelemetry-native agent evaluation

-Send OTLP. Score what your agent did. Get the results back as OpenTelemetry events.
+Send OTLP traces. Score what your agent did. Get the results back as OpenTelemetry GenAI eval events.
No SDK, no reruns, no platform to host.

diff --git a/docs/opentelemetry-pipeline.md b/docs/opentelemetry-pipeline.md index 8ecc3d69..0c844547 100644 --- a/docs/opentelemetry-pipeline.md +++ b/docs/opentelemetry-pipeline.md @@ -53,12 +53,25 @@ processors: check_interval: 1s limit_percentage: 80 spike_limit_percentage: 20 + filter/drop_genai_content_events: + error_mode: ignore + traces: + spanevent: + - IsMatch(spanevent.name, "^gen_ai\\.(client\\.inference\\.operation\\.details|(system|user|assistant|tool)\\.message|choice|content\\.(prompt|completion))$") + logs: + log_record: + - IsMatch(log.event_name, "^gen_ai\\.(client\\.inference\\.operation\\.details|(system|user|assistant|tool)\\.message|choice|content\\.(prompt|completion))$") + - IsMatch(log.attributes["event.name"], "^gen_ai\\.(client\\.inference\\.operation\\.details|(system|user|assistant|tool)\\.message|choice|content\\.(prompt|completion))$") transform/drop_genai_content: + error_mode: ignore trace_statements: - - delete_key(span.attributes, "gen_ai.input.messages") - - delete_key(span.attributes, "gen_ai.output.messages") - - delete_key(span.attributes, "gen_ai.system_instructions") + - delete_matching_keys(span.attributes, "^gen_ai\\.(input\\.messages|output\\.messages|system_instructions|tool\\.definitions|tool\\.call\\.(arguments|result)|retrieval\\.(query\\.text|documents)|memory\\.(query\\.text|records)|prompt\\.variable\\..+)$") + - delete_matching_keys(span.attributes, "^(gen_ai\\.(prompt|completion)(\\.[0-9]+\\..+)?|gcp\\.vertex\\.agent\\.(llm_request|llm_response|tool_call_args|tool_response|data)|traceloop\\.entity\\.(input|output))$") + log_statements: + - delete_matching_keys(log.attributes, "^gen_ai\\.(input\\.messages|output\\.messages|system_instructions|tool\\.definitions|tool\\.call\\.(arguments|result)|retrieval\\.(query\\.text|documents)|memory\\.(query\\.text|records)|prompt\\.variable\\..+)$") + - delete_matching_keys(log.attributes, "^(gen_ai\\.(prompt|completion)(\\.[0-9]+\\..+)?|gcp\\.vertex\\.agent\\.(llm_request|llm_response|tool_call_args|tool_response|data)|traceloop\\.entity\\.(input|output))$") filter/not_from_agentevals: + error_mode: ignore logs: log_record: - resource.attributes["service.name"] == "agentevals" @@ -90,7 +103,7 @@ service: pipelines: traces/backend: receivers: [otlp] - processors: [memory_limiter, transform/drop_genai_content] + processors: [memory_limiter, filter/drop_genai_content_events, transform/drop_genai_content] exporters: [otlp/backend] traces/agentevals: receivers: [otlp] @@ -98,7 +111,7 @@ service: exporters: [otlphttp/agentevals] logs/backend: receivers: [otlp] - processors: [memory_limiter] + processors: [memory_limiter, filter/drop_genai_content_events, transform/drop_genai_content] exporters: [otlp/backend, signaltometrics] logs/agentevals: receivers: [otlp] @@ -111,6 +124,7 @@ service: Notes: +* `filter/drop_genai_content_events` and `transform/drop_genai_content` remove content from spans, span events and logs: GenAI convention attributes and events, the older message events, and the ADK and OpenLLMetry attributes. Model, token, tool name and error data stay. Other instrumentations (for example OpenInference `input.value` and `output.value`) need their keys added. * `filter/not_from_agentevals` keeps agentevals' own results out of the agentevals branch. agentevals also drops them itself. * If you sample, sample whole traces and only on the backend branch. agentevals needs complete traces. * For producers that use OpenInference or OpenLLMetry attribute names, the contrib `gen_ai_normalizer` processor (alpha) can map them to GenAI conventions on the agentevals branch. diff --git a/examples/langchain_agent/README.md b/examples/langchain_agent/README.md index d1289072..a219e27f 100644 --- a/examples/langchain_agent/README.md +++ b/examples/langchain_agent/README.md @@ -25,7 +25,7 @@ with app.session(session_name=session_name): ... # run the agent ``` -* The SDK instruments the OpenAI client (`opentelemetry-instrumentation-openai-v2`) and sets up a log provider, because this instrumentation puts message content in log events. Content capture is turned on for you (`OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT=true`). +* The SDK instruments the OpenAI client (`opentelemetry-instrumentation-openai-v2`) and sets up a log provider, because this instrumentation puts message content in log events. Content capture is turned on for you when you have not set it: `OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT=true`, or `SPAN_ONLY` when `OTEL_SEMCONV_STABILITY_OPT_IN` includes `gen_ai_latest_experimental`. * Spans and GenAI log events produced inside the session go to agentevals on port 4318. Leaving the block flushes them. * LangChain only instruments the model client here, so each model call is its own trace. agentevals joins a call that answers a tool result to the turn before it, so the UI shows three turns. diff --git a/src/agentevals/otel/sdk_export.py b/src/agentevals/otel/sdk_export.py index 01610700..58884cb4 100644 --- a/src/agentevals/otel/sdk_export.py +++ b/src/agentevals/otel/sdk_export.py @@ -132,21 +132,24 @@ def resolve_endpoint(endpoint: str | None = None, ws_url: str | None = None) -> return DEFAULT_ENDPOINT -def _isolated(exporter: Any) -> Any: - """Never present the application's TLS client identity, which the exporter reads from the - environment when it is not passed explicitly.""" - for attr in ("_client_key_file", "_client_certificate_file"): - if hasattr(exporter, attr): - setattr(exporter, attr, None) - return exporter - - def _http_session() -> Any: """A requests session that ignores ``.netrc`` and proxy variables, so nothing configured for - other hosts is applied to the agentevals endpoint.""" + other hosts is applied to the agentevals endpoint. + + It also drops the TLS options of every request. The exporters build them in ``__init__`` from + ``OTEL_EXPORTER_OTLP_*CERTIFICATE`` and ``*CLIENT_KEY`` when they are not passed, and an empty + value cannot be passed, so the application's client certificate and CA bundle would otherwise + be used for agentevals. Requests verify against the default trust store with no client + certificate.""" import requests - session = requests.Session() + class _IsolatedSession(requests.Session): + def request(self, method: str, url: str, **kwargs: Any) -> Any: + kwargs["verify"] = True + kwargs["cert"] = None + return super().request(method, url, **kwargs) + + session = _IsolatedSession() session.trust_env = False return session @@ -155,14 +158,12 @@ def _span_exporter(endpoint: str) -> Any: from opentelemetry.exporter.otlp.proto.http import Compression from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter - return _isolated( - OTLPSpanExporter( - endpoint=f"{endpoint}/v1/traces", - headers=dict(_SDK_HEADER), - timeout=EXPORT_TIMEOUT_SECONDS, - compression=Compression.Gzip, - session=_http_session(), - ) + return OTLPSpanExporter( + endpoint=f"{endpoint}/v1/traces", + headers=dict(_SDK_HEADER), + timeout=EXPORT_TIMEOUT_SECONDS, + compression=Compression.Gzip, + session=_http_session(), ) @@ -170,14 +171,12 @@ def _log_exporter(endpoint: str) -> Any: from opentelemetry.exporter.otlp.proto.http import Compression from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter - return _isolated( - OTLPLogExporter( - endpoint=f"{endpoint}/v1/logs", - headers=dict(_SDK_HEADER), - timeout=EXPORT_TIMEOUT_SECONDS, - compression=Compression.Gzip, - session=_http_session(), - ) + return OTLPLogExporter( + endpoint=f"{endpoint}/v1/logs", + headers=dict(_SDK_HEADER), + timeout=EXPORT_TIMEOUT_SECONDS, + compression=Compression.Gzip, + session=_http_session(), ) diff --git a/src/agentevals/sdk.py b/src/agentevals/sdk.py index a50c15ef..b268ef3b 100644 --- a/src/agentevals/sdk.py +++ b/src/agentevals/sdk.py @@ -62,6 +62,19 @@ def my_agent(prompt): _DEFAULT_WS_URL = LEGACY_WS_URL +def _content_capture_default() -> str: + """The content capture value the instrumentations accept in the current semconv mode. + + In ``gen_ai_latest_experimental`` mode util-genai based instrumentations only accept a mode + name and fall back to no content for ``true``. In the default mode openai-v2 only accepts + ``true``. ``SPAN_ONLY`` puts content where agentevals reads it first, without a duplicate + copy in log events.""" + opt_in = os.environ.get("OTEL_SEMCONV_STABILITY_OPT_IN", "") + if "gen_ai_latest_experimental" in {value.strip() for value in opt_in.split(",")}: + return "SPAN_ONLY" + return "true" + + @dataclass(slots=True) class _OtelSetup: tracer_provider: SdkTracerProvider @@ -298,7 +311,7 @@ def _setup_otel( from opentelemetry.sdk.trace import TracerProvider if self.capture_message_content: - os.environ.setdefault("OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", "true") + os.environ.setdefault("OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", _content_capture_default()) if explicit_tracer_provider is not None: tracer_provider = explicit_tracer_provider diff --git a/tests/test_sdk.py b/tests/test_sdk.py index b9763d2c..b9cc2b9f 100644 --- a/tests/test_sdk.py +++ b/tests/test_sdk.py @@ -134,16 +134,37 @@ def test_span_exporter_ignores_application_otlp_settings(self, monkeypatch): monkeypatch.setenv("OTEL_EXPORTER_OTLP_HEADERS", "authorization=Bearer app-token") monkeypatch.setenv("OTEL_EXPORTER_OTLP_TRACES_HEADERS", "x-api-key=app-key") monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "https://observability.example.com") - monkeypatch.setenv("OTEL_EXPORTER_OTLP_CLIENT_KEY", "/etc/app/client.key") - monkeypatch.setenv("OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE", "/etc/app/client.crt") exporter = sdk_export._span_exporter("http://localhost:4318") assert exporter._endpoint == "http://localhost:4318/v1/traces" assert exporter._headers == {"x-agentevals-sdk": "1"} assert "authorization" not in {k.lower() for k in exporter._session.headers} - assert exporter._client_key_file is None - assert exporter._client_certificate_file is None assert exporter._session.trust_env is False + @pytest.mark.parametrize("factory", ["_span_exporter", "_log_exporter"]) + def test_exports_never_use_application_tls_settings(self, factory, monkeypatch): + import requests.adapters + from requests.models import Response + + monkeypatch.setenv("OTEL_EXPORTER_OTLP_CERTIFICATE", "/etc/app/ca.pem") + monkeypatch.setenv("OTEL_EXPORTER_OTLP_CLIENT_KEY", "/etc/app/client.key") + monkeypatch.setenv("OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE", "/etc/app/client.crt") + monkeypatch.setenv("OTEL_EXPORTER_OTLP_TRACES_CLIENT_CERTIFICATE", "/etc/app/traces.crt") + monkeypatch.setenv("OTEL_EXPORTER_OTLP_LOGS_CLIENT_CERTIFICATE", "/etc/app/logs.crt") + sent = [] + + def send(adapter, request, **kwargs): + sent.append(kwargs) + response = Response() + response.status_code = 200 + response.request = request + return response + + monkeypatch.setattr(requests.adapters.HTTPAdapter, "send", send) + getattr(sdk_export, factory)("https://agentevals.example:4318")._export(b"payload") + assert len(sent) == 1 + assert sent[0]["cert"] is None + assert sent[0]["verify"] is True + def test_log_exporter_ignores_application_otlp_settings(self, monkeypatch): monkeypatch.setenv("OTEL_EXPORTER_OTLP_HEADERS", "authorization=Bearer app-token") monkeypatch.setenv("OTEL_EXPORTER_OTLP_LOGS_HEADERS", "x-api-key=app-key")