fix: keep GenAI content out of the backend recipe, isolate SDK TLS, set content capture

README.md

@@ -6,10 +6,10 @@
   </picture>
 </p>
 
-<h1 align="center">Agent evaluation, OpenTelemetry native</h1>
+<h1 align="center">OpenTelemetry-native agent evaluation</h1>
 
 <p align="center">
-Send OTLP. Score what your agent did. Get the results back as OpenTelemetry events.<br>
+Send OTLP traces. Score what your agent did. Get the results back as OpenTelemetry GenAI eval events.<br>
 No SDK, no reruns, no platform to host.
 </p>
 

docs/opentelemetry-pipeline.md

@@ -53,12 +53,25 @@ processors:
     check_interval: 1s
     limit_percentage: 80
     spike_limit_percentage: 20
+  filter/drop_genai_content_events:
+    error_mode: ignore
+    traces:
+      spanevent:
+        - IsMatch(spanevent.name, "^gen_ai\\.(client\\.inference\\.operation\\.details|(system|user|assistant|tool)\\.message|choice|content\\.(prompt|completion))$")
+    logs:
+      log_record:
+        - IsMatch(log.event_name, "^gen_ai\\.(client\\.inference\\.operation\\.details|(system|user|assistant|tool)\\.message|choice|content\\.(prompt|completion))$")
+        - IsMatch(log.attributes["event.name"], "^gen_ai\\.(client\\.inference\\.operation\\.details|(system|user|assistant|tool)\\.message|choice|content\\.(prompt|completion))$")
   transform/drop_genai_content:
+    error_mode: ignore
     trace_statements:
-      - delete_key(span.attributes, "gen_ai.input.messages")
-      - delete_key(span.attributes, "gen_ai.output.messages")
-      - delete_key(span.attributes, "gen_ai.system_instructions")
+      - delete_matching_keys(span.attributes, "^gen_ai\\.(input\\.messages|output\\.messages|system_instructions|tool\\.definitions|tool\\.call\\.(arguments|result)|retrieval\\.(query\\.text|documents)|memory\\.(query\\.text|records)|prompt\\.variable\\..+)$")
+      - delete_matching_keys(span.attributes, "^(gen_ai\\.(prompt|completion)(\\.[0-9]+\\..+)?|gcp\\.vertex\\.agent\\.(llm_request|llm_response|tool_call_args|tool_response|data)|traceloop\\.entity\\.(input|output))$")
+    log_statements:
+      - delete_matching_keys(log.attributes, "^gen_ai\\.(input\\.messages|output\\.messages|system_instructions|tool\\.definitions|tool\\.call\\.(arguments|result)|retrieval\\.(query\\.text|documents)|memory\\.(query\\.text|records)|prompt\\.variable\\..+)$")
+      - delete_matching_keys(log.attributes, "^(gen_ai\\.(prompt|completion)(\\.[0-9]+\\..+)?|gcp\\.vertex\\.agent\\.(llm_request|llm_response|tool_call_args|tool_response|data)|traceloop\\.entity\\.(input|output))$")
   filter/not_from_agentevals:
+    error_mode: ignore
     logs:
       log_record:
         - resource.attributes["service.name"] == "agentevals"
@@ -90,15 +103,15 @@ service:
   pipelines:
     traces/backend:
       receivers: [otlp]
-      processors: [memory_limiter, transform/drop_genai_content]
+      processors: [memory_limiter, filter/drop_genai_content_events, transform/drop_genai_content]
       exporters: [otlp/backend]
     traces/agentevals:
       receivers: [otlp]
       processors: [memory_limiter]
       exporters: [otlphttp/agentevals]
     logs/backend:
       receivers: [otlp]
-      processors: [memory_limiter]
+      processors: [memory_limiter, filter/drop_genai_content_events, transform/drop_genai_content]
       exporters: [otlp/backend, signaltometrics]
     logs/agentevals:
       receivers: [otlp]
@@ -111,6 +124,7 @@ service:
 
 Notes:
 
+* `filter/drop_genai_content_events` and `transform/drop_genai_content` remove content from spans, span events and logs: GenAI convention attributes and events, the older message events, and the ADK and OpenLLMetry attributes. Model, token, tool name and error data stay. Other instrumentations (for example OpenInference `input.value` and `output.value`) need their keys added.
 * `filter/not_from_agentevals` keeps agentevals' own results out of the agentevals branch. agentevals also drops them itself.
 * If you sample, sample whole traces and only on the backend branch. agentevals needs complete traces.
 * For producers that use OpenInference or OpenLLMetry attribute names, the contrib `gen_ai_normalizer` processor (alpha) can map them to GenAI conventions on the agentevals branch.

examples/langchain_agent/README.md

@@ -25,7 +25,7 @@ with app.session(session_name=session_name):
     ...  # run the agent
 ```
 
-* The SDK instruments the OpenAI client (`opentelemetry-instrumentation-openai-v2`) and sets up a log provider, because this instrumentation puts message content in log events. Content capture is turned on for you (`OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT=true`).
+* The SDK instruments the OpenAI client (`opentelemetry-instrumentation-openai-v2`) and sets up a log provider, because this instrumentation puts message content in log events. Content capture is turned on for you when you have not set it: `OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT=true`, or `SPAN_ONLY` when `OTEL_SEMCONV_STABILITY_OPT_IN` includes `gen_ai_latest_experimental`.
 * Spans and GenAI log events produced inside the session go to agentevals on port 4318. Leaving the block flushes them.
 * LangChain only instruments the model client here, so each model call is its own trace. agentevals joins a call that answers a tool result to the turn before it, so the UI shows three turns.
 

src/agentevals/otel/sdk_export.py

@@ -132,21 +132,24 @@ def resolve_endpoint(endpoint: str | None = None, ws_url: str | None = None) ->
     return DEFAULT_ENDPOINT
 
 
-def _isolated(exporter: Any) -> Any:
-    """Never present the application's TLS client identity, which the exporter reads from the
-    environment when it is not passed explicitly."""
-    for attr in ("_client_key_file", "_client_certificate_file"):
-        if hasattr(exporter, attr):
-            setattr(exporter, attr, None)
-    return exporter
-
-
 def _http_session() -> Any:
     """A requests session that ignores ``.netrc`` and proxy variables, so nothing configured for
-    other hosts is applied to the agentevals endpoint."""
+    other hosts is applied to the agentevals endpoint.
+
+    It also drops the TLS options of every request. The exporters build them in ``__init__`` from
+    ``OTEL_EXPORTER_OTLP_*CERTIFICATE`` and ``*CLIENT_KEY`` when they are not passed, and an empty
+    value cannot be passed, so the application's client certificate and CA bundle would otherwise
+    be used for agentevals. Requests verify against the default trust store with no client
+    certificate."""
     import requests
 
-    session = requests.Session()
+    class _IsolatedSession(requests.Session):
+        def request(self, method: str, url: str, **kwargs: Any) -> Any:
+            kwargs["verify"] = True
+            kwargs["cert"] = None
+            return super().request(method, url, **kwargs)
+
+    session = _IsolatedSession()
     session.trust_env = False
     return session
 
@@ -155,29 +158,25 @@ def _span_exporter(endpoint: str) -> Any:
     from opentelemetry.exporter.otlp.proto.http import Compression
     from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
 
-    return _isolated(
-        OTLPSpanExporter(
-            endpoint=f"{endpoint}/v1/traces",
-            headers=dict(_SDK_HEADER),
-            timeout=EXPORT_TIMEOUT_SECONDS,
-            compression=Compression.Gzip,
-            session=_http_session(),
-        )
+    return OTLPSpanExporter(
+        endpoint=f"{endpoint}/v1/traces",
+        headers=dict(_SDK_HEADER),
+        timeout=EXPORT_TIMEOUT_SECONDS,
+        compression=Compression.Gzip,
+        session=_http_session(),
     )
 
 
 def _log_exporter(endpoint: str) -> Any:
     from opentelemetry.exporter.otlp.proto.http import Compression
     from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
 
-    return _isolated(
-        OTLPLogExporter(
-            endpoint=f"{endpoint}/v1/logs",
-            headers=dict(_SDK_HEADER),
-            timeout=EXPORT_TIMEOUT_SECONDS,
-            compression=Compression.Gzip,
-            session=_http_session(),
-        )
+    return OTLPLogExporter(
+        endpoint=f"{endpoint}/v1/logs",
+        headers=dict(_SDK_HEADER),
+        timeout=EXPORT_TIMEOUT_SECONDS,
+        compression=Compression.Gzip,
+        session=_http_session(),
     )
 
 

src/agentevals/sdk.py

@@ -62,6 +62,19 @@ def my_agent(prompt):
 _DEFAULT_WS_URL = LEGACY_WS_URL
 
 
+def _content_capture_default() -> str:
+    """The content capture value the instrumentations accept in the current semconv mode.
+
+    In ``gen_ai_latest_experimental`` mode util-genai based instrumentations only accept a mode
+    name and fall back to no content for ``true``. In the default mode openai-v2 only accepts
+    ``true``. ``SPAN_ONLY`` puts content where agentevals reads it first, without a duplicate
+    copy in log events."""
+    opt_in = os.environ.get("OTEL_SEMCONV_STABILITY_OPT_IN", "")
+    if "gen_ai_latest_experimental" in {value.strip() for value in opt_in.split(",")}:
+        return "SPAN_ONLY"
+    return "true"
+
+
 @dataclass(slots=True)
 class _OtelSetup:
     tracer_provider: SdkTracerProvider
@@ -298,7 +311,7 @@ def _setup_otel(
         from opentelemetry.sdk.trace import TracerProvider
 
         if self.capture_message_content:
-            os.environ.setdefault("OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", "true")
+            os.environ.setdefault("OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", _content_capture_default())
 
         if explicit_tracer_provider is not None:
             tracer_provider = explicit_tracer_provider

tests/test_sdk.py

@@ -134,16 +134,37 @@ def test_span_exporter_ignores_application_otlp_settings(self, monkeypatch):
         monkeypatch.setenv("OTEL_EXPORTER_OTLP_HEADERS", "authorization=Bearer app-token")
         monkeypatch.setenv("OTEL_EXPORTER_OTLP_TRACES_HEADERS", "x-api-key=app-key")
         monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "https://observability.example.com")
-        monkeypatch.setenv("OTEL_EXPORTER_OTLP_CLIENT_KEY", "/etc/app/client.key")
-        monkeypatch.setenv("OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE", "/etc/app/client.crt")
         exporter = sdk_export._span_exporter("http://localhost:4318")
         assert exporter._endpoint == "http://localhost:4318/v1/traces"
         assert exporter._headers == {"x-agentevals-sdk": "1"}
         assert "authorization" not in {k.lower() for k in exporter._session.headers}
-        assert exporter._client_key_file is None
-        assert exporter._client_certificate_file is None
         assert exporter._session.trust_env is False
 
+    @pytest.mark.parametrize("factory", ["_span_exporter", "_log_exporter"])
+    def test_exports_never_use_application_tls_settings(self, factory, monkeypatch):
+        import requests.adapters
+        from requests.models import Response
+
+        monkeypatch.setenv("OTEL_EXPORTER_OTLP_CERTIFICATE", "/etc/app/ca.pem")
+        monkeypatch.setenv("OTEL_EXPORTER_OTLP_CLIENT_KEY", "/etc/app/client.key")
+        monkeypatch.setenv("OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE", "/etc/app/client.crt")
+        monkeypatch.setenv("OTEL_EXPORTER_OTLP_TRACES_CLIENT_CERTIFICATE", "/etc/app/traces.crt")
+        monkeypatch.setenv("OTEL_EXPORTER_OTLP_LOGS_CLIENT_CERTIFICATE", "/etc/app/logs.crt")
+        sent = []
+
+        def send(adapter, request, **kwargs):
+            sent.append(kwargs)
+            response = Response()
+            response.status_code = 200
+            response.request = request
+            return response
+
+        monkeypatch.setattr(requests.adapters.HTTPAdapter, "send", send)
+        getattr(sdk_export, factory)("https://agentevals.example:4318")._export(b"payload")
+        assert len(sent) == 1
+        assert sent[0]["cert"] is None
+        assert sent[0]["verify"] is True
+
     def test_log_exporter_ignores_application_otlp_settings(self, monkeypatch):
         monkeypatch.setenv("OTEL_EXPORTER_OTLP_HEADERS", "authorization=Bearer app-token")
         monkeypatch.setenv("OTEL_EXPORTER_OTLP_LOGS_HEADERS", "x-api-key=app-key")