fix: prefer tool output for OpenAI Opus 5 TP4

README.md

@@ -305,8 +305,9 @@ Set `SKILLSPECTOR_REASONING_EFFORT` to override it with `low`, `high`, or `max`.
 The bundled 128,000-token context and 32,000-token output budgets are conservative
 application limits; they do not claim the hosted endpoint's maximum capacity.
 
-Structured output is requested through LangChain's `with_structured_output`,
-whose default forces a tool call. Some models reject a forced tool call with
+Structured output is requested through LangChain's `with_structured_output`.
+The default is client-specific: `ChatOpenAI` uses `json_schema`, while other
+clients may use tool calling. Some models reject a forced tool call with
 HTTP 400 (`tool_choice: type "tool" and "any" are not supported for this
 model`). The `anthropic` and `anthropic_proxy` providers route those models
 (`claude-fable-5-1`, `claude-mythos-5-1`, or any registry entry with
@@ -322,6 +323,12 @@ for endpoints that ignore both `response_format` and a forced `tool_choice`
 and answer in prose (for example iFlytek's `spark-x2.5`, which is bundled).
 `SKILLSPECTOR_STRUCTURED_OUTPUT_METHOD=json_schema|function_calling`
 overrides the method for any provider.
+The method precedence is `SKILLSPECTOR_STRUCTURED_OUTPUT_METHOD`, the provider's
+method hint, an analyzer preference, then the client default. TP4 prefers
+`function_calling` only for provider `openai` and the exact model label
+`azure/anthropic/claude-opus-5`, with reasoning/thinking controls unset; dated
+or suffixed labels and other providers keep their existing behavior. An explicit
+environment override or provider hint still wins when reasoning is configured.
 
 ```bash
 # Stock OpenAI

src/skillspector/llm_analyzer_base.py

@@ -951,7 +951,7 @@ def __init__(
             max_retries=native_retries,
         )
         self._structured_llm = (
-            bind_structured_output(self._llm, self.response_schema, model)
+            self._bind_structured_output(self._llm, self.response_schema)
             if self.response_schema
             else None
         )
@@ -962,6 +962,18 @@ def __init__(
             chat_model=self._llm,
         )
 
+    def _structured_output_preference(self, llm: object) -> str | None:
+        """Return an analyzer-specific binding preference, if any."""
+        return None
+
+    def _bind_structured_output(self, llm: object, schema: type) -> object:
+        return bind_structured_output(
+            llm,
+            schema,
+            self.model,
+            preferred_method=self._structured_output_preference(llm),
+        )
+
     def _remaining_timeout(self) -> float | None:
         if callable(self._timeout):
             remaining = self._timeout()
@@ -1022,7 +1034,7 @@ def _model_for_call(self) -> tuple[object, object | None]:
             chat_model_controls(llm),
         )
         structured = (
-            bind_structured_output(llm, self.response_schema, self.model)
+            self._bind_structured_output(llm, self.response_schema)
             if self.response_schema
             else None
         )

src/skillspector/llm_utils.py

@@ -45,8 +45,10 @@
 from typing import Any, NoReturn
 
 from google.auth.exceptions import RefreshError
+from langchain_core.exceptions import OutputParserException
 from langchain_core.language_models.chat_models import BaseChatModel
-from langchain_core.runnables import Runnable, RunnableLambda
+from langchain_core.runnables import Runnable, RunnableConfig, RunnableLambda
+from langchain_core.utils.function_calling import convert_to_openai_tool
 
 from skillspector.inference_usage import (
     InferenceUsageCollector,
@@ -381,15 +383,26 @@ def set_timeout(self, timeout: float | None) -> None:
 STRUCTURED_OUTPUT_METHODS = ("function_calling", "json_schema")
 
 
-def structured_output_kwargs(model: str, provider: object | None = None) -> dict[str, str]:
+def structured_output_kwargs(
+    model: str,
+    provider: object | None = None,
+    *,
+    preferred_method: str | None = None,
+) -> dict[str, str]:
     """Keyword arguments for ``with_structured_output`` when binding a schema for *model*.
 
     ``SKILLSPECTOR_STRUCTURED_OUTPUT_METHOD`` wins, then the active provider's
-    ``structured_output_method(model)`` hint, else LangChain's default (no kwargs).
+    ``structured_output_method(model)`` hint, then an analyzer preference, else
+    LangChain's default (no kwargs).
 
     Raises:
-        ValueError: when the environment override is not a known method.
+        ValueError: when the environment override or preference is not a known method.
     """
+    if preferred_method is not None and preferred_method not in STRUCTURED_OUTPUT_METHODS:
+        raise ValueError(
+            "preferred structured output method must be one of "
+            f"{', '.join(STRUCTURED_OUTPUT_METHODS)}; got {preferred_method!r}"
+        )
     override = os.environ.get("SKILLSPECTOR_STRUCTURED_OUTPUT_METHOD", "").strip().lower()
     if override:
         if override not in STRUCTURED_OUTPUT_METHODS:
@@ -402,11 +415,19 @@ def structured_output_kwargs(model: str, provider: object | None = None) -> dict
         provider = get_active_provider()
     hint = getattr(provider, "structured_output_method", None)
     method = hint(model) if callable(hint) else None
+    if method:
+        return {"method": method}
+    method = preferred_method
     return {"method": method} if method else {}
 
 
 def bind_structured_output(
-    llm: object, schema: type, model: str, provider: object | None = None
+    llm: object,
+    schema: type,
+    model: str,
+    provider: object | None = None,
+    *,
+    preferred_method: str | None = None,
 ) -> object:
     """``llm.with_structured_output(schema)`` with the method *model* needs.
 
@@ -416,11 +437,24 @@ def bind_structured_output(
     the prompt, and a prose answer raises :class:`StructuredOutputParseError`,
     which the analyzers retry like any other malformed structured response.
     """
-    kwargs = structured_output_kwargs(model, provider)
-    structured = llm.with_structured_output(schema, **kwargs)  # type: ignore[attr-defined]
-    if not _binds_unforced_tool_call(llm, kwargs.get("method")):
-        return structured
-    return _require_tool_call(structured, schema)
+    kwargs = structured_output_kwargs(model, provider, preferred_method=preferred_method)
+    unforced = _binds_unforced_tool_call(llm, kwargs.get("method"))
+    include_raw = isinstance(llm, BaseChatModel) and (
+        unforced or kwargs.get("method") == "function_calling"
+    )
+    binding_kwargs: dict[str, Any] = dict(kwargs)
+    if include_raw:
+        # Native parsers may silently keep only the first tool call. Retain the
+        # raw message so validation can reject extra or invalid calls instead.
+        binding_kwargs["include_raw"] = True
+    structured = llm.with_structured_output(schema, **binding_kwargs)  # type: ignore[attr-defined]
+    if unforced:
+        return _require_tool_call(structured, schema, include_raw=include_raw)
+    if kwargs.get("method") == "function_calling" and not isinstance(
+        structured, _StructuredAgentCLIModel
+    ):
+        return _validate_tool_call_result(structured, schema, include_raw=include_raw)
+    return structured
 
 
 def _binds_unforced_tool_call(llm: object, method: str | None) -> bool:
@@ -446,21 +480,82 @@ def _binds_unforced_tool_call(llm: object, method: str | None) -> bool:
 )
 
 
-def _require_tool_call(structured: Runnable, schema: type) -> Runnable:
+def _require_tool_call(
+    structured: Runnable, schema: type, *, include_raw: bool = False
+) -> Runnable:
     """Ask for the tool call in the prompt and fail closed when it does not happen."""
-    tool = schema.__name__ if isinstance(schema, type) else "response"
+    tool = (
+        convert_to_openai_tool(schema)["function"]["name"]
+        if include_raw
+        else schema.__name__
+        if isinstance(schema, type)
+        else "response"
+    )
 
     def _ask(prompt: str) -> str:
         return f"{prompt}\n\n{_TOOL_CALL_INSTRUCTION.format(tool=tool)}"
 
+    return RunnableLambda(_ask) | _validate_tool_call_result(
+        structured, schema, include_raw=include_raw
+    )
+
+
+def _validate_tool_call_result(
+    structured: Runnable, schema: type, *, include_raw: bool = False
+) -> Runnable:
+    """Validate forced/auto tool results without changing the bound model prompt."""
+    tool = (
+        convert_to_openai_tool(schema)["function"]["name"]
+        if include_raw
+        else schema.__name__
+        if isinstance(schema, type)
+        else "response"
+    )
+
     def _check(result: object) -> object:
+        if include_raw:
+            if not isinstance(result, dict):
+                raise StructuredOutputParseError(f"model returned invalid {tool} tool output")
+            raw = result.get("raw")
+            calls = getattr(raw, "tool_calls", None)
+            if (
+                not isinstance(calls, list)
+                or len(calls) != 1
+                or calls[0].get("name") != tool
+                or getattr(raw, "invalid_tool_calls", None)
+            ):
+                raise StructuredOutputParseError(
+                    f"model must return exactly one valid {tool} tool call"
+                )
+            error = result.get("parsing_error")
+            if error is not None:
+                raise StructuredOutputParseError(
+                    f"model returned invalid {tool} tool arguments"
+                ) from error
+            result = result.get("parsed")
         if result is None:
             raise StructuredOutputParseError(
                 f"model answered in prose instead of calling the {tool} tool"
             )
         return result
 
-    return RunnableLambda(_ask) | structured | RunnableLambda(_check)
+    def _invoke(prompt: Any, config: RunnableConfig) -> object:
+        try:
+            return _check(structured.invoke(prompt, config=config))
+        except OutputParserException as exc:
+            raise StructuredOutputParseError(
+                f"model returned invalid {tool} tool arguments"
+            ) from exc
+
+    async def _ainvoke(prompt: Any, config: RunnableConfig) -> object:
+        try:
+            return _check(await structured.ainvoke(prompt, config=config))
+        except OutputParserException as exc:
+            raise StructuredOutputParseError(
+                f"model returned invalid {tool} tool arguments"
+            ) from exc
+
+    return RunnableLambda(_invoke, _ainvoke)
 
 
 def get_chat_model(

src/skillspector/nodes/analyzers/mcp_tool_poisoning.py

@@ -22,13 +22,14 @@
 import re
 import time
 import unicodedata
-from collections.abc import Callable, Iterator
+from collections.abc import Callable, Iterator, Mapping
 from dataclasses import dataclass, field
 from typing import cast
 
+from langchain_openai.chat_models.base import BaseChatOpenAI
 from pydantic import BaseModel, Field, field_validator
 
-from skillspector.inference_usage import InferenceUsageRecord
+from skillspector.inference_usage import InferenceUsageRecord, chat_model_controls
 from skillspector.inspection_ledger import (
     InspectionLedgerEvent,
     LedgerOutcome,
@@ -45,7 +46,7 @@
     append_output_language_instruction,
     estimate_tokens,
 )
-from skillspector.llm_utils import run_async
+from skillspector.llm_utils import chat_model_provider_name, run_async
 from skillspector.model_info import get_max_input_tokens
 from skillspector.models import Finding, compute_match_fingerprint
 from skillspector.nodes.analyzers.static_runner import MAX_FINDINGS_PER_ANALYZER
@@ -55,6 +56,7 @@
     padding_run_match_fingerprint,
 )
 from skillspector.providers import get_active_provider
+from skillspector.providers.chat_models import resolve_reasoning_effort
 from skillspector.state import (
     AnalyzerNodeResponse,
     LLMCallRecord,
@@ -879,6 +881,28 @@ def _check_tp3(
     {"python", "javascript", "typescript", "shell", "ruby", "go", "rust"}
 )
 
+# Track the exact-label gateway workaround in the public PR:
+# https://github.com/NVIDIA/SkillSpector/pull/691
+# Dated/suffixed aliases are deliberately excluded. Only the openai route has
+# reproduced wrapper evidence; other providers retain their own method hints.
+_TP4_TOOL_OUTPUT_MODELS = frozenset({"azure/anthropic/claude-opus-5"})
+
+
+def _tp4_reasoning_configured(llm: BaseChatOpenAI) -> bool:
+    """Avoid an automatic forced tool choice when reasoning controls are set."""
+    if resolve_reasoning_effort() is not None:
+        return True
+    if chat_model_controls(llm).get("reasoning_effort") is not None:
+        return True
+    if any(getattr(llm, name, None) is not None for name in ("reasoning_effort", "reasoning")):
+        return True
+    for controls in (llm.model_kwargs, llm.extra_body):
+        if isinstance(controls, Mapping) and any(
+            controls.get(name) is not None for name in ("reasoning_effort", "reasoning", "thinking")
+        ):
+            return True
+    return False
+
 
 class _TP4AnalysisResult(BaseModel):
     """Validated response from the description-behavior mismatch check."""
@@ -903,6 +927,21 @@ class _TP4Analyzer(LLMAnalyzerBase):
 
     response_schema = _TP4AnalysisResult
 
+    def _structured_output_preference(self, llm: object) -> str | None:
+        # The OpenAI-compatible Opus 5 gateway sometimes wraps json_schema
+        # output in a {"json": ...} object instead of the requested schema.
+        # Tool calling returns the exact TP4 schema on the same gateway, but
+        # forcing a tool may conflict with configured reasoning/thinking.
+        # Explicit environment/provider method choices still take precedence.
+        if (
+            isinstance(llm, BaseChatOpenAI)
+            and chat_model_provider_name(llm) == "openai"
+            and self.model in _TP4_TOOL_OUTPUT_MODELS
+            and not _tp4_reasoning_configured(llm)
+        ):
+            return "function_calling"
+        return None
+
     def __init__(
         self,
         model: str,