From c31de3830097911eb8cb95f503f469145bdad905 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 24 Jul 2026 10:56:16 +0000 Subject: [PATCH 1/2] Route gpt-5.6 models through Responses API when function tools are bound MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit OpenAI's gpt-5.6 family (sol/terra/luna) applies a default reasoning_effort that the Chat Completions endpoint rejects when function tools are also present: Function tools with reasoning_effort are not supported for gpt-5.6-luna in /v1/chat/completions. To use function tools, use /v1/responses or set reasoning_effort to 'none'. This broke the chat-app agent page, where tools are always bound. Fix: add a `responses_api_for_tools` flag to ModelConfig (set on the gpt-5.6 family) and, when function tools are bound for such a model, construct the OpenAI chat model against the Responses API (`use_responses_api=True`, `output_version="responses/v1"`) — the same path already used for native web search. The Responses API supports reasoning and function tools together, so reasoning is preserved rather than disabled. Non-gpt-5.6 models and tool-less gpt-5.6 requests keep using Chat Completions, so behavior is unchanged outside the failing case. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01KBaBPmkTvezKBQnPoHqNFn --- tee_gateway/controllers/chat_controller.py | 34 +++++++- tee_gateway/llm_backend.py | 25 ++++-- tee_gateway/model_registry.py | 12 +++ tee_gateway/test/test_tool_forwarding.py | 92 ++++++++++++++++++++++ 4 files changed, 154 insertions(+), 9 deletions(-) diff --git a/tee_gateway/controllers/chat_controller.py b/tee_gateway/controllers/chat_controller.py index 1392b6a..5d042ee 100644 --- a/tee_gateway/controllers/chat_controller.py +++ b/tee_gateway/controllers/chat_controller.py @@ -131,6 +131,21 @@ def _build_tools_list(chat_request: CreateChatCompletionRequest, provider: str) return tools_list +def _needs_responses_api_for_tools(provider: str, cfg, tools_list: list) -> bool: + """Whether this request must use OpenAI's Responses API because of tools. + + The gpt-5.6 family (``responses_api_for_tools=True``) rejects function tools + combined with its default ``reasoning_effort`` on the Chat Completions + endpoint; the Responses API supports both together. Only relevant when + function tools are actually bound and the provider is OpenAI. + """ + return ( + provider == "openai" + and getattr(cfg, "responses_api_for_tools", False) + and bool(tools_list) + ) + + def _normalize_response_format(rf) -> dict: """Coerce response_format to a plain dict, preserving all fields including json_schema.""" if isinstance(rf, dict): @@ -235,6 +250,10 @@ def _create_non_streaming_response(chat_request: CreateChatCompletionRequest): if cfg.image_generation: return create_image_generation_response(chat_request, request_bytes) + # Build the tools list first: some OpenAI models (gpt-5.6 family) must be + # constructed against the Responses API when function tools are bound. + tools_list = _build_tools_list(chat_request, provider) + model = get_chat_model_cached( model=chat_request.model, temperature=float(chat_request.temperature) @@ -242,10 +261,12 @@ def _create_non_streaming_response(chat_request: CreateChatCompletionRequest): else 0.0, max_tokens=chat_request.max_tokens or 4096, web_search=bool(chat_request.web_search), + force_responses_api=_needs_responses_api_for_tools( + provider, cfg, tools_list + ), ) # Bind user tools and/or the native web search tool if requested. - tools_list = _build_tools_list(chat_request, provider) if tools_list: model = model.bind_tools(tools_list) @@ -377,12 +398,13 @@ def _create_streaming_response(chat_request: CreateChatCompletionRequest): """Handle streaming chat completion via direct LangChain call.""" try: provider = get_provider_from_model(chat_request.model) + cfg = get_model_config(chat_request.model) # OpenAI and Anthropic stream tool calls as fragments that must be # buffered and flushed once complete. Gemini emits complete tool calls. buffer_tool_calls = provider in ["openai", "anthropic"] # Gemini inline-image models return a single image rather than a token # stream — invoke once and emit the result inside the SSE envelope. - image_output_model = get_model_config(chat_request.model).image_output + image_output_model = cfg.image_output request_dict = _chat_request_to_dict(chat_request) request_bytes = json.dumps(request_dict, sort_keys=True).encode("utf-8") @@ -394,6 +416,10 @@ def _create_streaming_response(chat_request: CreateChatCompletionRequest): chat_request, request_bytes ) + # Build the tools list first: some OpenAI models (gpt-5.6 family) must be + # constructed against the Responses API when function tools are bound. + tools_list = _build_tools_list(chat_request, provider) + model = get_chat_model_cached( model=chat_request.model, temperature=float(chat_request.temperature) @@ -401,10 +427,12 @@ def _create_streaming_response(chat_request: CreateChatCompletionRequest): else 0.0, max_tokens=chat_request.max_tokens or 4096, web_search=bool(chat_request.web_search), + force_responses_api=_needs_responses_api_for_tools( + provider, cfg, tools_list + ), ) # Bind user tools and/or the native web search tool if requested. - tools_list = _build_tools_list(chat_request, provider) if tools_list: model = model.bind_tools(tools_list) diff --git a/tee_gateway/llm_backend.py b/tee_gateway/llm_backend.py index da7031c..b358957 100644 --- a/tee_gateway/llm_backend.py +++ b/tee_gateway/llm_backend.py @@ -157,17 +157,28 @@ def get_provider_from_model(model: str) -> str: @lru_cache(maxsize=64) def get_chat_model_cached( - model: str, temperature: float, max_tokens: int, web_search: bool = False + model: str, + temperature: float, + max_tokens: int, + web_search: bool = False, + force_responses_api: bool = False, ): """Get cached chat model instance using the injected ProviderConfig. - Models are cached by (model, temperature, max_tokens, web_search) tuple. - Cache is cleared by set_provider_config() after key injection. + Models are cached by (model, temperature, max_tokens, web_search, + force_responses_api) tuple. Cache is cleared by set_provider_config() after + key injection. When ``web_search`` is True, provider-specific native web search is enabled. Some providers (OpenAI, xAI) require search configuration at construction time; others (Anthropic, Google) enable it by binding a tool — see ``get_web_search_tool``. Providers without native web search ignore the flag. + + When ``force_responses_api`` is True, OpenAI models are constructed against + the Responses API (like the web-search path). The gpt-5.6 family rejects + function tools combined with its default ``reasoning_effort`` on Chat + Completions, so the chat controller sets this flag for those models when + tools are bound. Ignored by non-OpenAI providers. """ config = _provider_config if config is None: @@ -214,12 +225,14 @@ def get_chat_model_cached( if openai_http_client is None: raise RuntimeError("OpenAI HTTP client has not been initialized") - # OpenAI's built-in web_search tool is only available through the - # Responses API, so switch to it when web search is requested. The + # Switch to the Responses API when either (a) web search is requested — + # OpenAI's built-in web_search tool is only available there — or (b) the + # caller forced it because this model rejects function tools + its + # default reasoning_effort on Chat Completions (gpt-5.6 family). The # responses/v1 output format surfaces web_search_call items in the # message content, which we count for billing. openai_kwargs: dict[str, Any] = {} - if web_search: + if web_search or force_responses_api: openai_kwargs["use_responses_api"] = True openai_kwargs["output_version"] = "responses/v1" diff --git a/tee_gateway/model_registry.py b/tee_gateway/model_registry.py index 9e3d208..4525143 100644 --- a/tee_gateway/model_registry.py +++ b/tee_gateway/model_registry.py @@ -72,6 +72,15 @@ class ModelConfig: # means "use the provider default" (see WEB_SEARCH_PRICE_USD_BY_PROVIDER); # set an explicit value here to override a single model's web-search price. web_search_price_usd: Optional[Decimal] = None + # OpenAI's newest reasoning models (the gpt-5.6 family) apply a default + # ``reasoning_effort`` that the Chat Completions endpoint rejects when + # function tools are also present ("Function tools with reasoning_effort are + # not supported ... use /v1/responses or set reasoning_effort to 'none'"). + # When True and function tools are bound, the gateway routes the request + # through OpenAI's Responses API instead, which supports reasoning and + # function tools together (preserving reasoning rather than disabling it). + # OpenAI provider only; ignored elsewhere. + responses_api_for_tools: bool = False # Default per-search USD price charged when a model uses native web search. @@ -183,18 +192,21 @@ class SupportedModel(Enum): api_name="gpt-5.6-sol", input_price_usd=Decimal("0.000005"), output_price_usd=Decimal("0.00003"), + responses_api_for_tools=True, ) GPT_5_6_TERRA = ModelConfig( provider="openai", api_name="gpt-5.6-terra", input_price_usd=Decimal("0.0000025"), output_price_usd=Decimal("0.000015"), + responses_api_for_tools=True, ) GPT_5_6_LUNA = ModelConfig( provider="openai", api_name="gpt-5.6-luna", input_price_usd=Decimal("0.000001"), output_price_usd=Decimal("0.000006"), + responses_api_for_tools=True, ) # Image generation via OpenAI's /images/generations endpoint (gpt-image). # Unlike DALL·E, gpt-image models always return base64 (``b64_json``) and diff --git a/tee_gateway/test/test_tool_forwarding.py b/tee_gateway/test/test_tool_forwarding.py index f4ca16f..87e1615 100644 --- a/tee_gateway/test/test_tool_forwarding.py +++ b/tee_gateway/test/test_tool_forwarding.py @@ -275,6 +275,98 @@ def test_tee_metadata_in_response( # tee_id must carry the 0x prefix self.assertTrue(result["tee_id"].startswith("0x")) + @patch("tee_gateway.controllers.chat_controller.get_tee_keys") + @patch("tee_gateway.controllers.chat_controller.get_chat_model_cached") + @patch("tee_gateway.controllers.chat_controller.connexion") + def test_gpt56_with_tools_forces_responses_api( + self, mock_connexion, mock_get_model, mock_get_tee_keys + ): + """gpt-5.6 models must be built against the Responses API when tools are + bound (Chat Completions rejects tools + default reasoning_effort).""" + mock_connexion.request.is_json = True + mock_connexion.request.get_json.return_value = { + "model": "gpt-5.6-luna", + "messages": [{"role": "user", "content": "What is the weather?"}], + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather for a location", + "parameters": { + "type": "object", + "properties": {"location": {"type": "string"}}, + }, + }, + } + ], + "stream": False, + } + + mock_model = _make_mock_model(_MockLangChainResponse(content="Sunny.")) + mock_get_model.return_value = mock_model + mock_get_tee_keys.return_value = _make_mock_tee_keys() + + create_chat_completion(None) + + self.assertTrue(mock_get_model.call_args.kwargs["force_responses_api"]) + mock_model.bind_tools.assert_called_once() + + @patch("tee_gateway.controllers.chat_controller.get_tee_keys") + @patch("tee_gateway.controllers.chat_controller.get_chat_model_cached") + @patch("tee_gateway.controllers.chat_controller.connexion") + def test_gpt56_without_tools_stays_on_chat_completions( + self, mock_connexion, mock_get_model, mock_get_tee_keys + ): + """gpt-5.6 without tools has no conflict, so the Responses API is not forced.""" + mock_connexion.request.is_json = True + mock_connexion.request.get_json.return_value = { + "model": "gpt-5.6-luna", + "messages": [{"role": "user", "content": "Hello"}], + "stream": False, + } + + mock_get_model.return_value = _make_mock_model( + _MockLangChainResponse(content="Hi!") + ) + mock_get_tee_keys.return_value = _make_mock_tee_keys() + + create_chat_completion(None) + + self.assertFalse(mock_get_model.call_args.kwargs["force_responses_api"]) + + @patch("tee_gateway.controllers.chat_controller.get_tee_keys") + @patch("tee_gateway.controllers.chat_controller.get_chat_model_cached") + @patch("tee_gateway.controllers.chat_controller.connexion") + def test_non_reasoning_model_with_tools_stays_on_chat_completions( + self, mock_connexion, mock_get_model, mock_get_tee_keys + ): + """A non-gpt-5.6 model (gpt-4.1) with tools must NOT force the Responses API.""" + mock_connexion.request.is_json = True + mock_connexion.request.get_json.return_value = { + "model": "gpt-4.1", + "messages": [{"role": "user", "content": "What is the weather?"}], + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "parameters": {"type": "object", "properties": {}}, + }, + } + ], + "stream": False, + } + + mock_get_model.return_value = _make_mock_model( + _MockLangChainResponse(content="Sunny.") + ) + mock_get_tee_keys.return_value = _make_mock_tee_keys() + + create_chat_completion(None) + + self.assertFalse(mock_get_model.call_args.kwargs["force_responses_api"]) + @patch("tee_gateway.controllers.chat_controller.get_tee_keys") @patch("tee_gateway.controllers.chat_controller.get_chat_model_cached") @patch("tee_gateway.controllers.chat_controller.connexion") From 9807ff2afa9b5f9e8e98c51eca568a1229a5cc47 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 24 Jul 2026 11:01:24 +0000 Subject: [PATCH 2/2] CI: run ruff from the locked toolchain instead of ruff-action's latest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ruff lint job used astral-sh/ruff-action@v3, which pulls its own latest ruff release (currently 0.16.0) instead of the version pinned in uv.lock (0.15.20). The newer ruff enabled rules (RUF013, UP006, RUF059, I001) that flag pre-existing violations, and the action's default `src` of "." caused it to lint the whole repo (including tests/) rather than the `tee_gateway` target the job's args declared — so lint failed non-deterministically on code unrelated to any given change. Run ruff via `uv run` like the mypy job already does, so both linters use the locked toolchain (uv.lock as the single source of truth) and lint is reproducible and matches what developers run locally. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01KBaBPmkTvezKBQnPoHqNFn --- .github/workflows/lint.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 536f3e3..73d21a7 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -10,12 +10,12 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: astral-sh/ruff-action@v3 - with: - args: check tee_gateway - - uses: astral-sh/ruff-action@v3 - with: - args: format --check tee_gateway + - uses: astral-sh/setup-uv@v5 + # Run ruff from the locked toolchain (uv.lock) rather than ruff-action's + # bundled latest release, so lint is reproducible and matches what + # developers run locally — the same reason the mypy job uses `uv run`. + - run: uv run ruff check tee_gateway + - run: uv run ruff format --check tee_gateway mypy: runs-on: ubuntu-latest