diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 536f3e3..73d21a7 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -10,12 +10,12 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: astral-sh/ruff-action@v3 - with: - args: check tee_gateway - - uses: astral-sh/ruff-action@v3 - with: - args: format --check tee_gateway + - uses: astral-sh/setup-uv@v5 + # Run ruff from the locked toolchain (uv.lock) rather than ruff-action's + # bundled latest release, so lint is reproducible and matches what + # developers run locally — the same reason the mypy job uses `uv run`. + - run: uv run ruff check tee_gateway + - run: uv run ruff format --check tee_gateway mypy: runs-on: ubuntu-latest diff --git a/tee_gateway/controllers/chat_controller.py b/tee_gateway/controllers/chat_controller.py index 1392b6a..5d042ee 100644 --- a/tee_gateway/controllers/chat_controller.py +++ b/tee_gateway/controllers/chat_controller.py @@ -131,6 +131,21 @@ def _build_tools_list(chat_request: CreateChatCompletionRequest, provider: str) return tools_list +def _needs_responses_api_for_tools(provider: str, cfg, tools_list: list) -> bool: + """Whether this request must use OpenAI's Responses API because of tools. + + The gpt-5.6 family (``responses_api_for_tools=True``) rejects function tools + combined with its default ``reasoning_effort`` on the Chat Completions + endpoint; the Responses API supports both together. Only relevant when + function tools are actually bound and the provider is OpenAI. + """ + return ( + provider == "openai" + and getattr(cfg, "responses_api_for_tools", False) + and bool(tools_list) + ) + + def _normalize_response_format(rf) -> dict: """Coerce response_format to a plain dict, preserving all fields including json_schema.""" if isinstance(rf, dict): @@ -235,6 +250,10 @@ def _create_non_streaming_response(chat_request: CreateChatCompletionRequest): if cfg.image_generation: return create_image_generation_response(chat_request, request_bytes) + # Build the tools list first: some OpenAI models (gpt-5.6 family) must be + # constructed against the Responses API when function tools are bound. + tools_list = _build_tools_list(chat_request, provider) + model = get_chat_model_cached( model=chat_request.model, temperature=float(chat_request.temperature) @@ -242,10 +261,12 @@ def _create_non_streaming_response(chat_request: CreateChatCompletionRequest): else 0.0, max_tokens=chat_request.max_tokens or 4096, web_search=bool(chat_request.web_search), + force_responses_api=_needs_responses_api_for_tools( + provider, cfg, tools_list + ), ) # Bind user tools and/or the native web search tool if requested. - tools_list = _build_tools_list(chat_request, provider) if tools_list: model = model.bind_tools(tools_list) @@ -377,12 +398,13 @@ def _create_streaming_response(chat_request: CreateChatCompletionRequest): """Handle streaming chat completion via direct LangChain call.""" try: provider = get_provider_from_model(chat_request.model) + cfg = get_model_config(chat_request.model) # OpenAI and Anthropic stream tool calls as fragments that must be # buffered and flushed once complete. Gemini emits complete tool calls. buffer_tool_calls = provider in ["openai", "anthropic"] # Gemini inline-image models return a single image rather than a token # stream — invoke once and emit the result inside the SSE envelope. - image_output_model = get_model_config(chat_request.model).image_output + image_output_model = cfg.image_output request_dict = _chat_request_to_dict(chat_request) request_bytes = json.dumps(request_dict, sort_keys=True).encode("utf-8") @@ -394,6 +416,10 @@ def _create_streaming_response(chat_request: CreateChatCompletionRequest): chat_request, request_bytes ) + # Build the tools list first: some OpenAI models (gpt-5.6 family) must be + # constructed against the Responses API when function tools are bound. + tools_list = _build_tools_list(chat_request, provider) + model = get_chat_model_cached( model=chat_request.model, temperature=float(chat_request.temperature) @@ -401,10 +427,12 @@ def _create_streaming_response(chat_request: CreateChatCompletionRequest): else 0.0, max_tokens=chat_request.max_tokens or 4096, web_search=bool(chat_request.web_search), + force_responses_api=_needs_responses_api_for_tools( + provider, cfg, tools_list + ), ) # Bind user tools and/or the native web search tool if requested. - tools_list = _build_tools_list(chat_request, provider) if tools_list: model = model.bind_tools(tools_list) diff --git a/tee_gateway/llm_backend.py b/tee_gateway/llm_backend.py index da7031c..b358957 100644 --- a/tee_gateway/llm_backend.py +++ b/tee_gateway/llm_backend.py @@ -157,17 +157,28 @@ def get_provider_from_model(model: str) -> str: @lru_cache(maxsize=64) def get_chat_model_cached( - model: str, temperature: float, max_tokens: int, web_search: bool = False + model: str, + temperature: float, + max_tokens: int, + web_search: bool = False, + force_responses_api: bool = False, ): """Get cached chat model instance using the injected ProviderConfig. - Models are cached by (model, temperature, max_tokens, web_search) tuple. - Cache is cleared by set_provider_config() after key injection. + Models are cached by (model, temperature, max_tokens, web_search, + force_responses_api) tuple. Cache is cleared by set_provider_config() after + key injection. When ``web_search`` is True, provider-specific native web search is enabled. Some providers (OpenAI, xAI) require search configuration at construction time; others (Anthropic, Google) enable it by binding a tool — see ``get_web_search_tool``. Providers without native web search ignore the flag. + + When ``force_responses_api`` is True, OpenAI models are constructed against + the Responses API (like the web-search path). The gpt-5.6 family rejects + function tools combined with its default ``reasoning_effort`` on Chat + Completions, so the chat controller sets this flag for those models when + tools are bound. Ignored by non-OpenAI providers. """ config = _provider_config if config is None: @@ -214,12 +225,14 @@ def get_chat_model_cached( if openai_http_client is None: raise RuntimeError("OpenAI HTTP client has not been initialized") - # OpenAI's built-in web_search tool is only available through the - # Responses API, so switch to it when web search is requested. The + # Switch to the Responses API when either (a) web search is requested — + # OpenAI's built-in web_search tool is only available there — or (b) the + # caller forced it because this model rejects function tools + its + # default reasoning_effort on Chat Completions (gpt-5.6 family). The # responses/v1 output format surfaces web_search_call items in the # message content, which we count for billing. openai_kwargs: dict[str, Any] = {} - if web_search: + if web_search or force_responses_api: openai_kwargs["use_responses_api"] = True openai_kwargs["output_version"] = "responses/v1" diff --git a/tee_gateway/model_registry.py b/tee_gateway/model_registry.py index 9e3d208..4525143 100644 --- a/tee_gateway/model_registry.py +++ b/tee_gateway/model_registry.py @@ -72,6 +72,15 @@ class ModelConfig: # means "use the provider default" (see WEB_SEARCH_PRICE_USD_BY_PROVIDER); # set an explicit value here to override a single model's web-search price. web_search_price_usd: Optional[Decimal] = None + # OpenAI's newest reasoning models (the gpt-5.6 family) apply a default + # ``reasoning_effort`` that the Chat Completions endpoint rejects when + # function tools are also present ("Function tools with reasoning_effort are + # not supported ... use /v1/responses or set reasoning_effort to 'none'"). + # When True and function tools are bound, the gateway routes the request + # through OpenAI's Responses API instead, which supports reasoning and + # function tools together (preserving reasoning rather than disabling it). + # OpenAI provider only; ignored elsewhere. + responses_api_for_tools: bool = False # Default per-search USD price charged when a model uses native web search. @@ -183,18 +192,21 @@ class SupportedModel(Enum): api_name="gpt-5.6-sol", input_price_usd=Decimal("0.000005"), output_price_usd=Decimal("0.00003"), + responses_api_for_tools=True, ) GPT_5_6_TERRA = ModelConfig( provider="openai", api_name="gpt-5.6-terra", input_price_usd=Decimal("0.0000025"), output_price_usd=Decimal("0.000015"), + responses_api_for_tools=True, ) GPT_5_6_LUNA = ModelConfig( provider="openai", api_name="gpt-5.6-luna", input_price_usd=Decimal("0.000001"), output_price_usd=Decimal("0.000006"), + responses_api_for_tools=True, ) # Image generation via OpenAI's /images/generations endpoint (gpt-image). # Unlike DALL·E, gpt-image models always return base64 (``b64_json``) and diff --git a/tee_gateway/test/test_tool_forwarding.py b/tee_gateway/test/test_tool_forwarding.py index f4ca16f..87e1615 100644 --- a/tee_gateway/test/test_tool_forwarding.py +++ b/tee_gateway/test/test_tool_forwarding.py @@ -275,6 +275,98 @@ def test_tee_metadata_in_response( # tee_id must carry the 0x prefix self.assertTrue(result["tee_id"].startswith("0x")) + @patch("tee_gateway.controllers.chat_controller.get_tee_keys") + @patch("tee_gateway.controllers.chat_controller.get_chat_model_cached") + @patch("tee_gateway.controllers.chat_controller.connexion") + def test_gpt56_with_tools_forces_responses_api( + self, mock_connexion, mock_get_model, mock_get_tee_keys + ): + """gpt-5.6 models must be built against the Responses API when tools are + bound (Chat Completions rejects tools + default reasoning_effort).""" + mock_connexion.request.is_json = True + mock_connexion.request.get_json.return_value = { + "model": "gpt-5.6-luna", + "messages": [{"role": "user", "content": "What is the weather?"}], + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather for a location", + "parameters": { + "type": "object", + "properties": {"location": {"type": "string"}}, + }, + }, + } + ], + "stream": False, + } + + mock_model = _make_mock_model(_MockLangChainResponse(content="Sunny.")) + mock_get_model.return_value = mock_model + mock_get_tee_keys.return_value = _make_mock_tee_keys() + + create_chat_completion(None) + + self.assertTrue(mock_get_model.call_args.kwargs["force_responses_api"]) + mock_model.bind_tools.assert_called_once() + + @patch("tee_gateway.controllers.chat_controller.get_tee_keys") + @patch("tee_gateway.controllers.chat_controller.get_chat_model_cached") + @patch("tee_gateway.controllers.chat_controller.connexion") + def test_gpt56_without_tools_stays_on_chat_completions( + self, mock_connexion, mock_get_model, mock_get_tee_keys + ): + """gpt-5.6 without tools has no conflict, so the Responses API is not forced.""" + mock_connexion.request.is_json = True + mock_connexion.request.get_json.return_value = { + "model": "gpt-5.6-luna", + "messages": [{"role": "user", "content": "Hello"}], + "stream": False, + } + + mock_get_model.return_value = _make_mock_model( + _MockLangChainResponse(content="Hi!") + ) + mock_get_tee_keys.return_value = _make_mock_tee_keys() + + create_chat_completion(None) + + self.assertFalse(mock_get_model.call_args.kwargs["force_responses_api"]) + + @patch("tee_gateway.controllers.chat_controller.get_tee_keys") + @patch("tee_gateway.controllers.chat_controller.get_chat_model_cached") + @patch("tee_gateway.controllers.chat_controller.connexion") + def test_non_reasoning_model_with_tools_stays_on_chat_completions( + self, mock_connexion, mock_get_model, mock_get_tee_keys + ): + """A non-gpt-5.6 model (gpt-4.1) with tools must NOT force the Responses API.""" + mock_connexion.request.is_json = True + mock_connexion.request.get_json.return_value = { + "model": "gpt-4.1", + "messages": [{"role": "user", "content": "What is the weather?"}], + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "parameters": {"type": "object", "properties": {}}, + }, + } + ], + "stream": False, + } + + mock_get_model.return_value = _make_mock_model( + _MockLangChainResponse(content="Sunny.") + ) + mock_get_tee_keys.return_value = _make_mock_tee_keys() + + create_chat_completion(None) + + self.assertFalse(mock_get_model.call_args.kwargs["force_responses_api"]) + @patch("tee_gateway.controllers.chat_controller.get_tee_keys") @patch("tee_gateway.controllers.chat_controller.get_chat_model_cached") @patch("tee_gateway.controllers.chat_controller.connexion")