diff --git a/backend/app/core/batch/anthropic.py b/backend/app/core/batch/anthropic.py index dfcbcfd3c..0a6e6e888 100644 --- a/backend/app/core/batch/anthropic.py +++ b/backend/app/core/batch/anthropic.py @@ -1,5 +1,6 @@ """Anthropic batch provider implementation.""" +import json import logging from enum import Enum from typing import Any @@ -15,6 +16,8 @@ logger = logging.getLogger(__name__) +STRUCTURED_OUTPUTS_BETA = "structured-outputs-2025-12-15" + class MessageBatchStatus(str, Enum): IN_PROGRESS = "in_progress" @@ -67,13 +70,19 @@ def create_batch( try: requests = [] + needs_structured_outputs = False for item in jsonl_data: params = {**item.get("params", {})} params["model"] = params.get("model") or default_model params["max_tokens"] = params.get("max_tokens") or default_max_tokens + if "output_config" in params: + needs_structured_outputs = True requests.append({"custom_id": item[BATCH_KEY], "params": params}) - batch = self.client.messages.batches.create(requests=requests) + betas = [STRUCTURED_OUTPUTS_BETA] if needs_structured_outputs else [] + batch = self.client.beta.messages.batches.create( + requests=requests, betas=betas + ) result = { "provider_batch_id": batch.id, diff --git a/backend/app/crud/assessment/batch.py b/backend/app/crud/assessment/batch.py index 54dc6cf01..2c2416086 100644 --- a/backend/app/crud/assessment/batch.py +++ b/backend/app/crud/assessment/batch.py @@ -29,7 +29,7 @@ ) from app.models.batch_job import BatchJob, BatchJobType from app.models.evaluation import EvaluationDataset -from app.models.llm.constants import DEFAULT_ANTHROPIC_MAX_TOKENS +from app.models.llm.constants import DEFAULT_ASSESSMENT_BATCH_MAX_TOKENS from app.models.llm.request import ConfigBlob from app.services.assessment.mappers import ( map_kaapi_to_anthropic_params, @@ -530,7 +530,7 @@ def submit_assessment_batch( batch_config = { "model": mapped_params.get("model"), "max_tokens": mapped_params.get("max_tokens") - or DEFAULT_ANTHROPIC_MAX_TOKENS, + or DEFAULT_ASSESSMENT_BATCH_MAX_TOKENS, } batch_job = start_batch_job( diff --git a/backend/app/models/llm/constants.py b/backend/app/models/llm/constants.py index 021ba0909..ee72138dc 100644 --- a/backend/app/models/llm/constants.py +++ b/backend/app/models/llm/constants.py @@ -78,6 +78,8 @@ class Modality(StrEnum): DEFAULT_ANTHROPIC_MAX_TOKENS = 4096 +DEFAULT_ASSESSMENT_BATCH_MAX_TOKENS = 16384 + # Provider-native STT/TTS defaults (used when caller omits model). DEFAULT_SARVAM_STT_MODEL = "saaras:v3" DEFAULT_SARVAM_TTS_MODEL = "bulbul:v3" diff --git a/backend/app/tests/assessment/test_batch.py b/backend/app/tests/assessment/test_batch.py index 8e11389f1..87d4f5932 100644 --- a/backend/app/tests/assessment/test_batch.py +++ b/backend/app/tests/assessment/test_batch.py @@ -18,7 +18,7 @@ submit_assessment_batch, ) from app.models.assessment import AssessmentAttachment -from app.models.llm.constants import DEFAULT_ANTHROPIC_MAX_TOKENS +from app.models.llm.constants import DEFAULT_ASSESSMENT_BATCH_MAX_TOKENS from app.services.assessment.utils.attachments import ( _guess_image_mime_from_url, attachment_type_for_row, @@ -286,7 +286,7 @@ def test_anthropic_native_routes_to_anthropic_batch(self) -> None: assert start_batch.call_args.kwargs["provider_name"] == "anthropic" assert start_batch.call_args.kwargs["config"]["model"] == "claude-sonnet-4-6" assert start_batch.call_args.kwargs["config"]["max_tokens"] == ( - DEFAULT_ANTHROPIC_MAX_TOKENS + DEFAULT_ASSESSMENT_BATCH_MAX_TOKENS ) diff --git a/backend/app/tests/core/batch/test_anthropic.py b/backend/app/tests/core/batch/test_anthropic.py index 98fec95b0..0c26672b4 100644 --- a/backend/app/tests/core/batch/test_anthropic.py +++ b/backend/app/tests/core/batch/test_anthropic.py @@ -2,7 +2,7 @@ import pytest -from app.core.batch.anthropic import AnthropicBatchProvider +from app.core.batch.anthropic import STRUCTURED_OUTPUTS_BETA, AnthropicBatchProvider from app.models.llm.constants import ( DEFAULT_ANTHROPIC_MAX_TOKENS, DEFAULT_TEXT_MODELS, @@ -90,14 +90,16 @@ def test_create_batch_success(self, provider, mock_anthropic_client): mock_batch = create_mock_batch( batch_id="msgbatch_abc123", processing_status="in_progress" ) - mock_anthropic_client.messages.batches.create.return_value = mock_batch + mock_anthropic_client.beta.messages.batches.create.return_value = mock_batch result = provider.create_batch(jsonl_data, config) - mock_anthropic_client.messages.batches.create.assert_called_once() - requests = mock_anthropic_client.messages.batches.create.call_args.kwargs[ - "requests" - ] + mock_anthropic_client.beta.messages.batches.create.assert_called_once() + call_kwargs = ( + mock_anthropic_client.beta.messages.batches.create.call_args.kwargs + ) + requests = call_kwargs["requests"] + assert call_kwargs["betas"] == [] assert len(requests) == 2 assert requests[0]["custom_id"] == "req-1" assert requests[0]["params"]["model"] == "claude-sonnet-4-6" @@ -120,11 +122,11 @@ def test_create_batch_applies_config_defaults( config = {"model": "claude-opus-4-8", "max_tokens": 2048} mock_batch = create_mock_batch() - mock_anthropic_client.messages.batches.create.return_value = mock_batch + mock_anthropic_client.beta.messages.batches.create.return_value = mock_batch provider.create_batch(jsonl_data, config) - requests = mock_anthropic_client.messages.batches.create.call_args.kwargs[ + requests = mock_anthropic_client.beta.messages.batches.create.call_args.kwargs[ "requests" ] assert requests[0]["params"]["model"] == "claude-opus-4-8" @@ -142,11 +144,11 @@ def test_create_batch_applies_global_defaults( ] mock_batch = create_mock_batch() - mock_anthropic_client.messages.batches.create.return_value = mock_batch + mock_anthropic_client.beta.messages.batches.create.return_value = mock_batch provider.create_batch(jsonl_data, {}) - requests = mock_anthropic_client.messages.batches.create.call_args.kwargs[ + requests = mock_anthropic_client.beta.messages.batches.create.call_args.kwargs[ "requests" ] assert requests[0]["params"]["model"] == DEFAULT_TEXT_MODELS["anthropic"] @@ -169,21 +171,46 @@ def test_create_batch_does_not_override_request_params( config = {"model": "claude-opus-4-8", "max_tokens": 2048} mock_batch = create_mock_batch() - mock_anthropic_client.messages.batches.create.return_value = mock_batch + mock_anthropic_client.beta.messages.batches.create.return_value = mock_batch provider.create_batch(jsonl_data, config) - requests = mock_anthropic_client.messages.batches.create.call_args.kwargs[ + requests = mock_anthropic_client.beta.messages.batches.create.call_args.kwargs[ "requests" ] assert requests[0]["params"]["model"] == "claude-haiku-4-5" assert requests[0]["params"]["max_tokens"] == 256 + def test_create_batch_enables_structured_outputs_beta( + self, provider, mock_anthropic_client + ): + """Test that output_config in a request opts the batch into the beta.""" + jsonl_data = [ + { + "custom_id": "req-1", + "params": { + "messages": [{"role": "user", "content": "Hello"}], + "output_config": {"format": {"type": "json_schema", "schema": {}}}, + }, + } + ] + + mock_anthropic_client.beta.messages.batches.create.return_value = ( + create_mock_batch() + ) + + provider.create_batch(jsonl_data, {}) + + betas = mock_anthropic_client.beta.messages.batches.create.call_args.kwargs[ + "betas" + ] + assert betas == [STRUCTURED_OUTPUTS_BETA] + def test_create_batch_error(self, provider, mock_anthropic_client): """Test handling of batch creation error.""" jsonl_data = [{"custom_id": "req-1", "params": {}}] - mock_anthropic_client.messages.batches.create.side_effect = Exception( + mock_anthropic_client.beta.messages.batches.create.side_effect = Exception( "Batch creation failed" )