From 64b7a89a81cafc8fec481bf5a4120c1b8d59ecc8 Mon Sep 17 00:00:00 2001 From: Prashant Vasudevan <71649489+vprashrex@users.noreply.github.com> Date: Tue, 30 Jun 2026 13:30:06 +0530 Subject: [PATCH 1/4] refactor(assessment): Update max tokens constant for assessment batch and enhance Anthropic batch handling --- backend/app/core/batch/anthropic.py | 13 ++++++++++++- backend/app/crud/assessment/batch.py | 4 ++-- backend/app/models/llm/constants.py | 2 ++ backend/app/tests/assessment/test_batch.py | 4 ++-- 4 files changed, 18 insertions(+), 5 deletions(-) diff --git a/backend/app/core/batch/anthropic.py b/backend/app/core/batch/anthropic.py index dfcbcfd3c..09da55fbb 100644 --- a/backend/app/core/batch/anthropic.py +++ b/backend/app/core/batch/anthropic.py @@ -1,5 +1,6 @@ """Anthropic batch provider implementation.""" +import json import logging from enum import Enum from typing import Any @@ -15,6 +16,8 @@ logger = logging.getLogger(__name__) +STRUCTURED_OUTPUTS_BETA = "structured-outputs-2025-12-15" + class MessageBatchStatus(str, Enum): IN_PROGRESS = "in_progress" @@ -67,13 +70,21 @@ def create_batch( try: requests = [] + needs_structured_outputs = False for item in jsonl_data: params = {**item.get("params", {})} params["model"] = params.get("model") or default_model params["max_tokens"] = params.get("max_tokens") or default_max_tokens + if "output_config" in params: + needs_structured_outputs = True requests.append({"custom_id": item[BATCH_KEY], "params": params}) - batch = self.client.messages.batches.create(requests=requests) + logger.info(f"[create_batch] Anthropic batch requests jsonl: {json.dumps(requests, indent=2)}") + + betas = [STRUCTURED_OUTPUTS_BETA] if needs_structured_outputs else [] + batch = self.client.beta.messages.batches.create( + requests=requests, betas=betas + ) result = { "provider_batch_id": batch.id, diff --git a/backend/app/crud/assessment/batch.py b/backend/app/crud/assessment/batch.py index 54dc6cf01..2c2416086 100644 --- a/backend/app/crud/assessment/batch.py +++ b/backend/app/crud/assessment/batch.py @@ -29,7 +29,7 @@ ) from app.models.batch_job import BatchJob, BatchJobType from app.models.evaluation import EvaluationDataset -from app.models.llm.constants import DEFAULT_ANTHROPIC_MAX_TOKENS +from app.models.llm.constants import DEFAULT_ASSESSMENT_BATCH_MAX_TOKENS from app.models.llm.request import ConfigBlob from app.services.assessment.mappers import ( map_kaapi_to_anthropic_params, @@ -530,7 +530,7 @@ def submit_assessment_batch( batch_config = { "model": mapped_params.get("model"), "max_tokens": mapped_params.get("max_tokens") - or DEFAULT_ANTHROPIC_MAX_TOKENS, + or DEFAULT_ASSESSMENT_BATCH_MAX_TOKENS, } batch_job = start_batch_job( diff --git a/backend/app/models/llm/constants.py b/backend/app/models/llm/constants.py index 021ba0909..ee72138dc 100644 --- a/backend/app/models/llm/constants.py +++ b/backend/app/models/llm/constants.py @@ -78,6 +78,8 @@ class Modality(StrEnum): DEFAULT_ANTHROPIC_MAX_TOKENS = 4096 +DEFAULT_ASSESSMENT_BATCH_MAX_TOKENS = 16384 + # Provider-native STT/TTS defaults (used when caller omits model). DEFAULT_SARVAM_STT_MODEL = "saaras:v3" DEFAULT_SARVAM_TTS_MODEL = "bulbul:v3" diff --git a/backend/app/tests/assessment/test_batch.py b/backend/app/tests/assessment/test_batch.py index 8e11389f1..87d4f5932 100644 --- a/backend/app/tests/assessment/test_batch.py +++ b/backend/app/tests/assessment/test_batch.py @@ -18,7 +18,7 @@ submit_assessment_batch, ) from app.models.assessment import AssessmentAttachment -from app.models.llm.constants import DEFAULT_ANTHROPIC_MAX_TOKENS +from app.models.llm.constants import DEFAULT_ASSESSMENT_BATCH_MAX_TOKENS from app.services.assessment.utils.attachments import ( _guess_image_mime_from_url, attachment_type_for_row, @@ -286,7 +286,7 @@ def test_anthropic_native_routes_to_anthropic_batch(self) -> None: assert start_batch.call_args.kwargs["provider_name"] == "anthropic" assert start_batch.call_args.kwargs["config"]["model"] == "claude-sonnet-4-6" assert start_batch.call_args.kwargs["config"]["max_tokens"] == ( - DEFAULT_ANTHROPIC_MAX_TOKENS + DEFAULT_ASSESSMENT_BATCH_MAX_TOKENS ) From 5c999f9aa1284a17a0b9bcd766c68a65b31db11f Mon Sep 17 00:00:00 2001 From: Prashant Vasudevan <71649489+vprashrex@users.noreply.github.com> Date: Tue, 30 Jun 2026 13:31:05 +0530 Subject: [PATCH 2/4] format(assessment): Format logger info statement for better readability in create_batch method --- backend/app/core/batch/anthropic.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/app/core/batch/anthropic.py b/backend/app/core/batch/anthropic.py index 09da55fbb..e664bae62 100644 --- a/backend/app/core/batch/anthropic.py +++ b/backend/app/core/batch/anthropic.py @@ -79,7 +79,9 @@ def create_batch( needs_structured_outputs = True requests.append({"custom_id": item[BATCH_KEY], "params": params}) - logger.info(f"[create_batch] Anthropic batch requests jsonl: {json.dumps(requests, indent=2)}") + logger.info( + f"[create_batch] Anthropic batch requests jsonl: {json.dumps(requests, indent=2)}" + ) betas = [STRUCTURED_OUTPUTS_BETA] if needs_structured_outputs else [] batch = self.client.beta.messages.batches.create( From 6c133662499fe0fcbe7a319c5dba23f4dc18fc7b Mon Sep 17 00:00:00 2001 From: Prashant Vasudevan <71649489+vprashrex@users.noreply.github.com> Date: Tue, 30 Jun 2026 15:47:16 +0530 Subject: [PATCH 3/4] fix(assessment): enable structured outputs on Anthropic batch and fix tests Route batch creation through the beta endpoint with the structured-outputs beta so output_config (JSON schema) is forwarded instead of stripped, and update the provider tests to assert the beta path and beta flag. Co-Authored-By: Claude Opus 4.8 --- backend/app/core/batch/anthropic.py | 4 -- .../app/tests/core/batch/test_anthropic.py | 53 ++++++++++++++----- 2 files changed, 40 insertions(+), 17 deletions(-) diff --git a/backend/app/core/batch/anthropic.py b/backend/app/core/batch/anthropic.py index e664bae62..0a6e6e888 100644 --- a/backend/app/core/batch/anthropic.py +++ b/backend/app/core/batch/anthropic.py @@ -79,10 +79,6 @@ def create_batch( needs_structured_outputs = True requests.append({"custom_id": item[BATCH_KEY], "params": params}) - logger.info( - f"[create_batch] Anthropic batch requests jsonl: {json.dumps(requests, indent=2)}" - ) - betas = [STRUCTURED_OUTPUTS_BETA] if needs_structured_outputs else [] batch = self.client.beta.messages.batches.create( requests=requests, betas=betas diff --git a/backend/app/tests/core/batch/test_anthropic.py b/backend/app/tests/core/batch/test_anthropic.py index 98fec95b0..f4c950694 100644 --- a/backend/app/tests/core/batch/test_anthropic.py +++ b/backend/app/tests/core/batch/test_anthropic.py @@ -2,7 +2,7 @@ import pytest -from app.core.batch.anthropic import AnthropicBatchProvider +from app.core.batch.anthropic import STRUCTURED_OUTPUTS_BETA, AnthropicBatchProvider from app.models.llm.constants import ( DEFAULT_ANTHROPIC_MAX_TOKENS, DEFAULT_TEXT_MODELS, @@ -90,14 +90,14 @@ def test_create_batch_success(self, provider, mock_anthropic_client): mock_batch = create_mock_batch( batch_id="msgbatch_abc123", processing_status="in_progress" ) - mock_anthropic_client.messages.batches.create.return_value = mock_batch + mock_anthropic_client.beta.messages.batches.create.return_value = mock_batch result = provider.create_batch(jsonl_data, config) - mock_anthropic_client.messages.batches.create.assert_called_once() - requests = mock_anthropic_client.messages.batches.create.call_args.kwargs[ - "requests" - ] + mock_anthropic_client.beta.messages.batches.create.assert_called_once() + call_kwargs = mock_anthropic_client.beta.messages.batches.create.call_args.kwargs + requests = call_kwargs["requests"] + assert call_kwargs["betas"] == [] assert len(requests) == 2 assert requests[0]["custom_id"] == "req-1" assert requests[0]["params"]["model"] == "claude-sonnet-4-6" @@ -120,11 +120,11 @@ def test_create_batch_applies_config_defaults( config = {"model": "claude-opus-4-8", "max_tokens": 2048} mock_batch = create_mock_batch() - mock_anthropic_client.messages.batches.create.return_value = mock_batch + mock_anthropic_client.beta.messages.batches.create.return_value = mock_batch provider.create_batch(jsonl_data, config) - requests = mock_anthropic_client.messages.batches.create.call_args.kwargs[ + requests = mock_anthropic_client.beta.messages.batches.create.call_args.kwargs[ "requests" ] assert requests[0]["params"]["model"] == "claude-opus-4-8" @@ -142,11 +142,11 @@ def test_create_batch_applies_global_defaults( ] mock_batch = create_mock_batch() - mock_anthropic_client.messages.batches.create.return_value = mock_batch + mock_anthropic_client.beta.messages.batches.create.return_value = mock_batch provider.create_batch(jsonl_data, {}) - requests = mock_anthropic_client.messages.batches.create.call_args.kwargs[ + requests = mock_anthropic_client.beta.messages.batches.create.call_args.kwargs[ "requests" ] assert requests[0]["params"]["model"] == DEFAULT_TEXT_MODELS["anthropic"] @@ -169,21 +169,48 @@ def test_create_batch_does_not_override_request_params( config = {"model": "claude-opus-4-8", "max_tokens": 2048} mock_batch = create_mock_batch() - mock_anthropic_client.messages.batches.create.return_value = mock_batch + mock_anthropic_client.beta.messages.batches.create.return_value = mock_batch provider.create_batch(jsonl_data, config) - requests = mock_anthropic_client.messages.batches.create.call_args.kwargs[ + requests = mock_anthropic_client.beta.messages.batches.create.call_args.kwargs[ "requests" ] assert requests[0]["params"]["model"] == "claude-haiku-4-5" assert requests[0]["params"]["max_tokens"] == 256 + def test_create_batch_enables_structured_outputs_beta( + self, provider, mock_anthropic_client + ): + """Test that output_config in a request opts the batch into the beta.""" + jsonl_data = [ + { + "custom_id": "req-1", + "params": { + "messages": [{"role": "user", "content": "Hello"}], + "output_config": { + "format": {"type": "json_schema", "schema": {}} + }, + }, + } + ] + + mock_anthropic_client.beta.messages.batches.create.return_value = ( + create_mock_batch() + ) + + provider.create_batch(jsonl_data, {}) + + betas = mock_anthropic_client.beta.messages.batches.create.call_args.kwargs[ + "betas" + ] + assert betas == [STRUCTURED_OUTPUTS_BETA] + def test_create_batch_error(self, provider, mock_anthropic_client): """Test handling of batch creation error.""" jsonl_data = [{"custom_id": "req-1", "params": {}}] - mock_anthropic_client.messages.batches.create.side_effect = Exception( + mock_anthropic_client.beta.messages.batches.create.side_effect = Exception( "Batch creation failed" ) From a94785342751bbf3755ac0abf7ec25ae0350028e Mon Sep 17 00:00:00 2001 From: Prashant Vasudevan <71649489+vprashrex@users.noreply.github.com> Date: Tue, 30 Jun 2026 15:47:49 +0530 Subject: [PATCH 4/4] fix(tests): simplify call_kwargs assignment and streamline output_config structure --- backend/app/tests/core/batch/test_anthropic.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/app/tests/core/batch/test_anthropic.py b/backend/app/tests/core/batch/test_anthropic.py index f4c950694..0c26672b4 100644 --- a/backend/app/tests/core/batch/test_anthropic.py +++ b/backend/app/tests/core/batch/test_anthropic.py @@ -95,7 +95,9 @@ def test_create_batch_success(self, provider, mock_anthropic_client): result = provider.create_batch(jsonl_data, config) mock_anthropic_client.beta.messages.batches.create.assert_called_once() - call_kwargs = mock_anthropic_client.beta.messages.batches.create.call_args.kwargs + call_kwargs = ( + mock_anthropic_client.beta.messages.batches.create.call_args.kwargs + ) requests = call_kwargs["requests"] assert call_kwargs["betas"] == [] assert len(requests) == 2 @@ -188,9 +190,7 @@ def test_create_batch_enables_structured_outputs_beta( "custom_id": "req-1", "params": { "messages": [{"role": "user", "content": "Hello"}], - "output_config": { - "format": {"type": "json_schema", "schema": {}} - }, + "output_config": {"format": {"type": "json_schema", "schema": {}}}, }, } ]