Assessment: Anthropic Batch fix

backend/app/core/batch/anthropic.py

@@ -1,5 +1,6 @@
 """Anthropic batch provider implementation."""
 
+import json
 import logging
 from enum import Enum
 from typing import Any
@@ -15,6 +16,8 @@
 
 logger = logging.getLogger(__name__)
 
+STRUCTURED_OUTPUTS_BETA = "structured-outputs-2025-12-15"
+
 
 class MessageBatchStatus(str, Enum):
     IN_PROGRESS = "in_progress"
@@ -67,13 +70,19 @@ def create_batch(
 
         try:
             requests = []
+            needs_structured_outputs = False
             for item in jsonl_data:
                 params = {**item.get("params", {})}
                 params["model"] = params.get("model") or default_model
                 params["max_tokens"] = params.get("max_tokens") or default_max_tokens
+                if "output_config" in params:
+                    needs_structured_outputs = True
                 requests.append({"custom_id": item[BATCH_KEY], "params": params})
 
-            batch = self.client.messages.batches.create(requests=requests)
+            betas = [STRUCTURED_OUTPUTS_BETA] if needs_structured_outputs else []
+            batch = self.client.beta.messages.batches.create(
+                requests=requests, betas=betas
+            )
 
             result = {
                 "provider_batch_id": batch.id,

backend/app/crud/assessment/batch.py

@@ -29,7 +29,7 @@
 )
 from app.models.batch_job import BatchJob, BatchJobType
 from app.models.evaluation import EvaluationDataset
-from app.models.llm.constants import DEFAULT_ANTHROPIC_MAX_TOKENS
+from app.models.llm.constants import DEFAULT_ASSESSMENT_BATCH_MAX_TOKENS
 from app.models.llm.request import ConfigBlob
 from app.services.assessment.mappers import (
     map_kaapi_to_anthropic_params,
@@ -530,7 +530,7 @@ def submit_assessment_batch(
         batch_config = {
             "model": mapped_params.get("model"),
             "max_tokens": mapped_params.get("max_tokens")
-            or DEFAULT_ANTHROPIC_MAX_TOKENS,
+            or DEFAULT_ASSESSMENT_BATCH_MAX_TOKENS,
         }
 
         batch_job = start_batch_job(

backend/app/models/llm/constants.py

@@ -78,6 +78,8 @@ class Modality(StrEnum):
 
 DEFAULT_ANTHROPIC_MAX_TOKENS = 4096
 
+DEFAULT_ASSESSMENT_BATCH_MAX_TOKENS = 16384
+
 # Provider-native STT/TTS defaults (used when caller omits model).
 DEFAULT_SARVAM_STT_MODEL = "saaras:v3"
 DEFAULT_SARVAM_TTS_MODEL = "bulbul:v3"

backend/app/tests/assessment/test_batch.py

@@ -18,7 +18,7 @@
     submit_assessment_batch,
 )
 from app.models.assessment import AssessmentAttachment
-from app.models.llm.constants import DEFAULT_ANTHROPIC_MAX_TOKENS
+from app.models.llm.constants import DEFAULT_ASSESSMENT_BATCH_MAX_TOKENS
 from app.services.assessment.utils.attachments import (
     _guess_image_mime_from_url,
     attachment_type_for_row,
@@ -286,7 +286,7 @@ def test_anthropic_native_routes_to_anthropic_batch(self) -> None:
         assert start_batch.call_args.kwargs["provider_name"] == "anthropic"
         assert start_batch.call_args.kwargs["config"]["model"] == "claude-sonnet-4-6"
         assert start_batch.call_args.kwargs["config"]["max_tokens"] == (
-            DEFAULT_ANTHROPIC_MAX_TOKENS
+            DEFAULT_ASSESSMENT_BATCH_MAX_TOKENS
         )
 
 

backend/app/tests/core/batch/test_anthropic.py

@@ -2,7 +2,7 @@
 
 import pytest
 
-from app.core.batch.anthropic import AnthropicBatchProvider
+from app.core.batch.anthropic import STRUCTURED_OUTPUTS_BETA, AnthropicBatchProvider
 from app.models.llm.constants import (
     DEFAULT_ANTHROPIC_MAX_TOKENS,
     DEFAULT_TEXT_MODELS,
@@ -90,14 +90,16 @@ def test_create_batch_success(self, provider, mock_anthropic_client):
         mock_batch = create_mock_batch(
             batch_id="msgbatch_abc123", processing_status="in_progress"
         )
-        mock_anthropic_client.messages.batches.create.return_value = mock_batch
+        mock_anthropic_client.beta.messages.batches.create.return_value = mock_batch
 
         result = provider.create_batch(jsonl_data, config)
 
-        mock_anthropic_client.messages.batches.create.assert_called_once()
-        requests = mock_anthropic_client.messages.batches.create.call_args.kwargs[
-            "requests"
-        ]
+        mock_anthropic_client.beta.messages.batches.create.assert_called_once()
+        call_kwargs = (
+            mock_anthropic_client.beta.messages.batches.create.call_args.kwargs
+        )
+        requests = call_kwargs["requests"]
+        assert call_kwargs["betas"] == []
         assert len(requests) == 2
         assert requests[0]["custom_id"] == "req-1"
         assert requests[0]["params"]["model"] == "claude-sonnet-4-6"
@@ -120,11 +122,11 @@ def test_create_batch_applies_config_defaults(
         config = {"model": "claude-opus-4-8", "max_tokens": 2048}
 
         mock_batch = create_mock_batch()
-        mock_anthropic_client.messages.batches.create.return_value = mock_batch
+        mock_anthropic_client.beta.messages.batches.create.return_value = mock_batch
 
         provider.create_batch(jsonl_data, config)
 
-        requests = mock_anthropic_client.messages.batches.create.call_args.kwargs[
+        requests = mock_anthropic_client.beta.messages.batches.create.call_args.kwargs[
             "requests"
         ]
         assert requests[0]["params"]["model"] == "claude-opus-4-8"
@@ -142,11 +144,11 @@ def test_create_batch_applies_global_defaults(
         ]
 
         mock_batch = create_mock_batch()
-        mock_anthropic_client.messages.batches.create.return_value = mock_batch
+        mock_anthropic_client.beta.messages.batches.create.return_value = mock_batch
 
         provider.create_batch(jsonl_data, {})
 
-        requests = mock_anthropic_client.messages.batches.create.call_args.kwargs[
+        requests = mock_anthropic_client.beta.messages.batches.create.call_args.kwargs[
             "requests"
         ]
         assert requests[0]["params"]["model"] == DEFAULT_TEXT_MODELS["anthropic"]
@@ -169,21 +171,46 @@ def test_create_batch_does_not_override_request_params(
         config = {"model": "claude-opus-4-8", "max_tokens": 2048}
 
         mock_batch = create_mock_batch()
-        mock_anthropic_client.messages.batches.create.return_value = mock_batch
+        mock_anthropic_client.beta.messages.batches.create.return_value = mock_batch
 
         provider.create_batch(jsonl_data, config)
 
-        requests = mock_anthropic_client.messages.batches.create.call_args.kwargs[
+        requests = mock_anthropic_client.beta.messages.batches.create.call_args.kwargs[
             "requests"
         ]
         assert requests[0]["params"]["model"] == "claude-haiku-4-5"
         assert requests[0]["params"]["max_tokens"] == 256
 
+    def test_create_batch_enables_structured_outputs_beta(
+        self, provider, mock_anthropic_client
+    ):
+        """Test that output_config in a request opts the batch into the beta."""
+        jsonl_data = [
+            {
+                "custom_id": "req-1",
+                "params": {
+                    "messages": [{"role": "user", "content": "Hello"}],
+                    "output_config": {"format": {"type": "json_schema", "schema": {}}},
+                },
+            }
+        ]
+
+        mock_anthropic_client.beta.messages.batches.create.return_value = (
+            create_mock_batch()
+        )
+
+        provider.create_batch(jsonl_data, {})
+
+        betas = mock_anthropic_client.beta.messages.batches.create.call_args.kwargs[
+            "betas"
+        ]
+        assert betas == [STRUCTURED_OUTPUTS_BETA]
+
     def test_create_batch_error(self, provider, mock_anthropic_client):
         """Test handling of batch creation error."""
         jsonl_data = [{"custom_id": "req-1", "params": {}}]
 
-        mock_anthropic_client.messages.batches.create.side_effect = Exception(
+        mock_anthropic_client.beta.messages.batches.create.side_effect = Exception(
             "Batch creation failed"
         )