diff --git a/backend/app/core/config.py b/backend/app/core/config.py index 8d938d4e3..fdd9e2ebb 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -115,6 +115,8 @@ def SQLALCHEMY_DATABASE_URI(self) -> PostgresDsn: GCP_SA_KEY: str = "" GCS_AUDIO_BUCKET: str = "" + GEMINI_API_KEY: str = "" + GEMINI_DEFAULT_INFERENCE_ROUTE: str = "aistudio" # RabbitMQ configuration for Celery broker RABBITMQ_HOST: str = "localhost" RABBITMQ_PORT: int = 5672 diff --git a/backend/app/models/llm/constants.py b/backend/app/models/llm/constants.py index ee72138dc..c96fc5132 100644 --- a/backend/app/models/llm/constants.py +++ b/backend/app/models/llm/constants.py @@ -9,6 +9,7 @@ class Provider(StrEnum): ELEVENLABS = "elevenlabs" ANTHROPIC = "anthropic" GOOGLE_AISTUDIO = "google-aistudio" + GOOGLE_VERTEX = "google-vertex" PROXY = "proxy" @@ -47,6 +48,7 @@ class Provider(StrEnum): "elevenlabs-native", "anthropic-native", "google-aistudio-native", + "google-vertex-native", ] diff --git a/backend/app/services/llm/providers/google_ai.py b/backend/app/services/llm/providers/google_ai.py index 6388a6ddb..b9ce73e3c 100644 --- a/backend/app/services/llm/providers/google_ai.py +++ b/backend/app/services/llm/providers/google_ai.py @@ -94,7 +94,12 @@ def __init__( self.gcs_bucket = gcs_bucket or settings.GCS_AUDIO_BUCKET def endpoint(self, model: str) -> str: - host = N + host = "" + if self.location == "global": + host = "aiplatform.googleapis.com" + else: + host = f"{self.location}-aiplatform.googleapis.com" + return ( f"https://{host}/v1" f"/projects/{self.project_id}/locations/{self.location}" diff --git a/backend/app/services/llm/providers/google_aistudio.py b/backend/app/services/llm/providers/google_aistudio.py index 7614d7503..3e9ac632a 100644 --- a/backend/app/services/llm/providers/google_aistudio.py +++ b/backend/app/services/llm/providers/google_aistudio.py @@ -33,6 +33,7 @@ from app.models.llm.response import AudioOutput, AudioContent from app.services.llm.providers.base import BaseProvider, ContentPart, MultiModalInput from app.services.llm.mappers import BCP47_LOCALE_TO_GEMINI_LANG +from app.core.config import settings from app.core.audio_utils import ( AudioRef, convert_pcm_to_mp3, @@ -55,9 +56,10 @@ def __init__(self, client: genai.Client): @staticmethod def create_client(credentials: dict[str, Any]) -> Any: - if "api_key" not in credentials: + api_key = credentials.get("api_key") or settings.GEMINI_API_KEY + if not api_key: raise ValueError("API Key for Google Gemini Not Set") - return genai.Client(api_key=credentials["api_key"]) + return genai.Client(api_key=api_key) @staticmethod def format_parts( diff --git a/backend/app/services/llm/providers/registry.py b/backend/app/services/llm/providers/registry.py index 04ae9fd7a..b4532d9d3 100644 --- a/backend/app/services/llm/providers/registry.py +++ b/backend/app/services/llm/providers/registry.py @@ -1,9 +1,11 @@ import logging from sqlmodel import Session +from app.core.config import settings from app.services.llm.providers.base import BaseProvider from app.services.llm.providers.open_ai import OpenAIProvider from app.services.llm.providers.google_aistudio import GoogleAIProvider +from app.services.llm.providers.google_ai import GoogleVertexAIProvider from app.services.llm.providers.sarvam_ai import SarvamAIProvider from app.services.llm.providers.eleven_ai import ElevenlabsAIProvider from app.services.llm.providers.claude import ClaudeProvider @@ -15,34 +17,46 @@ class LLMProvider: OPENAI = "openai" SARVAMAI = "sarvamai" ELEVENLABS = "elevenlabs" - GOOGLE = "google" ANTHROPIC = "anthropic" + + # Platform-routed Google. Resolved via GEMINI_DEFAULT_INFERENCE_ROUTE. + GOOGLE = "google" + GOOGLE_NATIVE = "google-native" + # Explicit Google backends. Bypass env routing; use caller's own creds. + GOOGLE_VERTEX = "google-vertex" + GOOGLE_VERTEX_NATIVE = "google-vertex-native" GOOGLE_AISTUDIO = "google-aistudio" GOOGLE_AISTUDIO_NATIVE = "google-aistudio-native" + OPENAI_NATIVE = "openai-native" - GOOGLE_NATIVE = "google-native" SARVAMAI_NATIVE = "sarvamai-native" ELEVENLABS_NATIVE = "elevenlabs-native" ANTHROPIC_NATIVE = "anthropic-native" _registry: dict[str, type[BaseProvider]] = { OPENAI: OpenAIProvider, - GOOGLE: GoogleAIProvider, SARVAMAI: SarvamAIProvider, ELEVENLABS: ElevenlabsAIProvider, ANTHROPIC: ClaudeProvider, + GOOGLE: GoogleVertexAIProvider, # placeholder; env decides at resolve time + GOOGLE_NATIVE: GoogleVertexAIProvider, + GOOGLE_VERTEX: GoogleVertexAIProvider, + GOOGLE_VERTEX_NATIVE: GoogleVertexAIProvider, GOOGLE_AISTUDIO: GoogleAIProvider, GOOGLE_AISTUDIO_NATIVE: GoogleAIProvider, OPENAI_NATIVE: OpenAIProvider, - GOOGLE_NATIVE: GoogleAIProvider, SARVAMAI_NATIVE: SarvamAIProvider, ELEVENLABS_NATIVE: ElevenlabsAIProvider, ANTHROPIC_NATIVE: ClaudeProvider, } + _GOOGLE_ROUTED = {GOOGLE, GOOGLE_NATIVE} + @classmethod def get_provider_class(cls, provider_type: str) -> type[BaseProvider]: - """Return the provider class for a given name.""" + if provider_type in cls._GOOGLE_ROUTED: + route = settings.GEMINI_DEFAULT_INFERENCE_ROUTE + return GoogleAIProvider if route == "aistudio" else GoogleVertexAIProvider provider = cls._registry.get(provider_type) if not provider: raise ValueError( @@ -53,7 +67,6 @@ def get_provider_class(cls, provider_type: str) -> type[BaseProvider]: @classmethod def supported_providers(cls) -> list[str]: - """Return a list of supported provider names.""" return list(cls._registry.keys()) @@ -63,8 +76,16 @@ def get_llm_provider( from app.crud.credentials import get_provider_credential provider_class = LLMProvider.get_provider_class(provider_type) + is_platform_routed = provider_type in LLMProvider._GOOGLE_ROUTED - credential_provider = provider_type.replace("-native", "") + if is_platform_routed: + credential_provider = ( + LLMProvider.GOOGLE_AISTUDIO + if provider_class is GoogleAIProvider + else LLMProvider.GOOGLE_VERTEX + ) + else: + credential_provider = provider_type.replace("-native", "") credentials = get_provider_credential( session=session, @@ -73,16 +94,17 @@ def get_llm_provider( org_id=organization_id, ) - if not credentials: + if not credentials and not is_platform_routed: raise ValueError( f"Credentials for provider '{credential_provider}' not configured for this project." ) + credentials = credentials or {} + try: client = provider_class.create_client(credentials=credentials) return provider_class(client=client) except ValueError: - # Re-raise ValueError for credential/configuration errors raise except Exception as e: logger.error(f"Failed to initialize {provider_type} client: {e}", exc_info=True) diff --git a/backend/app/tests/services/llm/providers/test_registry.py b/backend/app/tests/services/llm/providers/test_registry.py index c1307802c..65dcb6167 100644 --- a/backend/app/tests/services/llm/providers/test_registry.py +++ b/backend/app/tests/services/llm/providers/test_registry.py @@ -10,6 +10,8 @@ from app.services.llm.providers.base import BaseProvider from app.services.llm.providers.open_ai import OpenAIProvider +from app.services.llm.providers.google_ai import GoogleVertexAIProvider +from app.services.llm.providers.google_aistudio import GoogleAIProvider from app.services.llm.providers.registry import ( LLMProvider, get_llm_provider, @@ -18,26 +20,17 @@ class TestProviderRegistry: - """Test cases for the PROVIDER_REGISTRY constant.""" - def test_registry_contains_openai(self): - """Test that registry contains OpenAI provider.""" assert "openai-native" in LLMProvider._registry assert LLMProvider._registry["openai-native"] == OpenAIProvider def test_registry_values_are_provider_classes(self): - """Test that all registry values are BaseProvider subclasses.""" for provider_type, provider_class in LLMProvider._registry.items(): - assert issubclass( - provider_class, BaseProvider - ), f"Provider '{provider_type}' class must inherit from BaseProvider" + assert issubclass(provider_class, BaseProvider) class TestGetLLMProvider: - """Test cases for the get_llm_provider function.""" - def test_get_llm_provider_with_openai(self, db: Session): - """Test getting OpenAI provider successfully.""" project = get_project(db) with patch("app.crud.credentials.get_provider_credential") as mock_get_creds: @@ -59,7 +52,26 @@ def test_get_llm_provider_with_openai(self, db: Session): org_id=project.organization_id, ) - mock_get_creds.return_value = {"wrong_key": "value"} + def test_get_llm_provider_with_invalid_provider(self, db: Session): + project = get_project(db) + + with pytest.raises(ValueError) as exc_info: + get_llm_provider( + session=db, + provider_type="invalid_provider", + project_id=project.id, + organization_id=project.organization_id, + ) + + assert "invalid_provider" in str(exc_info.value) + assert "is not supported" in str(exc_info.value) + + def test_openai_missing_credentials_raises(self, db: Session): + project = get_project(db) + + with patch("app.crud.credentials.get_provider_credential") as mock_get_creds: + mock_get_creds.return_value = None + with pytest.raises(ValueError) as exc_info: get_llm_provider( session=db, @@ -67,29 +79,68 @@ def test_get_llm_provider_with_openai(self, db: Session): project_id=project.id, organization_id=project.organization_id, ) - assert "OpenAI credentials not configured for this project." in str( - exc_info.value + + assert "not configured for this project" in str(exc_info.value) + + # ---- Explicit google-vertex / google-aistudio: bypass env, need creds ---- + + def test_explicit_google_vertex_bypasses_env(self, db: Session): + project = get_project(db) + + with patch( + "app.services.llm.providers.registry.settings" + ) as mock_settings, patch( + "app.crud.credentials.get_provider_credential" + ) as mock_get_creds: + mock_settings.GEMINI_DEFAULT_INFERENCE_ROUTE = "aistudio" + mock_get_creds.return_value = { + "api_key": "byok-key", + "project_id": "byok-project", + "location": "us-central1", + } + + provider = get_llm_provider( + session=db, + provider_type="google-vertex", + project_id=project.id, + organization_id=project.organization_id, ) - def test_get_llm_provider_with_invalid_provider(self, db: Session): - """Test that invalid provider type raises ValueError.""" + assert isinstance(provider, GoogleVertexAIProvider) + mock_get_creds.assert_called_once_with( + session=db, + provider="google-vertex", + project_id=project.id, + org_id=project.organization_id, + ) + + def test_explicit_google_aistudio_bypasses_env(self, db: Session): project = get_project(db) - with pytest.raises(ValueError) as exc_info: - get_llm_provider( + with patch( + "app.services.llm.providers.registry.settings" + ) as mock_settings, patch( + "app.crud.credentials.get_provider_credential" + ) as mock_get_creds: + mock_settings.GEMINI_DEFAULT_INFERENCE_ROUTE = "vertex" + mock_get_creds.return_value = {"api_key": "byok-key"} + + provider = get_llm_provider( session=db, - provider_type="invalid_provider", + provider_type="google-aistudio", project_id=project.id, organization_id=project.organization_id, ) - error_message = str(exc_info.value) - assert "invalid_provider" in error_message - assert "is not supported" in error_message - assert "openai-native" in error_message + assert isinstance(provider, GoogleAIProvider) + mock_get_creds.assert_called_once_with( + session=db, + provider="google-aistudio", + project_id=project.id, + org_id=project.organization_id, + ) - def test_get_llm_provider_with_missing_credentials(self, db: Session): - """Test handling of errors when credentials are not found.""" + def test_explicit_google_vertex_without_creds_raises(self, db: Session): project = get_project(db) with patch("app.crud.credentials.get_provider_credential") as mock_get_creds: @@ -98,28 +149,135 @@ def test_get_llm_provider_with_missing_credentials(self, db: Session): with pytest.raises(ValueError) as exc_info: get_llm_provider( session=db, - provider_type="openai-native", + provider_type="google-vertex", project_id=project.id, organization_id=project.organization_id, ) - assert "not configured for this project" in str(exc_info.value) - - def test_google_native_routes_to_aistudio(self, db: Session): - """``google`` / ``google-native`` currently route to GoogleAIProvider - (AI Studio). Vertex routing is temporarily disabled.""" - from app.services.llm.providers.google_aistudio import GoogleAIProvider + assert "google-vertex" in str(exc_info.value) + def test_explicit_google_aistudio_without_creds_raises(self, db: Session): project = get_project(db) with patch("app.crud.credentials.get_provider_credential") as mock_get_creds: - mock_get_creds.return_value = {"api_key": "test-api-key"} + mock_get_creds.return_value = None + + with pytest.raises(ValueError) as exc_info: + get_llm_provider( + session=db, + provider_type="google-aistudio", + project_id=project.id, + organization_id=project.organization_id, + ) + + assert "google-aistudio" in str(exc_info.value) + + # ---- Platform-routed `google`: env decides, platform fallback allowed ---- + + def test_google_routes_to_vertex_when_env_vertex(self, db: Session): + project = get_project(db) + + with patch( + "app.services.llm.providers.registry.settings" + ) as mock_settings, patch( + "app.crud.credentials.get_provider_credential" + ) as mock_get_creds: + mock_settings.GEMINI_DEFAULT_INFERENCE_ROUTE = "vertex" + mock_get_creds.return_value = { + "api_key": "byok-key", + "project_id": "byok-project", + "location": "us-central1", + } + + provider = get_llm_provider( + session=db, + provider_type="google", + project_id=project.id, + organization_id=project.organization_id, + ) + + assert isinstance(provider, GoogleVertexAIProvider) + mock_get_creds.assert_called_once_with( + session=db, + provider="google-vertex", + project_id=project.id, + org_id=project.organization_id, + ) + + def test_google_routes_to_aistudio_when_env_aistudio(self, db: Session): + project = get_project(db) + + with patch( + "app.services.llm.providers.registry.settings" + ) as mock_settings, patch( + "app.crud.credentials.get_provider_credential" + ) as mock_get_creds: + mock_settings.GEMINI_DEFAULT_INFERENCE_ROUTE = "aistudio" + mock_get_creds.return_value = {"api_key": "byok-key"} + + provider = get_llm_provider( + session=db, + provider_type="google", + project_id=project.id, + organization_id=project.organization_id, + ) + + assert isinstance(provider, GoogleAIProvider) + mock_get_creds.assert_called_once_with( + session=db, + provider="google-aistudio", + project_id=project.id, + org_id=project.organization_id, + ) + + def test_google_vertex_route_falls_back_to_platform_defaults(self, db: Session): + project = get_project(db) + + with patch( + "app.services.llm.providers.registry.settings" + ) as registry_settings, patch( + "app.services.llm.providers.google_ai.settings" + ) as google_settings, patch( + "app.crud.credentials.get_provider_credential" + ) as mock_get_creds: + registry_settings.GEMINI_DEFAULT_INFERENCE_ROUTE = "vertex" + google_settings.GCP_VERTEX_API_KEY = "platform-key" + google_settings.GCP_PROJECT_ID = "platform-project" + google_settings.GCP_VERTEX_LOCATION = "us-central1" + google_settings.GCP_SA_KEY = "" + google_settings.GCS_AUDIO_BUCKET = "" + mock_get_creds.return_value = None + + provider = get_llm_provider( + session=db, + provider_type="google", + project_id=project.id, + organization_id=project.organization_id, + ) + + assert isinstance(provider, GoogleVertexAIProvider) + assert provider.client.api_key == "platform-key" + assert provider.client.project_id == "platform-project" + + def test_google_aistudio_route_falls_back_to_platform_defaults(self, db: Session): + project = get_project(db) + + with patch( + "app.services.llm.providers.registry.settings" + ) as registry_settings, patch( + "app.services.llm.providers.google_aistudio.settings" + ) as aistudio_settings, patch( + "app.crud.credentials.get_provider_credential" + ) as mock_get_creds: + registry_settings.GEMINI_DEFAULT_INFERENCE_ROUTE = "aistudio" + aistudio_settings.GEMINI_API_KEY = "platform-gemini-key" + mock_get_creds.return_value = None provider = get_llm_provider( session=db, - provider_type="google-native", + provider_type="google", project_id=project.id, organization_id=project.organization_id, ) - assert isinstance(provider, GoogleAIProvider) + assert isinstance(provider, GoogleAIProvider) diff --git a/backend/docs/gemini-routing-flows.md b/backend/docs/gemini-routing-flows.md new file mode 100644 index 000000000..4e2b587fd --- /dev/null +++ b/backend/docs/gemini-routing-flows.md @@ -0,0 +1,96 @@ +# Gemini Provider Routing — Flow Report + +Registry: `app/services/llm/providers/registry.py` +Env: `GEMINI_DEFAULT_INFERENCE_ROUTE` (`"aistudio"` | `"vertex"`, default `"aistudio"`) +Platform fallbacks: `GEMINI_API_KEY` (aistudio), `GCP_VERTEX_API_KEY` + `GCP_PROJECT_ID` + `GCP_VERTEX_LOCATION` + `GCP_SA_KEY` (vertex) + +## Provider names + +| Name | Meaning | +|---|---| +| `google` / `google-native` | Platform-routed. Backend chosen by env. | +| `google-vertex` / `google-vertex-native` | Explicit Vertex AI. Uses caller's creds. | +| `google-aistudio` / `google-aistudio-native` | Explicit AI Studio. Uses caller's creds. | + +## Happy paths + +### H1 — Explicit `google-vertex` with BYOK creds +Caller: `provider_type="google-vertex"`, project has `credential` row `provider='google-vertex'`. +Flow: `get_provider_class` → `GoogleVertexAIProvider`. Credential lookup key = `google-vertex`. Row found → creds injected into `create_client` → BYOK client returned. + +### H2 — Explicit `google-aistudio` with BYOK creds +Caller: `provider_type="google-aistudio"`, project has `credential` row `provider='google-aistudio'`. +Flow: `get_provider_class` → `GoogleAIProvider`. Lookup key = `google-aistudio`. Row found → BYOK api_key used. + +### H3 — Platform-routed `google`, env=`vertex`, BYOK vertex creds +Caller: `provider_type="google"`, `GEMINI_DEFAULT_INFERENCE_ROUTE="vertex"`, row `provider='google-vertex'` exists. +Flow: `_GOOGLE_ROUTED` hit → env → vertex. Lookup key rewritten to `google-vertex`. Row found → BYOK vertex client. + +### H4 — Platform-routed `google`, env=`aistudio`, BYOK aistudio creds +Symmetric to H3 with `GoogleAIProvider` + `google-aistudio` lookup key. + +### H5 — Platform-routed `google`, env=`vertex`, no BYOK → platform fallback +Caller: `provider_type="google"`, env=vertex, no `google-vertex` row. +Flow: creds = None. `is_platform_routed=True` so the missing-creds raise is skipped. `credentials={}` passed to `GoogleVertexAIProvider.create_client`, which per-field falls back to `GCP_VERTEX_*` settings. Client built from platform defaults. + +### H6 — Platform-routed `google`, env=`aistudio`, no BYOK → platform fallback +Symmetric to H5. `GoogleAIProvider.create_client` uses `settings.GEMINI_API_KEY`. + +### H7 — Any non-Google provider (e.g. `openai-native`) +Unchanged: lookup key = `openai`, missing creds raise, present creds → client. + +## Edge cases + +### E1 — Explicit `google-vertex` without creds +`get_provider_credential` returns None. `is_platform_routed=False` → raise +`ValueError("Credentials for provider 'google-vertex' not configured for this project.")`. +Rationale: explicit provider names bypass platform fallback by design. + +### E2 — Explicit `google-aistudio` without creds +Symmetric to E1. Raises for `'google-aistudio'`. + +### E3 — Platform-routed `google` with no BYOK and no platform key +env=aistudio, `google-aistudio` row missing, `GEMINI_API_KEY=""`. +Flow: creds=None, `credentials={}`, `create_client` → `api_key` empty → raises `ValueError("API Key for Google Gemini Not Set")`. +Same shape on vertex if all `GCP_*` platform vars are empty and the row is missing (per existing vertex behavior). + +### E4 — env unset (empty string) +`settings.GEMINI_DEFAULT_INFERENCE_ROUTE=""`. In `get_provider_class`: `route == "aistudio"` is False → falls to vertex branch. Effectively `""` behaves like `"vertex"`. +Not enforced as an error. If you want to hard-fail on bad values, add the `Literal` back. + +### E5 — env set to garbage (e.g. `"gemini-pro"`) +Treated as "not aistudio" → routes to vertex. Silent misconfiguration. +Same mitigation as E4. + +### E6 — `-native` variants +`google-native` behaves identically to `google` (env-routed). +`google-vertex-native`, `google-aistudio-native` behave identically to their non-native peers. +`-native` is stripped for the credential lookup key. + +### E7 — Legacy `google` credential rows (pre-migration) +If DB still has rows keyed `provider='google'` from the old semantics, this code will not find them when the caller asks for `google-vertex` or when `google`+env=vertex remaps to `google-vertex`. Requires the pending data migration to rename `google` → `google-vertex`. + +### E8 — Non-ValueError exception in `create_client` +Caught by the outer `except Exception` in `get_llm_provider`. Logged with `exc_info`, re-raised as `RuntimeError(f"Could not connect to {provider_type} services.")`. `ValueError` from missing keys is preserved as-is. + +### E9 — Invalid provider name +`get_provider_class` → not in `_registry`, not in `_GOOGLE_ROUTED` → raises `ValueError("Provider '' is not supported...")` with the full supported list. + +## Decision table + +| provider_type | env | BYOK row present | Result | +|---|---|---|---| +| `google-vertex` | any | yes | vertex, BYOK | +| `google-vertex` | any | no | raise | +| `google-aistudio` | any | yes | aistudio, BYOK | +| `google-aistudio` | any | no | raise | +| `google` | `vertex` | `google-vertex` yes | vertex, BYOK | +| `google` | `vertex` | no | vertex, platform fallback | +| `google` | `aistudio` | `google-aistudio` yes | aistudio, BYOK | +| `google` | `aistudio` | no | aistudio, platform fallback | +| `google` | `""` / other | — | vertex (silent) | + +## Pending / not covered here + +- DB data migration renaming `credential.provider='google'` rows (to `'google-vertex'` per the new semantics). Tracked separately. +- `.env.example` update for `GEMINI_API_KEY` and `GEMINI_DEFAULT_INFERENCE_ROUTE`.