From de45b101e57f3c9a0a79e8778df87d2d95871d6f Mon Sep 17 00:00:00 2001 From: Stackbilt Date: Mon, 22 Jun 2026 17:19:10 -0500 Subject: [PATCH] feat: first-class cfGateway config for per-provider AI Gateway passthrough (closes #97) Co-Authored-By: Claude Sonnet 4.6 --- CHANGELOG.md | 3 + README.md | 42 ++++ src/__tests__/cf-gateway.test.ts | 209 +++++++++++++++++++ src/errors.ts | 99 +++++---- src/index.ts | 180 ++++++++-------- src/providers/anthropic.ts | 15 +- src/providers/base.ts | 338 +++++++++++++++++++------------ src/providers/cerebras.ts | 15 +- src/providers/groq.ts | 15 +- src/providers/nvidia.ts | 15 +- src/providers/openai.ts | 15 +- src/types.ts | 123 ++++++----- 12 files changed, 749 insertions(+), 320 deletions(-) create mode 100644 src/__tests__/cf-gateway.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ec3c22..479147e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,9 @@ Format follows [Keep a Changelog](https://keepachangelog.com/). Versions use [Se ## [Unreleased] +### Added +- **First-class `cfGateway` config (#97)** — `OpenAIConfig`, `AnthropicConfig`, `CerebrasConfig`, `GroqConfig`, and `NvidiaConfig` accept `cfGateway?: { accountId, gatewayId }`. When set (and no explicit `baseUrl` override is present), the provider derives its base URL as `https://gateway.ai.cloudflare.com/v1/{accountId}/{gatewayId}/{suffix}` per provider (`openai/v1`, `anthropic`, `cerebras/v1`, `groq/openai/v1`, `nvidia-nim/v1`) and injects `cf-aig-cache-ttl`, `cf-aig-cache-key`, and `cf-aig-skip-cache` headers from `LLMRequest.gatewayMetadata` (each header only when its field is defined). An explicit `baseUrl` always wins and disables gateway-active header injection. Empty `accountId`/`gatewayId` throws `CfGatewayInvalidConfigError` (`CF_GATEWAY_INVALID_CONFIG`) synchronously in the constructor. New `GatewayMetadata.skipCache?: boolean` field and exported `CfGatewayConfig` type. + ## [1.18.0] — 2026-06-19 ### Added diff --git a/README.md b/README.md index e392c48..dda7871 100644 --- a/README.md +++ b/README.md @@ -146,6 +146,48 @@ const llm = LLMProviders.fromEnv(env, { { apiKey: 'nvapi-...' } ``` +## Cloudflare AI Gateway Passthrough + +Each HTTP provider (OpenAI, Anthropic, Cerebras, Groq, NVIDIA) accepts a `cfGateway` +option that routes requests through a [Cloudflare AI Gateway](https://developers.cloudflare.com/ai-gateway/) +without you hand-building the gateway URL: + +```typescript +const provider = new OpenAIProvider({ + apiKey: 'sk-...', + cfGateway: { accountId: 'YOUR_ACCOUNT_ID', gatewayId: 'my-gateway' }, +}); +// baseUrl is derived as: +// https://gateway.ai.cloudflare.com/v1/YOUR_ACCOUNT_ID/my-gateway/openai/v1 +``` + +Per-provider gateway suffixes: `openai/v1`, `anthropic`, `cerebras/v1`, +`groq/openai/v1`, `nvidia-nim/v1`. + +When `cfGateway` is active, the provider injects `cf-aig-*` headers from +`request.gatewayMetadata` on each call (each header only when its field is set): + +| `gatewayMetadata` field | Header | Wire value | +| ----------------------- | ------------------ | ----------------- | +| `cacheTtl` (number) | `cf-aig-cache-ttl` | `"300"` | +| `cacheKey` (string) | `cf-aig-cache-key` | passthrough | +| `skipCache` (boolean) | `cf-aig-skip-cache`| `"true"`/`"false"`| + +An explicit `baseUrl` always wins: if you set both `baseUrl` and `cfGateway`, the +`baseUrl` is used verbatim and no `cf-aig-*` headers are injected. Empty +`accountId` or `gatewayId` throws `CfGatewayInvalidConfigError` +(`CF_GATEWAY_INVALID_CONFIG`) synchronously in the constructor. + +### Composable with semantic routing + +`cfGateway` is network-layer infrastructure and composes with this package's +application-layer concerns rather than replacing them. Use **llm-providers** for +semantic routing (use-case → model selection) and per-provider circuit breakers; +use **Cloudflare AI Gateway** for network-layer load balancing, response caching, +and upstream 429 interception. The factory still picks the provider/model and +trips its breaker on failures; the gateway sits underneath each provider's +outbound HTTP and handles caching and rate-limit absorption transparently. + ## Vision Inputs Vision requests use `request.images`. The factory routes image inputs only to providers that advertise `supportsVision === true`; providers that cannot see images reject the request instead of silently dropping image content. diff --git a/src/__tests__/cf-gateway.test.ts b/src/__tests__/cf-gateway.test.ts new file mode 100644 index 0000000..07da93d --- /dev/null +++ b/src/__tests__/cf-gateway.test.ts @@ -0,0 +1,209 @@ +/** + * Cloudflare AI Gateway passthrough (cfGateway) tests + * Covers URL derivation, constructor validation, header injection/omission, + * and the baseUrl override regression across all five HTTP providers. + */ + +import { describe, it, expect, beforeEach, vi } from 'vitest'; +import { OpenAIProvider } from '../providers/openai'; +import { AnthropicProvider } from '../providers/anthropic'; +import { CerebrasProvider } from '../providers/cerebras'; +import { GroqProvider } from '../providers/groq'; +import { NvidiaProvider } from '../providers/nvidia'; +import { CfGatewayInvalidConfigError } from '../errors'; +import { defaultCircuitBreakerManager } from '../utils/circuit-breaker'; +import type { LLMRequest } from '../types'; + +const mockFetch = vi.fn(); +vi.stubGlobal('fetch', mockFetch); + +const ACCOUNT = 'acct-123'; +const GATEWAY = 'gw-abc'; + +const chatCompletionBody = (model: string) => ({ + ok: true, + json: async () => ({ + id: 'chatcmpl-123', + object: 'chat.completion', + created: 1700000000, + model, + choices: [{ index: 0, message: { role: 'assistant', content: 'Hi!' }, finish_reason: 'stop' }], + usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 }, + }), + headers: new Headers({ 'content-type': 'application/json' }), +}); + +const anthropicBody = () => ({ + ok: true, + json: async () => ({ + id: 'msg_123', + type: 'message', + role: 'assistant', + model: 'claude-3-5-haiku-20241022', + content: [{ type: 'text', text: 'Hi!' }], + stop_reason: 'end_turn', + usage: { input_tokens: 10, output_tokens: 5 }, + }), + headers: new Headers({ 'content-type': 'application/json' }), +}); + +const baseRequest: LLMRequest = { + messages: [{ role: 'user', content: 'Hello' }], + maxTokens: 100, +}; + +beforeEach(() => { + vi.clearAllMocks(); + defaultCircuitBreakerManager.resetAll(); +}); + +describe('cfGateway URL derivation', () => { + const expectedUrl = (suffix: string) => + `https://gateway.ai.cloudflare.com/v1/${ACCOUNT}/${GATEWAY}/${suffix}`; + + it('derives the OpenAI gateway URL', async () => { + mockFetch.mockResolvedValueOnce(chatCompletionBody('gpt-4o-mini')); + const provider = new OpenAIProvider({ apiKey: 'k', cfGateway: { accountId: ACCOUNT, gatewayId: GATEWAY } }); + await provider.generateResponse({ ...baseRequest, model: 'gpt-4o-mini' }); + expect(mockFetch.mock.calls[0][0]).toBe(`${expectedUrl('openai/v1')}/chat/completions`); + }); + + it('derives the Anthropic gateway URL (no version segment)', async () => { + mockFetch.mockResolvedValueOnce(anthropicBody()); + const provider = new AnthropicProvider({ apiKey: 'k', cfGateway: { accountId: ACCOUNT, gatewayId: GATEWAY } }); + await provider.generateResponse({ ...baseRequest, model: 'claude-3-5-haiku-20241022' }); + expect(mockFetch.mock.calls[0][0]).toBe(`${expectedUrl('anthropic')}/v1/messages`); + }); + + it('derives the Cerebras gateway URL', async () => { + mockFetch.mockResolvedValueOnce(chatCompletionBody('llama-3.1-8b')); + const provider = new CerebrasProvider({ apiKey: 'k', cfGateway: { accountId: ACCOUNT, gatewayId: GATEWAY } }); + await provider.generateResponse({ ...baseRequest, model: 'llama-3.1-8b' }); + expect(mockFetch.mock.calls[0][0]).toBe(`${expectedUrl('cerebras/v1')}/chat/completions`); + }); + + it('derives the Groq gateway URL', async () => { + mockFetch.mockResolvedValueOnce(chatCompletionBody('llama-3.3-70b-versatile')); + const provider = new GroqProvider({ apiKey: 'k', cfGateway: { accountId: ACCOUNT, gatewayId: GATEWAY } }); + await provider.generateResponse({ ...baseRequest, model: 'llama-3.3-70b-versatile' }); + expect(mockFetch.mock.calls[0][0]).toBe(`${expectedUrl('groq/openai/v1')}/chat/completions`); + }); + + it('derives the NVIDIA gateway URL', async () => { + mockFetch.mockResolvedValueOnce(chatCompletionBody('meta/llama-3.3-70b-instruct')); + const provider = new NvidiaProvider({ apiKey: 'k', cfGateway: { accountId: ACCOUNT, gatewayId: GATEWAY } }); + await provider.generateResponse({ ...baseRequest, model: 'meta/llama-3.3-70b-instruct' }); + expect(mockFetch.mock.calls[0][0]).toBe(`${expectedUrl('nvidia-nim/v1')}/chat/completions`); + }); +}); + +describe('cfGateway constructor validation', () => { + it('throws synchronously on empty accountId', () => { + expect(() => new OpenAIProvider({ apiKey: 'k', cfGateway: { accountId: '', gatewayId: GATEWAY } })) + .toThrow(CfGatewayInvalidConfigError); + try { + new OpenAIProvider({ apiKey: 'k', cfGateway: { accountId: '', gatewayId: GATEWAY } }); + } catch (err) { + expect((err as CfGatewayInvalidConfigError).code).toBe('CF_GATEWAY_INVALID_CONFIG'); + expect((err as CfGatewayInvalidConfigError).field).toBe('accountId'); + expect((err as CfGatewayInvalidConfigError).provider).toBe('openai'); + expect((err as Error).message).toContain('openai'); + } + }); + + it('throws synchronously on empty gatewayId', () => { + try { + new GroqProvider({ apiKey: 'k', cfGateway: { accountId: ACCOUNT, gatewayId: '' } }); + throw new Error('should have thrown'); + } catch (err) { + expect(err).toBeInstanceOf(CfGatewayInvalidConfigError); + expect((err as CfGatewayInvalidConfigError).field).toBe('gatewayId'); + expect((err as CfGatewayInvalidConfigError).provider).toBe('groq'); + } + }); +}); + +describe('cfGateway baseUrl override', () => { + it('explicit baseUrl wins and disables header injection', async () => { + mockFetch.mockResolvedValueOnce(chatCompletionBody('gpt-4o-mini')); + const provider = new OpenAIProvider({ + apiKey: 'k', + baseUrl: 'https://custom.example.com/v1', + cfGateway: { accountId: ACCOUNT, gatewayId: GATEWAY }, + }); + await provider.generateResponse({ + ...baseRequest, + model: 'gpt-4o-mini', + gatewayMetadata: { cacheTtl: 300, cacheKey: 'k1', skipCache: false }, + }); + const [url, options] = mockFetch.mock.calls[0]; + expect(url).toBe('https://custom.example.com/v1/chat/completions'); + expect(options.headers['cf-aig-cache-ttl']).toBeUndefined(); + expect(options.headers['cf-aig-cache-key']).toBeUndefined(); + expect(options.headers['cf-aig-skip-cache']).toBeUndefined(); + }); +}); + +describe('cfGateway header injection', () => { + it('injects all cf-aig-* headers when gatewayMetadata fields are set', async () => { + mockFetch.mockResolvedValueOnce(chatCompletionBody('gpt-4o-mini')); + const provider = new OpenAIProvider({ apiKey: 'k', cfGateway: { accountId: ACCOUNT, gatewayId: GATEWAY } }); + await provider.generateResponse({ + ...baseRequest, + model: 'gpt-4o-mini', + gatewayMetadata: { cacheTtl: 300, cacheKey: 'my-key', skipCache: true }, + }); + const headers = mockFetch.mock.calls[0][1].headers; + expect(headers['cf-aig-cache-ttl']).toBe('300'); + expect(headers['cf-aig-cache-key']).toBe('my-key'); + expect(headers['cf-aig-skip-cache']).toBe('true'); + }); + + it('serializes skipCache=false as the string "false"', async () => { + mockFetch.mockResolvedValueOnce(chatCompletionBody('gpt-4o-mini')); + const provider = new OpenAIProvider({ apiKey: 'k', cfGateway: { accountId: ACCOUNT, gatewayId: GATEWAY } }); + await provider.generateResponse({ + ...baseRequest, + model: 'gpt-4o-mini', + gatewayMetadata: { skipCache: false }, + }); + const headers = mockFetch.mock.calls[0][1].headers; + expect(headers['cf-aig-skip-cache']).toBe('false'); + }); + + it('omits headers whose gatewayMetadata fields are undefined', async () => { + mockFetch.mockResolvedValueOnce(chatCompletionBody('gpt-4o-mini')); + const provider = new OpenAIProvider({ apiKey: 'k', cfGateway: { accountId: ACCOUNT, gatewayId: GATEWAY } }); + await provider.generateResponse({ + ...baseRequest, + model: 'gpt-4o-mini', + gatewayMetadata: { cacheTtl: 60 }, + }); + const headers = mockFetch.mock.calls[0][1].headers; + expect(headers['cf-aig-cache-ttl']).toBe('60'); + expect(headers['cf-aig-cache-key']).toBeUndefined(); + expect(headers['cf-aig-skip-cache']).toBeUndefined(); + }); + + it('omits all cf-aig-* headers when gatewayMetadata is absent', async () => { + mockFetch.mockResolvedValueOnce(chatCompletionBody('gpt-4o-mini')); + const provider = new OpenAIProvider({ apiKey: 'k', cfGateway: { accountId: ACCOUNT, gatewayId: GATEWAY } }); + await provider.generateResponse({ ...baseRequest, model: 'gpt-4o-mini' }); + const headers = mockFetch.mock.calls[0][1].headers; + expect(headers['cf-aig-cache-ttl']).toBeUndefined(); + expect(headers['cf-aig-cache-key']).toBeUndefined(); + expect(headers['cf-aig-skip-cache']).toBeUndefined(); + }); + + it('does not inject cf-aig-* headers when gateway is inactive (default base URL)', async () => { + mockFetch.mockResolvedValueOnce(chatCompletionBody('gpt-4o-mini')); + const provider = new OpenAIProvider({ apiKey: 'k' }); + await provider.generateResponse({ + ...baseRequest, + model: 'gpt-4o-mini', + gatewayMetadata: { cacheTtl: 300, cacheKey: 'k1', skipCache: true }, + }); + const headers = mockFetch.mock.calls[0][1].headers; + expect(headers['cf-aig-skip-cache']).toBeUndefined(); + }); +}); diff --git a/src/errors.ts b/src/errors.ts index ac045e4..7f1e293 100755 --- a/src/errors.ts +++ b/src/errors.ts @@ -97,7 +97,22 @@ export class ConfigurationError extends LLMProviderError { } } -export class CircuitBreakerOpenError extends LLMProviderError { +export class CfGatewayInvalidConfigError extends LLMProviderError { + /** The specific cfGateway field that was empty or missing. */ + field: 'accountId' | 'gatewayId'; + + constructor(provider: string, field: 'accountId' | 'gatewayId') { + super( + `cfGateway.${field} must be a non-empty string for the ${provider} provider`, + 'CF_GATEWAY_INVALID_CONFIG', + provider, + false + ); + this.field = field; + } +} + +export class CircuitBreakerOpenError extends LLMProviderError { retryAfterSec: number; consecutiveFailures: number; @@ -111,47 +126,47 @@ export class CircuitBreakerOpenError extends LLMProviderError { ); this.retryAfterSec = retryAfterSec; this.consecutiveFailures = consecutiveFailures; - } -} - -export class ToolLoopLimitError extends LLMProviderError { - constructor(provider: string, message: string = 'Tool loop limit exceeded') { - super(message, 'TOOL_LOOP_LIMIT', provider, false, 400); - } -} - -export class ToolLoopAbortedError extends LLMProviderError { - constructor(provider: string, message: string = 'Tool loop aborted') { - super(message, 'TOOL_LOOP_ABORTED', provider, false, 400); - } -} - -/** - * Thrown when a provider's response envelope fails runtime schema validation. - * - * Indicates the upstream API silently changed shape - a field was renamed, - * removed, or had its type changed. Non-retryable: retrying hits the same - * broken shape. The factory treats this as fallback-eligible so traffic - * routes to a healthy provider while the drift is investigated. - */ -export class SchemaDriftError extends LLMProviderError { - path: string; - expected: string; - actual: string; - - constructor(provider: string, path: string, expected: string, actual: string) { - super( - `Response schema drift at ${path}: expected ${expected}, got ${actual}`, - 'SCHEMA_DRIFT', - provider, - false, - 422 - ); - this.path = path; - this.expected = expected; - this.actual = actual; - } -} + } +} + +export class ToolLoopLimitError extends LLMProviderError { + constructor(provider: string, message: string = 'Tool loop limit exceeded') { + super(message, 'TOOL_LOOP_LIMIT', provider, false, 400); + } +} + +export class ToolLoopAbortedError extends LLMProviderError { + constructor(provider: string, message: string = 'Tool loop aborted') { + super(message, 'TOOL_LOOP_ABORTED', provider, false, 400); + } +} + +/** + * Thrown when a provider's response envelope fails runtime schema validation. + * + * Indicates the upstream API silently changed shape - a field was renamed, + * removed, or had its type changed. Non-retryable: retrying hits the same + * broken shape. The factory treats this as fallback-eligible so traffic + * routes to a healthy provider while the drift is investigated. + */ +export class SchemaDriftError extends LLMProviderError { + path: string; + expected: string; + actual: string; + + constructor(provider: string, path: string, expected: string, actual: string) { + super( + `Response schema drift at ${path}: expected ${expected}, got ${actual}`, + 'SCHEMA_DRIFT', + provider, + false, + 422 + ); + this.path = path; + this.expected = expected; + this.actual = actual; + } +} /** * Error factory for creating provider-specific errors from HTTP responses diff --git a/src/index.ts b/src/index.ts index 9caf1d8..686ecb6 100755 --- a/src/index.ts +++ b/src/index.ts @@ -8,10 +8,11 @@ export type { LLMProvider, LLMImageInput, - GatewayMetadata, - CacheHints, - CacheObservability, - ResponseCacheAdapter, + GatewayMetadata, + CfGatewayConfig, + CacheHints, + CacheObservability, + ResponseCacheAdapter, LLMRequest, LLMResponse, LLMMessage, @@ -23,12 +24,12 @@ export type { BuiltInToolResult, ToolCall, ToolResult, - ProviderConfig, - OpenAIConfig, - AnthropicConfig, - CloudflareAIGatewayOptions, - CloudflareConfig, - CerebrasConfig, + ProviderConfig, + OpenAIConfig, + AnthropicConfig, + CloudflareAIGatewayOptions, + CloudflareConfig, + CerebrasConfig, GroqConfig, NvidiaConfig, ModelCapabilities, @@ -58,43 +59,43 @@ export type { BatchRequest, BatchResponse, BatchJob -} from './types.js'; - -// Canonical provider contract -export { - canonicalToLLMRequest, - normalizeLLMRequest, - normalizeLLMResponse, -} from './canonical.js'; -export type { - CanonicalDegradation, - CanonicalFallbackHop, - CanonicalLLMRequest, - CanonicalLLMResponse, - CanonicalMediaPart, - CanonicalMessage, - CanonicalOutputFormat, - CanonicalProviderOptions, - CanonicalRequestMetadata, - CanonicalRequirements, - CanonicalResponseError, - CanonicalRoutingMetadata, - CanonicalSamplingOptions, - CanonicalTool, - CanonicalToolMode, -} from './canonical.js'; - -// Gateway / Worker route planning -export { getGatewayRoutePlan } from './gateway-routing.js'; -export type { - GatewayRouteCachePlan, - GatewayRouteCapabilityReport, - GatewayRoutePlan, - GatewayRouteRequirements, -} from './gateway-routing.js'; - -// Provider implementations -export { BaseProvider } from './providers/base.js'; +} from './types.js'; + +// Canonical provider contract +export { + canonicalToLLMRequest, + normalizeLLMRequest, + normalizeLLMResponse, +} from './canonical.js'; +export type { + CanonicalDegradation, + CanonicalFallbackHop, + CanonicalLLMRequest, + CanonicalLLMResponse, + CanonicalMediaPart, + CanonicalMessage, + CanonicalOutputFormat, + CanonicalProviderOptions, + CanonicalRequestMetadata, + CanonicalRequirements, + CanonicalResponseError, + CanonicalRoutingMetadata, + CanonicalSamplingOptions, + CanonicalTool, + CanonicalToolMode, +} from './canonical.js'; + +// Gateway / Worker route planning +export { getGatewayRoutePlan } from './gateway-routing.js'; +export type { + GatewayRouteCachePlan, + GatewayRouteCapabilityReport, + GatewayRoutePlan, + GatewayRouteRequirements, +} from './gateway-routing.js'; + +// Provider implementations +export { BaseProvider } from './providers/base.js'; export { OpenAIProvider } from './providers/openai.js'; export { AnthropicProvider } from './providers/anthropic.js'; export { CloudflareProvider } from './providers/cloudflare.js'; @@ -113,29 +114,29 @@ export { MODEL_CATALOG, MODEL_RECOMMENDATIONS, PROVIDER_FALLBACK_ORDER, - getCatalogEntry, - getProviderDefaultModel, - getProviderDefaultModelForWorkload, - getProviderForCatalogModel, - getProvidersForCatalogModel, - modelSupportsBuiltInTools, - getRecommendedModel, - getRecommendedModelForWorkload, - getRoutingInfo, - inferUseCaseFromRequest, - normalizeModelWorkload, - rankModels, -} from './model-catalog.js'; -export type { - ModelCatalogEntry, - ModelLifecycle, - ModelPreferenceMap, - ModelRecommendationUseCase, - ModelSelectionContext, - ModelWorkloadClass, - ProviderName, - RoutingInfo, -} from './model-catalog.js'; + getCatalogEntry, + getProviderDefaultModel, + getProviderDefaultModelForWorkload, + getProviderForCatalogModel, + getProvidersForCatalogModel, + modelSupportsBuiltInTools, + getRecommendedModel, + getRecommendedModelForWorkload, + getRoutingInfo, + inferUseCaseFromRequest, + normalizeModelWorkload, + rankModels, +} from './model-catalog.js'; +export type { + ModelCatalogEntry, + ModelLifecycle, + ModelPreferenceMap, + ModelRecommendationUseCase, + ModelSelectionContext, + ModelWorkloadClass, + ProviderName, + RoutingInfo, +} from './model-catalog.js'; // Local imports for use within this file import { LLMProviderFactory } from './factory.js'; @@ -168,6 +169,7 @@ export { ContentFilterError, TokenLimitError, ConfigurationError, + CfGatewayInvalidConfigError, CircuitBreakerOpenError, SchemaDriftError, ToolLoopLimitError, @@ -195,11 +197,11 @@ export type { Logger } from './utils/logger.js'; // Observability hooks export { noopHooks, composeHooks } from './utils/hooks.js'; export type { - ObservabilityHooks, - RequestStartEvent, - RequestEndEvent, - CacheEvent, - RequestErrorEvent, + ObservabilityHooks, + RequestStartEvent, + RequestEndEvent, + CacheEvent, + RequestErrorEvent, RetryEvent, FallbackEvent, CircuitStateChangeEvent, @@ -211,9 +213,9 @@ export type { SchemaDriftEvent, } from './utils/hooks.js'; -// Exhaustion registry -export { ExhaustionRegistry, defaultExhaustionRegistry } from './utils/exhaustion.js'; -export type { ExhaustionEntry, ExhaustionRegistrySnapshot } from './utils/exhaustion.js'; +// Exhaustion registry +export { ExhaustionRegistry, defaultExhaustionRegistry } from './utils/exhaustion.js'; +export type { ExhaustionEntry, ExhaustionRegistrySnapshot } from './utils/exhaustion.js'; // Latency histogram export { LatencyHistogram, defaultLatencyHistogram } from './utils/latency-histogram.js'; @@ -221,12 +223,12 @@ export type { LatencySummary } from './utils/latency-histogram.js'; // Utility classes export { RetryManager, defaultRetryManager, withRetry, retry } from './utils/retry.js'; -export { - CircuitBreaker, - CircuitBreakerManager, - defaultCircuitBreakerManager -} from './utils/circuit-breaker.js'; -export type { CircuitBreakerManagerSnapshot, CircuitBreakerSnapshot } from './utils/circuit-breaker.js'; +export { + CircuitBreaker, + CircuitBreakerManager, + defaultCircuitBreakerManager +} from './utils/circuit-breaker.js'; +export type { CircuitBreakerManagerSnapshot, CircuitBreakerSnapshot } from './utils/circuit-breaker.js'; export { CostTracker, CostOptimizer, @@ -495,11 +497,11 @@ export function createLLMProviders(config: ProviderFactoryConfig): LLMProviders */ export default LLMProviders; -/** - * Version and metadata - */ -export { VERSION } from './version.js'; -export const SUPPORTED_PROVIDERS = ['openai', 'anthropic', 'cloudflare', 'cerebras', 'groq', 'nvidia'] as const; +/** + * Version and metadata + */ +export { VERSION } from './version.js'; +export const SUPPORTED_PROVIDERS = ['openai', 'anthropic', 'cloudflare', 'cerebras', 'groq', 'nvidia'] as const; /** * Common model mappings for easy reference diff --git a/src/providers/anthropic.ts b/src/providers/anthropic.ts index d51b1d6..6fc2055 100755 --- a/src/providers/anthropic.ts +++ b/src/providers/anthropic.ts @@ -13,7 +13,7 @@ import type { ToolCall, TokenUsage } from '../types.js'; -import { BaseProvider } from './base.js'; +import { BaseProvider, resolveCfGateway } from './base.js'; import { LLMErrorFactory, AuthenticationError, @@ -173,7 +173,15 @@ export class AnthropicProvider extends BaseProvider { } this.apiKey = config.apiKey; - this.baseUrl = config.baseUrl || 'https://api.anthropic.com'; + const resolved = resolveCfGateway({ + provider: 'anthropic', + baseUrl: config.baseUrl, + cfGateway: config.cfGateway, + suffix: 'anthropic', + defaultBaseUrl: 'https://api.anthropic.com', + }); + this.baseUrl = resolved.resolvedBaseUrl; + this.cfGatewayActive = resolved.cfGatewayActive; this.version = config.version || '2023-06-01'; } @@ -373,7 +381,8 @@ export class AnthropicProvider extends BaseProvider { 'x-api-key': this.apiKey, 'anthropic-version': this.version, 'Content-Type': 'application/json', - ...this.getAIGatewayHeaders(request) + ...this.getAIGatewayHeaders(request), + ...this.getCfGatewayHeaders(request) }; const options: RequestInit = { diff --git a/src/providers/base.ts b/src/providers/base.ts index 3f0e382..dc865c9 100755 --- a/src/providers/base.ts +++ b/src/providers/base.ts @@ -3,34 +3,92 @@ * Abstract base class for all LLM providers with common functionality */ -import type { - LLMProvider, - LLMImageInput, - LLMRequest, - LLMResponse, - ProviderConfig, - ModelCapabilities, - ProviderMetrics, - ToolCall, - TokenUsage -} from '../types.js'; +import type { + LLMProvider, + LLMImageInput, + LLMRequest, + LLMResponse, + ProviderConfig, + ModelCapabilities, + ProviderMetrics, + ToolCall, + TokenUsage +} from '../types.js'; +import type { CfGatewayConfig } from '../types.js'; import type { Logger } from '../utils/logger.js'; import { noopLogger } from '../utils/logger.js'; import { RetryManager } from '../utils/retry.js'; import { CircuitBreaker, defaultCircuitBreakerManager } from '../utils/circuit-breaker.js'; import { CostTracker } from '../utils/cost-tracker.js'; import { defaultLatencyHistogram } from '../utils/latency-histogram.js'; -import { ConfigurationError, TimeoutError, InvalidRequestError } from '../errors.js'; +import { ConfigurationError, TimeoutError, InvalidRequestError, CfGatewayInvalidConfigError } from '../errors.js'; + +/** Per-provider path suffix on the Cloudflare AI Gateway base URL. */ +export type CfGatewayProviderSuffix = + | 'openai/v1' + | 'anthropic' + | 'cerebras/v1' + | 'groq/openai/v1' + | 'nvidia-nim/v1'; + +export interface CfGatewayResolvedState { + resolvedBaseUrl: string; + cfGatewayActive: boolean; +} + +/** + * Resolve a provider base URL from explicit override / cfGateway / default. + * Precedence: explicit `baseUrl` wins (cfGatewayActive=false); else derive from + * `cfGateway`; else fall back to `defaultBaseUrl`. Throws synchronously when + * `cfGateway` is supplied with an empty `accountId` or `gatewayId`. + */ +export function resolveCfGateway(args: { + provider: string; + baseUrl?: string; + cfGateway?: CfGatewayConfig; + suffix: CfGatewayProviderSuffix; + defaultBaseUrl: string; +}): CfGatewayResolvedState { + const { provider, baseUrl, cfGateway, suffix, defaultBaseUrl } = args; + + if (cfGateway) { + if (!cfGateway.accountId) { + throw new CfGatewayInvalidConfigError(provider, 'accountId'); + } + if (!cfGateway.gatewayId) { + throw new CfGatewayInvalidConfigError(provider, 'gatewayId'); + } + } + + if (baseUrl) { + return { resolvedBaseUrl: baseUrl, cfGatewayActive: false }; + } + + if (cfGateway) { + return { + resolvedBaseUrl: `https://gateway.ai.cloudflare.com/v1/${cfGateway.accountId}/${cfGateway.gatewayId}/${suffix}`, + cfGatewayActive: true, + }; + } + + return { resolvedBaseUrl: defaultBaseUrl, cfGatewayActive: false }; +} export abstract class BaseProvider implements LLMProvider { abstract name: string; abstract models: string[]; - abstract supportsStreaming: boolean; - abstract supportsTools: boolean; - abstract supportsBatching: boolean; - abstract supportsVision?: boolean; + abstract supportsStreaming: boolean; + abstract supportsTools: boolean; + abstract supportsBatching: boolean; + abstract supportsVision?: boolean; protected config: ProviderConfig; + /** + * True when this provider derived its base URL from `cfGateway` (no explicit + * `baseUrl` override). Gates injection of cf-aig-* headers in makeRequest. + * Set by each provider constructor via {@link resolveCfGateway}. + */ + protected cfGatewayActive = false; protected logger: Logger; protected retryManager: RetryManager; protected circuitBreaker: CircuitBreaker; @@ -252,7 +310,7 @@ export abstract class BaseProvider implements LLMProvider { /** * Validate request before processing */ - protected validateRequest(request: LLMRequest): void { + protected validateRequest(request: LLMRequest): void { if (!request.messages || request.messages.length === 0) { throw new ConfigurationError(this.name, 'Request must contain at least one message'); } @@ -266,67 +324,93 @@ export abstract class BaseProvider implements LLMProvider { } // Validate model if specified - if (request.model && !this.models.includes(request.model)) { - throw new ConfigurationError( - this.name, - `Model '${request.model}' not supported. Available models: ${this.models.join(', ')}` - ); - } - - if ((request.images?.length ?? 0) > 0 && this.supportsVision !== true) { - throw new ConfigurationError(this.name, `${this.name} does not support image input`); - } - - if (request.images) { - for (const image of request.images) { - this.validateImageInput(image); - } - } - } - - protected estimateTextTokens(request: LLMRequest): number { - return ( - (request.systemPrompt ? Math.ceil(request.systemPrompt.length / 4) : 0) + - request.messages.reduce((sum, msg) => sum + Math.ceil(msg.content.length / 4), 0) - ); - } - - protected validateImageInput(image: LLMImageInput): void { - if (!image.data && !image.url) { - throw new ConfigurationError(this.name, 'Image input must include data or url'); - } - - if (image.data && !image.mimeType) { - throw new ConfigurationError(this.name, 'Base64 image input must include mimeType'); - } - } - - protected getAIGatewayHeaders(request?: LLMRequest): Record { - const baseUrl = typeof this.config.baseUrl === 'string' ? this.config.baseUrl : ''; - if (!baseUrl.includes('gateway.ai.cloudflare.com') || !request?.gatewayMetadata) { - return {}; - } - - const headers: Record = {}; - const metadata = { - ...(request.gatewayMetadata.customMetadata ?? {}), - ...(request.gatewayMetadata.requestId ? { requestId: request.gatewayMetadata.requestId } : {}), - ...(request.requestId ? { llmRequestId: request.requestId } : {}), - ...(request.tenantId ? { tenantId: request.tenantId } : {}), - }; - - if (Object.keys(metadata).length > 0) { - headers['cf-aig-metadata'] = JSON.stringify(metadata); - } - if (request.gatewayMetadata.cacheKey) { - headers['cf-aig-cache-key'] = request.gatewayMetadata.cacheKey; - } - if (typeof request.gatewayMetadata.cacheTtl === 'number') { - headers['cf-aig-cache-ttl'] = String(request.gatewayMetadata.cacheTtl); - } - - return headers; - } + if (request.model && !this.models.includes(request.model)) { + throw new ConfigurationError( + this.name, + `Model '${request.model}' not supported. Available models: ${this.models.join(', ')}` + ); + } + + if ((request.images?.length ?? 0) > 0 && this.supportsVision !== true) { + throw new ConfigurationError(this.name, `${this.name} does not support image input`); + } + + if (request.images) { + for (const image of request.images) { + this.validateImageInput(image); + } + } + } + + protected estimateTextTokens(request: LLMRequest): number { + return ( + (request.systemPrompt ? Math.ceil(request.systemPrompt.length / 4) : 0) + + request.messages.reduce((sum, msg) => sum + Math.ceil(msg.content.length / 4), 0) + ); + } + + protected validateImageInput(image: LLMImageInput): void { + if (!image.data && !image.url) { + throw new ConfigurationError(this.name, 'Image input must include data or url'); + } + + if (image.data && !image.mimeType) { + throw new ConfigurationError(this.name, 'Base64 image input must include mimeType'); + } + } + + protected getAIGatewayHeaders(request?: LLMRequest): Record { + const baseUrl = typeof this.config.baseUrl === 'string' ? this.config.baseUrl : ''; + if (!baseUrl.includes('gateway.ai.cloudflare.com') || !request?.gatewayMetadata) { + return {}; + } + + const headers: Record = {}; + const metadata = { + ...(request.gatewayMetadata.customMetadata ?? {}), + ...(request.gatewayMetadata.requestId ? { requestId: request.gatewayMetadata.requestId } : {}), + ...(request.requestId ? { llmRequestId: request.requestId } : {}), + ...(request.tenantId ? { tenantId: request.tenantId } : {}), + }; + + if (Object.keys(metadata).length > 0) { + headers['cf-aig-metadata'] = JSON.stringify(metadata); + } + if (request.gatewayMetadata.cacheKey) { + headers['cf-aig-cache-key'] = request.gatewayMetadata.cacheKey; + } + if (typeof request.gatewayMetadata.cacheTtl === 'number') { + headers['cf-aig-cache-ttl'] = String(request.gatewayMetadata.cacheTtl); + } + + return headers; + } + + /** + * cf-aig-* headers injected when this provider was configured via `cfGateway` + * (cfGatewayActive). Derived from `request.gatewayMetadata`; each header is + * only present when its source field is defined. + */ + protected getCfGatewayHeaders(request?: LLMRequest): Record { + if (!this.cfGatewayActive || !request?.gatewayMetadata) { + return {}; + } + + const headers: Record = {}; + const { cacheTtl, cacheKey, skipCache } = request.gatewayMetadata; + + if (typeof cacheTtl === 'number') { + headers['cf-aig-cache-ttl'] = String(cacheTtl); + } + if (typeof cacheKey === 'string') { + headers['cf-aig-cache-key'] = cacheKey; + } + if (typeof skipCache === 'boolean') { + headers['cf-aig-skip-cache'] = String(skipCache); + } + + return headers; + } /** * Validate and sanitize tool calls returned by the provider. @@ -398,60 +482,60 @@ export abstract class BaseProvider implements LLMProvider { /** * Calculate token usage cost */ - protected calculateCost( - inputTokens: number, - outputTokens: number, - model: string - ): number { + protected calculateCost( + inputTokens: number, + outputTokens: number, + model: string + ): number { const capabilities = this.getModelCapabilities()[model]; if (!capabilities) return 0; const inputCost = (inputTokens / 1000) * capabilities.inputTokenCost; const outputCost = (outputTokens / 1000) * capabilities.outputTokenCost; - - return inputCost + outputCost; - } - - protected parseOpenAICompatibleStreamUsage( - rawUsage: unknown, - model: string - ): TokenUsage | undefined { - if (!rawUsage || typeof rawUsage !== 'object' || Array.isArray(rawUsage)) return undefined; - - const usage = rawUsage as Record; - const inputTokens = usage['prompt_tokens']; - const outputTokens = usage['completion_tokens']; - const totalTokens = usage['total_tokens']; - - if ( - typeof inputTokens !== 'number' || - typeof outputTokens !== 'number' || - typeof totalTokens !== 'number' - ) { - return undefined; - } - - const parsed: TokenUsage = { - inputTokens, - outputTokens, - totalTokens, - cost: this.calculateCost(inputTokens, outputTokens, model), - }; - - const details = usage['prompt_tokens_details']; - if (details && typeof details === 'object' && !Array.isArray(details)) { - const cachedTokens = (details as Record)['cached_tokens']; - if (typeof cachedTokens === 'number') { - parsed.cachedInputTokens = cachedTokens; - } - } - - return parsed; - } - - /** - * Common response formatting - */ + + return inputCost + outputCost; + } + + protected parseOpenAICompatibleStreamUsage( + rawUsage: unknown, + model: string + ): TokenUsage | undefined { + if (!rawUsage || typeof rawUsage !== 'object' || Array.isArray(rawUsage)) return undefined; + + const usage = rawUsage as Record; + const inputTokens = usage['prompt_tokens']; + const outputTokens = usage['completion_tokens']; + const totalTokens = usage['total_tokens']; + + if ( + typeof inputTokens !== 'number' || + typeof outputTokens !== 'number' || + typeof totalTokens !== 'number' + ) { + return undefined; + } + + const parsed: TokenUsage = { + inputTokens, + outputTokens, + totalTokens, + cost: this.calculateCost(inputTokens, outputTokens, model), + }; + + const details = usage['prompt_tokens_details']; + if (details && typeof details === 'object' && !Array.isArray(details)) { + const cachedTokens = (details as Record)['cached_tokens']; + if (typeof cachedTokens === 'number') { + parsed.cachedInputTokens = cachedTokens; + } + } + + return parsed; + } + + /** + * Common response formatting + */ protected buildResponse( content: string, usage: { inputTokens: number; outputTokens: number }, diff --git a/src/providers/cerebras.ts b/src/providers/cerebras.ts index 02c9b4e..57717ae 100644 --- a/src/providers/cerebras.ts +++ b/src/providers/cerebras.ts @@ -4,7 +4,7 @@ */ import type { LLMRequest, LLMResponse, CerebrasConfig, ModelCapabilities, ToolCall, TokenUsage } from '../types.js'; -import { BaseProvider } from './base.js'; +import { BaseProvider, resolveCfGateway } from './base.js'; import { LLMErrorFactory, AuthenticationError, @@ -161,7 +161,15 @@ export class CerebrasProvider extends BaseProvider { } this.apiKey = config.apiKey; - this.baseUrl = config.baseUrl || 'https://api.cerebras.ai/v1'; + const resolved = resolveCfGateway({ + provider: 'cerebras', + baseUrl: config.baseUrl, + cfGateway: config.cfGateway, + suffix: 'cerebras/v1', + defaultBaseUrl: 'https://api.cerebras.ai/v1', + }); + this.baseUrl = resolved.resolvedBaseUrl; + this.cfGatewayActive = resolved.cfGatewayActive; } async generateResponse(request: LLMRequest): Promise { @@ -403,7 +411,8 @@ export class CerebrasProvider extends BaseProvider { const headers: Record = { 'Authorization': `Bearer ${this.apiKey}`, 'Content-Type': 'application/json', - ...this.getAIGatewayHeaders(request) + ...this.getAIGatewayHeaders(request), + ...this.getCfGatewayHeaders(request) }; const options: RequestInit = { diff --git a/src/providers/groq.ts b/src/providers/groq.ts index 72c02b9..a282bd6 100644 --- a/src/providers/groq.ts +++ b/src/providers/groq.ts @@ -4,7 +4,7 @@ */ import type { LLMRequest, LLMResponse, GroqConfig, ModelCapabilities, ProviderBalance, ToolCall, TokenUsage, BuiltInTool, BuiltInToolType, BuiltInToolResult } from '../types.js'; -import { BaseProvider } from './base.js'; +import { BaseProvider, resolveCfGateway } from './base.js'; import { LLMErrorFactory, AuthenticationError, @@ -212,7 +212,15 @@ export class GroqProvider extends BaseProvider { } this.apiKey = config.apiKey; - this.baseUrl = config.baseUrl || 'https://api.groq.com/openai/v1'; + const resolved = resolveCfGateway({ + provider: 'groq', + baseUrl: config.baseUrl, + cfGateway: config.cfGateway, + suffix: 'groq/openai/v1', + defaultBaseUrl: 'https://api.groq.com/openai/v1', + }); + this.baseUrl = resolved.resolvedBaseUrl; + this.cfGatewayActive = resolved.cfGatewayActive; } async generateResponse(request: LLMRequest): Promise { @@ -463,7 +471,8 @@ export class GroqProvider extends BaseProvider { const headers: Record = { 'Authorization': `Bearer ${this.apiKey}`, 'Content-Type': 'application/json', - ...this.getAIGatewayHeaders(request) + ...this.getAIGatewayHeaders(request), + ...this.getCfGatewayHeaders(request) }; const options: RequestInit = { diff --git a/src/providers/nvidia.ts b/src/providers/nvidia.ts index 58cfa43..6f61d12 100644 --- a/src/providers/nvidia.ts +++ b/src/providers/nvidia.ts @@ -4,7 +4,7 @@ */ import type { LLMRequest, LLMResponse, NvidiaConfig, ModelCapabilities, ProviderBalance, ToolCall, TokenUsage } from '../types.js'; -import { BaseProvider } from './base.js'; +import { BaseProvider, resolveCfGateway } from './base.js'; import { LLMErrorFactory, AuthenticationError, @@ -155,7 +155,15 @@ export class NvidiaProvider extends BaseProvider { } this.apiKey = config.apiKey; - this.baseUrl = config.baseUrl || 'https://integrate.api.nvidia.com/v1'; + const resolved = resolveCfGateway({ + provider: 'nvidia', + baseUrl: config.baseUrl, + cfGateway: config.cfGateway, + suffix: 'nvidia-nim/v1', + defaultBaseUrl: 'https://integrate.api.nvidia.com/v1', + }); + this.baseUrl = resolved.resolvedBaseUrl; + this.cfGatewayActive = resolved.cfGatewayActive; } async generateResponse(request: LLMRequest): Promise { @@ -438,7 +446,8 @@ export class NvidiaProvider extends BaseProvider { const headers: Record = { 'Authorization': `Bearer ${this.apiKey}`, 'Content-Type': 'application/json', - ...this.getAIGatewayHeaders(request) + ...this.getAIGatewayHeaders(request), + ...this.getCfGatewayHeaders(request) }; const options: RequestInit = { diff --git a/src/providers/openai.ts b/src/providers/openai.ts index 1ab5cd9..8c4a40f 100755 --- a/src/providers/openai.ts +++ b/src/providers/openai.ts @@ -4,7 +4,7 @@ */ import type { LLMRequest, LLMResponse, OpenAIConfig, ModelCapabilities, ToolCall, Tool, TokenUsage } from '../types.js'; -import { BaseProvider } from './base.js'; +import { BaseProvider, resolveCfGateway } from './base.js'; import { LLMErrorFactory, AuthenticationError, @@ -161,7 +161,15 @@ export class OpenAIProvider extends BaseProvider { } this.apiKey = config.apiKey; - this.baseUrl = config.baseUrl || 'https://api.openai.com/v1'; + const resolved = resolveCfGateway({ + provider: 'openai', + baseUrl: config.baseUrl, + cfGateway: config.cfGateway, + suffix: 'openai/v1', + defaultBaseUrl: 'https://api.openai.com/v1', + }); + this.baseUrl = resolved.resolvedBaseUrl; + this.cfGatewayActive = resolved.cfGatewayActive; this.organization = config.organization; this.project = config.project; } @@ -289,7 +297,8 @@ export class OpenAIProvider extends BaseProvider { const headers: Record = { 'Authorization': `Bearer ${this.apiKey}`, 'Content-Type': 'application/json', - ...this.getAIGatewayHeaders(request) + ...this.getAIGatewayHeaders(request), + ...this.getCfGatewayHeaders(request) }; if (this.organization) { diff --git a/src/types.ts b/src/types.ts index 094e77f..897277b 100755 --- a/src/types.ts +++ b/src/types.ts @@ -27,9 +27,28 @@ export interface GatewayMetadata { cacheKey?: string; /** cf-aig-cache-ttl: Cloudflare AI Gateway response cache TTL in seconds. */ cacheTtl?: number; + /** cf-aig-skip-cache: bypass the Cloudflare AI Gateway response cache for this request. */ + skipCache?: boolean; customMetadata?: Record; } +/** + * First-class Cloudflare AI Gateway passthrough config attached to an HTTP + * provider via `cfGateway`. When set (and no explicit `baseUrl` override is + * present), the provider derives its base URL as + * `https://gateway.ai.cloudflare.com/v1/{accountId}/{gatewayId}/{suffix}` and + * injects `cf-aig-*` headers from `LLMRequest.gatewayMetadata`. + * + * Both fields are required; an empty string for either throws synchronously in + * the provider constructor (CF_GATEWAY_INVALID_CONFIG). + */ +export interface CfGatewayConfig { + /** Cloudflare account ID. Must be non-empty. */ + accountId: string; + /** AI Gateway ID (the named gateway within the account). Must be non-empty. */ + gatewayId: string; +} + /** * Provider prompt/prefix cache hints. Distinct from GatewayMetadata.cacheKey/cacheTtl, * which control Cloudflare AI Gateway *response* caching. These hints control @@ -48,9 +67,9 @@ export interface ResponseCacheAdapter { put(key: string, value: string, ttlSeconds?: number): Promise; } -export interface CacheHints { - /** 'off' disables any provider cache opt-in. Default: let provider decide. */ - strategy?: 'off' | 'provider-prefix' | 'response' | 'both'; +export interface CacheHints { + /** 'off' disables any provider cache opt-in. Default: let provider decide. */ + strategy?: 'off' | 'provider-prefix' | 'response' | 'both'; /** Provider-specific cache key hint (e.g. OpenAI prompt_cache_key). */ key?: string; /** Desired TTL. Clamped to provider-supported values at translation time. */ @@ -58,37 +77,37 @@ export interface CacheHints { /** Cloudflare Workers AI: x-session-affinity value for prefix caching. */ sessionId?: string; /** Which prefix to mark cacheable when explicit breakpoints are required (Anthropic). */ - cacheablePrefix?: 'auto' | 'system' | 'tools' | 'messages'; -} - -export interface CacheObservability { - /** Provider-side prompt/prefix cache behavior, normalized from TokenUsage fields. */ - providerPrefix?: { - requested?: boolean; - strategy?: CacheHints['strategy']; - sessionId?: string; - cachedInputTokens?: number; - cacheReadInputTokens?: number; - cacheWriteInputTokens?: number; - cacheCreationInputTokens?: number; - hit?: boolean; - write?: boolean; - }; - /** Cloudflare AI Gateway exact-response cache behavior when the provider exposes it. */ - aiGateway?: { - requested?: boolean; - status?: string; - cacheKey?: string; - cacheTtl?: number; - }; - /** Factory/local response cache behavior through ResponseCacheAdapter. */ - factory?: { - status: 'hit' | 'miss' | 'write' | 'error' | 'bypass'; - key?: string; - ttlSeconds?: number; - error?: string; - }; -} + cacheablePrefix?: 'auto' | 'system' | 'tools' | 'messages'; +} + +export interface CacheObservability { + /** Provider-side prompt/prefix cache behavior, normalized from TokenUsage fields. */ + providerPrefix?: { + requested?: boolean; + strategy?: CacheHints['strategy']; + sessionId?: string; + cachedInputTokens?: number; + cacheReadInputTokens?: number; + cacheWriteInputTokens?: number; + cacheCreationInputTokens?: number; + hit?: boolean; + write?: boolean; + }; + /** Cloudflare AI Gateway exact-response cache behavior when the provider exposes it. */ + aiGateway?: { + requested?: boolean; + status?: string; + cacheKey?: string; + cacheTtl?: number; + }; + /** Factory/local response cache behavior through ResponseCacheAdapter. */ + factory?: { + status: 'hit' | 'miss' | 'write' | 'error' | 'bypass'; + key?: string; + ttlSeconds?: number; + error?: string; + }; +} export interface ToolCall { id: string; @@ -204,21 +223,21 @@ export interface BuiltInToolResult { results: Array<{ title: string; url: string; content: string; score: number }>; } -export interface LLMResponse { - id?: string; - /** Chain-of-thought / thinking trace when the provider exposes it separately. */ - reasoning?: string; - message: string; - content?: string; // Alternative to message for consistency - usage: TokenUsage; +export interface LLMResponse { + id?: string; + /** Chain-of-thought / thinking trace when the provider exposes it separately. */ + reasoning?: string; + message: string; + content?: string; // Alternative to message for consistency + usage: TokenUsage; model: string; provider: string; - responseTime: number; - finishReason?: 'stop' | 'length' | 'tool_calls' | 'content_filter'; - toolCalls?: ToolCall[]; - cache?: CacheObservability; - metadata?: Record; -} + responseTime: number; + finishReason?: 'stop' | 'length' | 'tool_calls' | 'content_filter'; + toolCalls?: ToolCall[]; + cache?: CacheObservability; + metadata?: Record; +} export interface TokenUsage { inputTokens: number; @@ -285,10 +304,14 @@ export interface ProviderConfig { export interface OpenAIConfig extends ProviderConfig { organization?: string; project?: string; + /** Route through Cloudflare AI Gateway. Ignored when `baseUrl` is set. */ + cfGateway?: CfGatewayConfig; } export interface AnthropicConfig extends ProviderConfig { version?: string; + /** Route through Cloudflare AI Gateway. Ignored when `baseUrl` is set. */ + cfGateway?: CfGatewayConfig; } export interface CloudflareAIGatewayOptions { @@ -316,14 +339,20 @@ export interface CloudflareConfig extends ProviderConfig { export interface CerebrasConfig extends ProviderConfig { // Cerebras uses OpenAI-compatible API; no extra fields needed beyond ProviderConfig + /** Route through Cloudflare AI Gateway. Ignored when `baseUrl` is set. */ + cfGateway?: CfGatewayConfig; } export interface GroqConfig extends ProviderConfig { // Groq uses OpenAI-compatible API; no extra fields needed beyond ProviderConfig + /** Route through Cloudflare AI Gateway. Ignored when `baseUrl` is set. */ + cfGateway?: CfGatewayConfig; } export interface NvidiaConfig extends ProviderConfig { // NVIDIA NIM uses OpenAI-compatible API; no extra fields needed beyond ProviderConfig + /** Route through Cloudflare AI Gateway. Ignored when `baseUrl` is set. */ + cfGateway?: CfGatewayConfig; } export interface LLMError extends Error {