diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..d7b9f70 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,8 @@ +# Normalize line endings to LF on commit across all platforms +* text=auto eol=lf +*.ts text eol=lf +*.js text eol=lf +*.mjs text eol=lf +*.json text eol=lf +*.md text eol=lf +*.sh text eol=lf diff --git a/CHANGELOG.md b/CHANGELOG.md index 479147e..2098628 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,8 +5,16 @@ Format follows [Keep a Changelog](https://keepachangelog.com/). Versions use [Se ## [Unreleased] +## [1.19.0] — 2026-06-23 + ### Added -- **First-class `cfGateway` config (#97)** — `OpenAIConfig`, `AnthropicConfig`, `CerebrasConfig`, `GroqConfig`, and `NvidiaConfig` accept `cfGateway?: { accountId, gatewayId }`. When set (and no explicit `baseUrl` override is present), the provider derives its base URL as `https://gateway.ai.cloudflare.com/v1/{accountId}/{gatewayId}/{suffix}` per provider (`openai/v1`, `anthropic`, `cerebras/v1`, `groq/openai/v1`, `nvidia-nim/v1`) and injects `cf-aig-cache-ttl`, `cf-aig-cache-key`, and `cf-aig-skip-cache` headers from `LLMRequest.gatewayMetadata` (each header only when its field is defined). An explicit `baseUrl` always wins and disables gateway-active header injection. Empty `accountId`/`gatewayId` throws `CfGatewayInvalidConfigError` (`CF_GATEWAY_INVALID_CONFIG`) synchronously in the constructor. New `GatewayMetadata.skipCache?: boolean` field and exported `CfGatewayConfig` type. +- **First-class `cfGateway` config (#97)** — All five HTTP providers (`OpenAIConfig`, `AnthropicConfig`, `CerebrasConfig`, `GroqConfig`, `NvidiaConfig`) accept `cfGateway?: { accountId, gatewayId }`. + - When set (and no explicit `baseUrl` override), the provider derives its base URL as `https://gateway.ai.cloudflare.com/v1/{accountId}/{gatewayId}/{suffix}` — suffixes: `openai/v1`, `anthropic`, `cerebras/v1`, `groq/openai/v1`, `nvidia-nim/v1`. + - `cf-aig-cache-ttl`, `cf-aig-cache-key`, and `cf-aig-skip-cache` headers are injected per-call from `LLMRequest.gatewayMetadata` (each header only when its field is set). + - An explicit `baseUrl` always wins and suppresses gateway header injection. + - Empty `accountId` or `gatewayId` throws `CfGatewayInvalidConfigError` (`CF_GATEWAY_INVALID_CONFIG`) synchronously in the constructor. +- **`GatewayMetadata.skipCache?: boolean`** — new field; maps to `cf-aig-skip-cache` when `cfGateway` is active. +- **`CfGatewayConfig`** — exported type for the `cfGateway` option shape. ## [1.18.0] — 2026-06-19 diff --git a/package-lock.json b/package-lock.json index baca9ee..f706649 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@stackbilt/llm-providers", - "version": "1.15.1", + "version": "1.19.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@stackbilt/llm-providers", - "version": "1.15.1", + "version": "1.19.0", "license": "Apache-2.0", "devDependencies": { "@cloudflare/workers-types": "^4.0.0", diff --git a/package.json b/package.json index 0d631c0..8b073fe 100755 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@stackbilt/llm-providers", - "version": "1.18.0", + "version": "1.19.0", "description": "Multi-LLM failover with circuit breakers, cost tracking, and intelligent retry. Cloudflare Workers native.", "author": "Stackbilt ", "license": "Apache-2.0", diff --git a/src/providers/base.ts b/src/providers/base.ts index dc865c9..753f92e 100755 --- a/src/providers/base.ts +++ b/src/providers/base.ts @@ -1,587 +1,588 @@ -/** - * Base Provider - * Abstract base class for all LLM providers with common functionality - */ - -import type { - LLMProvider, - LLMImageInput, - LLMRequest, - LLMResponse, - ProviderConfig, - ModelCapabilities, - ProviderMetrics, - ToolCall, - TokenUsage -} from '../types.js'; -import type { CfGatewayConfig } from '../types.js'; -import type { Logger } from '../utils/logger.js'; -import { noopLogger } from '../utils/logger.js'; -import { RetryManager } from '../utils/retry.js'; -import { CircuitBreaker, defaultCircuitBreakerManager } from '../utils/circuit-breaker.js'; -import { CostTracker } from '../utils/cost-tracker.js'; -import { defaultLatencyHistogram } from '../utils/latency-histogram.js'; -import { ConfigurationError, TimeoutError, InvalidRequestError, CfGatewayInvalidConfigError } from '../errors.js'; - -/** Per-provider path suffix on the Cloudflare AI Gateway base URL. */ -export type CfGatewayProviderSuffix = - | 'openai/v1' - | 'anthropic' - | 'cerebras/v1' - | 'groq/openai/v1' - | 'nvidia-nim/v1'; - -export interface CfGatewayResolvedState { - resolvedBaseUrl: string; - cfGatewayActive: boolean; -} - -/** - * Resolve a provider base URL from explicit override / cfGateway / default. - * Precedence: explicit `baseUrl` wins (cfGatewayActive=false); else derive from - * `cfGateway`; else fall back to `defaultBaseUrl`. Throws synchronously when - * `cfGateway` is supplied with an empty `accountId` or `gatewayId`. - */ -export function resolveCfGateway(args: { - provider: string; - baseUrl?: string; - cfGateway?: CfGatewayConfig; - suffix: CfGatewayProviderSuffix; - defaultBaseUrl: string; -}): CfGatewayResolvedState { - const { provider, baseUrl, cfGateway, suffix, defaultBaseUrl } = args; - - if (cfGateway) { - if (!cfGateway.accountId) { - throw new CfGatewayInvalidConfigError(provider, 'accountId'); - } - if (!cfGateway.gatewayId) { - throw new CfGatewayInvalidConfigError(provider, 'gatewayId'); - } - } - - if (baseUrl) { - return { resolvedBaseUrl: baseUrl, cfGatewayActive: false }; - } - - if (cfGateway) { - return { - resolvedBaseUrl: `https://gateway.ai.cloudflare.com/v1/${cfGateway.accountId}/${cfGateway.gatewayId}/${suffix}`, - cfGatewayActive: true, - }; - } - - return { resolvedBaseUrl: defaultBaseUrl, cfGatewayActive: false }; -} - -export abstract class BaseProvider implements LLMProvider { - abstract name: string; - abstract models: string[]; - abstract supportsStreaming: boolean; - abstract supportsTools: boolean; - abstract supportsBatching: boolean; - abstract supportsVision?: boolean; - - protected config: ProviderConfig; - /** - * True when this provider derived its base URL from `cfGateway` (no explicit - * `baseUrl` override). Gates injection of cf-aig-* headers in makeRequest. - * Set by each provider constructor via {@link resolveCfGateway}. - */ - protected cfGatewayActive = false; - protected logger: Logger; - protected retryManager: RetryManager; - protected circuitBreaker: CircuitBreaker; - protected costTracker: CostTracker; - protected metrics: ProviderMetrics; - - constructor(config: ProviderConfig = {}) { - this.config = { - timeout: config.timeout ?? 30000, - maxRetries: config.maxRetries ?? 3, - retryDelay: config.retryDelay ?? 1000, - ...config - }; - - this.logger = config.logger ?? noopLogger; - - this.retryManager = new RetryManager({ - maxRetries: this.config.maxRetries, - initialDelay: this.config.retryDelay - }, this.logger); - - // Note: this.name is set by the subclass after super() returns. - // The circuit breaker name is updated lazily on first use. - this.circuitBreaker = new CircuitBreaker('pending', {}, this.logger); - this.costTracker = new CostTracker({}, undefined, this.logger); - - this.metrics = { - requestCount: 0, - successCount: 0, - errorCount: 0, - averageLatency: 0, - totalCost: 0, - rateLimitHits: 0, - lastUsed: 0 - }; - } - - /** - * Abstract method that must be implemented by each provider - */ - abstract generateResponse(request: LLMRequest): Promise; - - /** - * Abstract method for configuration validation - */ - abstract validateConfig(): boolean; - - /** - * Abstract method to get available models - */ - abstract getModels(): string[]; - - /** - * Abstract method to estimate cost - */ - abstract estimateCost(request: LLMRequest): number; - - /** - * Abstract method for health check - */ - abstract healthCheck(): Promise; - - /** - * Common HTTP request method with timeout and error handling - */ - protected async makeRequest( - url: string, - options: RequestInit = {}, - timeoutMs?: number - ): Promise { - const timeout = timeoutMs || this.config.timeout || 30000; - - const controller = new AbortController(); - const timeoutId = setTimeout(() => controller.abort(), timeout); - - try { - const response = await fetch(url, { - ...options, - signal: controller.signal, - headers: { - 'Content-Type': 'application/json', - 'User-Agent': 'ai-platform/llm-providers', - ...options.headers - } - }); - - clearTimeout(timeoutId); - return response; - } catch (error) { - clearTimeout(timeoutId); - - if (error instanceof Error && error.name === 'AbortError') { - throw new TimeoutError(this.name, `Request timeout after ${timeout}ms`); - } - - throw error; - } - } - - /** - * Execute request with circuit breaker and retry logic - */ - protected async executeWithResiliency( - operation: () => Promise - ): Promise { - return this.getCircuitBreaker().execute( - () => this.retryManager.execute(operation) - ); - } - - /** - * Update metrics after request - */ - protected updateMetrics( - responseTime: number, - success: boolean, - cost: number = 0 - ): void { - const measuredLatency = Math.max(responseTime, 1); - - this.metrics.requestCount++; - this.metrics.lastUsed = Date.now(); - - if (success) { - this.metrics.successCount++; - } else { - this.metrics.errorCount++; - } - - // Update average latency - const totalLatency = this.metrics.averageLatency * (this.metrics.requestCount - 1); - this.metrics.averageLatency = (totalLatency + measuredLatency) / this.metrics.requestCount; - - this.metrics.totalCost += cost; - - // Record to latency histogram for percentile tracking - defaultLatencyHistogram.record(this.name, measuredLatency); - } - - /** - * Get provider metrics - */ - getMetrics(): ProviderMetrics { - return { ...this.metrics }; - } - - /** - * Reset metrics - */ - resetMetrics(): void { - this.metrics = { - requestCount: 0, - successCount: 0, - errorCount: 0, - averageLatency: 0, - totalCost: 0, - rateLimitHits: 0, - lastUsed: 0 - }; - } - - /** - * Get provider health status - */ - getHealth(): { - healthy: boolean; - circuitBreakerState: string; - metrics: ProviderMetrics; - lastError?: number; - } { - const circuitState = this.getCircuitBreaker().getState(); - const successRate = this.metrics.requestCount > 0 - ? this.metrics.successCount / this.metrics.requestCount - : 1; - - return { - healthy: circuitState.state === 'CLOSED' && successRate > 0.8, - circuitBreakerState: circuitState.state, - metrics: this.getMetrics(), - lastError: circuitState.lastFailure - }; - } - - /** - * Update provider configuration - */ - updateConfig(config: Partial): void { - this.config = { ...this.config, ...config }; - } - - /** - * Get current configuration (without sensitive data) - */ - getConfig(): Omit { - const { apiKey, ...safeConfig } = this.config; - return safeConfig; - } - - /** - * Providers share the named singleton breaker so factory-level routing and - * per-provider execution observe the same failure history. - */ - protected getCircuitBreaker(): CircuitBreaker { - if (this.circuitBreaker.name !== this.name) { - this.circuitBreaker = defaultCircuitBreakerManager.getBreaker(this.name); - } - - return this.circuitBreaker; - } - - /** - * Common model capability definitions - */ - protected getModelCapabilities(): Record { - // Override in subclasses to provide model-specific capabilities - return {}; - } - - /** - * Validate request before processing - */ - protected validateRequest(request: LLMRequest): void { - if (!request.messages || request.messages.length === 0) { - throw new ConfigurationError(this.name, 'Request must contain at least one message'); - } - - if (request.maxTokens && request.maxTokens < 1) { - throw new ConfigurationError(this.name, 'maxTokens must be greater than 0'); - } - - if (request.temperature && (request.temperature < 0 || request.temperature > 2)) { - throw new ConfigurationError(this.name, 'temperature must be between 0 and 2'); - } - - // Validate model if specified - if (request.model && !this.models.includes(request.model)) { - throw new ConfigurationError( - this.name, - `Model '${request.model}' not supported. Available models: ${this.models.join(', ')}` - ); - } - - if ((request.images?.length ?? 0) > 0 && this.supportsVision !== true) { - throw new ConfigurationError(this.name, `${this.name} does not support image input`); - } - - if (request.images) { - for (const image of request.images) { - this.validateImageInput(image); - } - } - } - - protected estimateTextTokens(request: LLMRequest): number { - return ( - (request.systemPrompt ? Math.ceil(request.systemPrompt.length / 4) : 0) + - request.messages.reduce((sum, msg) => sum + Math.ceil(msg.content.length / 4), 0) - ); - } - - protected validateImageInput(image: LLMImageInput): void { - if (!image.data && !image.url) { - throw new ConfigurationError(this.name, 'Image input must include data or url'); - } - - if (image.data && !image.mimeType) { - throw new ConfigurationError(this.name, 'Base64 image input must include mimeType'); - } - } - - protected getAIGatewayHeaders(request?: LLMRequest): Record { - const baseUrl = typeof this.config.baseUrl === 'string' ? this.config.baseUrl : ''; - if (!baseUrl.includes('gateway.ai.cloudflare.com') || !request?.gatewayMetadata) { - return {}; - } - - const headers: Record = {}; - const metadata = { - ...(request.gatewayMetadata.customMetadata ?? {}), - ...(request.gatewayMetadata.requestId ? { requestId: request.gatewayMetadata.requestId } : {}), - ...(request.requestId ? { llmRequestId: request.requestId } : {}), - ...(request.tenantId ? { tenantId: request.tenantId } : {}), - }; - - if (Object.keys(metadata).length > 0) { - headers['cf-aig-metadata'] = JSON.stringify(metadata); - } - if (request.gatewayMetadata.cacheKey) { - headers['cf-aig-cache-key'] = request.gatewayMetadata.cacheKey; - } - if (typeof request.gatewayMetadata.cacheTtl === 'number') { - headers['cf-aig-cache-ttl'] = String(request.gatewayMetadata.cacheTtl); - } - - return headers; - } - - /** - * cf-aig-* headers injected when this provider was configured via `cfGateway` - * (cfGatewayActive). Derived from `request.gatewayMetadata`; each header is - * only present when its source field is defined. - */ - protected getCfGatewayHeaders(request?: LLMRequest): Record { - if (!this.cfGatewayActive || !request?.gatewayMetadata) { - return {}; - } - - const headers: Record = {}; - const { cacheTtl, cacheKey, skipCache } = request.gatewayMetadata; - - if (typeof cacheTtl === 'number') { - headers['cf-aig-cache-ttl'] = String(cacheTtl); - } - if (typeof cacheKey === 'string') { - headers['cf-aig-cache-key'] = cacheKey; - } - if (typeof skipCache === 'boolean') { - headers['cf-aig-skip-cache'] = String(skipCache); - } - - return headers; - } - - /** - * Validate and sanitize tool calls returned by the provider. - * - * Ensures each tool call has the required fields (`id`, `type`, `function.name`, - * `function.arguments`) and that their types are correct. Malformed entries are - * dropped and logged rather than propagated to the caller. - * - * Returns `undefined` when the input is empty or all entries are invalid, so the - * result can be assigned directly to `response.toolCalls`. - */ - protected validateToolCalls(toolCalls: ToolCall[] | undefined): ToolCall[] | undefined { - if (!toolCalls || toolCalls.length === 0) { - return undefined; - } - - const valid: ToolCall[] = []; - - for (let i = 0; i < toolCalls.length; i++) { - const tc = toolCalls[i]; - - // Must be an object - if (tc == null || typeof tc !== 'object') { - this.logger.warn(`[${this.name}] Dropping tool_call[${i}]: not an object`); - continue; - } - - // id — must be a non-empty string - if (typeof tc.id !== 'string' || tc.id.length === 0) { - this.logger.warn(`[${this.name}] Dropping tool_call[${i}]: missing or empty id`); - continue; - } - - // type — must be 'function' - if (tc.type !== 'function') { - this.logger.warn(`[${this.name}] Dropping tool_call[${i}]: invalid type "${String(tc.type)}"`); - continue; - } - - // function — must be an object with name and arguments - if (tc.function == null || typeof tc.function !== 'object') { - this.logger.warn(`[${this.name}] Dropping tool_call[${i}]: missing function object`); - continue; - } - - if (typeof tc.function.name !== 'string' || tc.function.name.length === 0) { - this.logger.warn(`[${this.name}] Dropping tool_call[${i}]: missing or empty function.name`); - continue; - } - - if (typeof tc.function.arguments !== 'string') { - this.logger.warn(`[${this.name}] Dropping tool_call[${i}]: function.arguments is not a string`); - continue; - } - - valid.push({ - id: tc.id, - type: 'function', - function: { - name: tc.function.name, - arguments: tc.function.arguments, - }, - }); - } - - return valid.length > 0 ? valid : undefined; - } - - /** - * Calculate token usage cost - */ - protected calculateCost( - inputTokens: number, - outputTokens: number, - model: string - ): number { - const capabilities = this.getModelCapabilities()[model]; - if (!capabilities) return 0; - - const inputCost = (inputTokens / 1000) * capabilities.inputTokenCost; - const outputCost = (outputTokens / 1000) * capabilities.outputTokenCost; - - return inputCost + outputCost; - } - - protected parseOpenAICompatibleStreamUsage( - rawUsage: unknown, - model: string - ): TokenUsage | undefined { - if (!rawUsage || typeof rawUsage !== 'object' || Array.isArray(rawUsage)) return undefined; - - const usage = rawUsage as Record; - const inputTokens = usage['prompt_tokens']; - const outputTokens = usage['completion_tokens']; - const totalTokens = usage['total_tokens']; - - if ( - typeof inputTokens !== 'number' || - typeof outputTokens !== 'number' || - typeof totalTokens !== 'number' - ) { - return undefined; - } - - const parsed: TokenUsage = { - inputTokens, - outputTokens, - totalTokens, - cost: this.calculateCost(inputTokens, outputTokens, model), - }; - - const details = usage['prompt_tokens_details']; - if (details && typeof details === 'object' && !Array.isArray(details)) { - const cachedTokens = (details as Record)['cached_tokens']; - if (typeof cachedTokens === 'number') { - parsed.cachedInputTokens = cachedTokens; - } - } - - return parsed; - } - - /** - * Common response formatting - */ - protected buildResponse( - content: string, - usage: { inputTokens: number; outputTokens: number }, - model: string, - responseTime: number, - metadata?: Record - ): LLMResponse { - const cost = this.calculateCost(usage.inputTokens, usage.outputTokens, model); - - return { - message: content, - content, - usage: { - inputTokens: usage.inputTokens, - outputTokens: usage.outputTokens, - totalTokens: usage.inputTokens + usage.outputTokens, - cost - }, - model, - provider: this.name, - responseTime, - finishReason: 'stop', - metadata - }; - } - - /** - * Log request/response for debugging - */ - protected logRequest(request: LLMRequest, response?: LLMResponse, error?: Error): void { - const logData = { - provider: this.name, - model: request.model, - messageCount: request.messages.length, - requestId: request.requestId, - tenantId: request.tenantId, - success: !error, - responseTime: response?.responseTime, - usage: response?.usage, - error: error?.message - }; - - if (error) { - this.logger.error(`[${this.name}] Request failed:`, logData); - } else { - this.logger.debug(`[${this.name}] Request completed:`, logData); - } - } -} +/** + * Base Provider + * Abstract base class for all LLM providers with common functionality + */ + +import type { + LLMProvider, + LLMImageInput, + LLMRequest, + LLMResponse, + ProviderConfig, + ModelCapabilities, + ProviderMetrics, + ToolCall, + TokenUsage +} from '../types.js'; +import type { CfGatewayConfig } from '../types.js'; +import type { Logger } from '../utils/logger.js'; +import { noopLogger } from '../utils/logger.js'; +import { RetryManager } from '../utils/retry.js'; +import { CircuitBreaker, defaultCircuitBreakerManager } from '../utils/circuit-breaker.js'; +import { CostTracker } from '../utils/cost-tracker.js'; +import { defaultLatencyHistogram } from '../utils/latency-histogram.js'; +import { ConfigurationError, TimeoutError, InvalidRequestError, CfGatewayInvalidConfigError } from '../errors.js'; + +/** Per-provider path suffix on the Cloudflare AI Gateway base URL. */ +export type CfGatewayProviderSuffix = + | 'openai/v1' + | 'anthropic' + | 'cerebras/v1' + | 'groq/openai/v1' + | 'nvidia-nim/v1'; + +export interface CfGatewayResolvedState { + resolvedBaseUrl: string; + cfGatewayActive: boolean; +} + +/** + * Resolve a provider base URL from explicit override / cfGateway / default. + * Precedence: explicit `baseUrl` wins (cfGatewayActive=false); else derive from + * `cfGateway`; else fall back to `defaultBaseUrl`. Throws synchronously when + * `cfGateway` is supplied with an empty `accountId` or `gatewayId`. + */ +export function resolveCfGateway(args: { + provider: string; + baseUrl?: string; + cfGateway?: CfGatewayConfig; + suffix: CfGatewayProviderSuffix; + defaultBaseUrl: string; +}): CfGatewayResolvedState { + const { provider, baseUrl, cfGateway, suffix, defaultBaseUrl } = args; + + if (baseUrl) { + return { resolvedBaseUrl: baseUrl, cfGatewayActive: false }; + } + + if (cfGateway) { + if (!cfGateway.accountId) { + throw new CfGatewayInvalidConfigError(provider, 'accountId'); + } + if (!cfGateway.gatewayId) { + throw new CfGatewayInvalidConfigError(provider, 'gatewayId'); + } + return { + resolvedBaseUrl: `https://gateway.ai.cloudflare.com/v1/${cfGateway.accountId}/${cfGateway.gatewayId}/${suffix}`, + cfGatewayActive: true, + }; + } + + return { resolvedBaseUrl: defaultBaseUrl, cfGatewayActive: false }; +} + +export abstract class BaseProvider implements LLMProvider { + abstract name: string; + abstract models: string[]; + abstract supportsStreaming: boolean; + abstract supportsTools: boolean; + abstract supportsBatching: boolean; + abstract supportsVision?: boolean; + + protected config: ProviderConfig; + /** + * True when this provider derived its base URL from `cfGateway` (no explicit + * `baseUrl` override). Gates injection of cf-aig-* headers in makeRequest. + * Set by each provider constructor via {@link resolveCfGateway}. + */ + protected cfGatewayActive = false; + protected logger: Logger; + protected retryManager: RetryManager; + protected circuitBreaker: CircuitBreaker; + protected costTracker: CostTracker; + protected metrics: ProviderMetrics; + + constructor(config: ProviderConfig = {}) { + this.config = { + timeout: config.timeout ?? 30000, + maxRetries: config.maxRetries ?? 3, + retryDelay: config.retryDelay ?? 1000, + ...config + }; + + this.logger = config.logger ?? noopLogger; + + this.retryManager = new RetryManager({ + maxRetries: this.config.maxRetries, + initialDelay: this.config.retryDelay + }, this.logger); + + // Note: this.name is set by the subclass after super() returns. + // The circuit breaker name is updated lazily on first use. + this.circuitBreaker = new CircuitBreaker('pending', {}, this.logger); + this.costTracker = new CostTracker({}, undefined, this.logger); + + this.metrics = { + requestCount: 0, + successCount: 0, + errorCount: 0, + averageLatency: 0, + totalCost: 0, + rateLimitHits: 0, + lastUsed: 0 + }; + } + + /** + * Abstract method that must be implemented by each provider + */ + abstract generateResponse(request: LLMRequest): Promise; + + /** + * Abstract method for configuration validation + */ + abstract validateConfig(): boolean; + + /** + * Abstract method to get available models + */ + abstract getModels(): string[]; + + /** + * Abstract method to estimate cost + */ + abstract estimateCost(request: LLMRequest): number; + + /** + * Abstract method for health check + */ + abstract healthCheck(): Promise; + + /** + * Common HTTP request method with timeout and error handling + */ + protected async makeRequest( + url: string, + options: RequestInit = {}, + timeoutMs?: number + ): Promise { + const timeout = timeoutMs || this.config.timeout || 30000; + + const controller = new AbortController(); + const timeoutId = setTimeout(() => controller.abort(), timeout); + + try { + const response = await fetch(url, { + ...options, + signal: controller.signal, + headers: { + 'Content-Type': 'application/json', + 'User-Agent': 'ai-platform/llm-providers', + ...options.headers + } + }); + + clearTimeout(timeoutId); + return response; + } catch (error) { + clearTimeout(timeoutId); + + if (error instanceof Error && error.name === 'AbortError') { + throw new TimeoutError(this.name, `Request timeout after ${timeout}ms`); + } + + throw error; + } + } + + /** + * Execute request with circuit breaker and retry logic + */ + protected async executeWithResiliency( + operation: () => Promise + ): Promise { + return this.getCircuitBreaker().execute( + () => this.retryManager.execute(operation) + ); + } + + /** + * Update metrics after request + */ + protected updateMetrics( + responseTime: number, + success: boolean, + cost: number = 0 + ): void { + const measuredLatency = Math.max(responseTime, 1); + + this.metrics.requestCount++; + this.metrics.lastUsed = Date.now(); + + if (success) { + this.metrics.successCount++; + } else { + this.metrics.errorCount++; + } + + // Update average latency + const totalLatency = this.metrics.averageLatency * (this.metrics.requestCount - 1); + this.metrics.averageLatency = (totalLatency + measuredLatency) / this.metrics.requestCount; + + this.metrics.totalCost += cost; + + // Record to latency histogram for percentile tracking + defaultLatencyHistogram.record(this.name, measuredLatency); + } + + /** + * Get provider metrics + */ + getMetrics(): ProviderMetrics { + return { ...this.metrics }; + } + + /** + * Reset metrics + */ + resetMetrics(): void { + this.metrics = { + requestCount: 0, + successCount: 0, + errorCount: 0, + averageLatency: 0, + totalCost: 0, + rateLimitHits: 0, + lastUsed: 0 + }; + } + + /** + * Get provider health status + */ + getHealth(): { + healthy: boolean; + circuitBreakerState: string; + metrics: ProviderMetrics; + lastError?: number; + } { + const circuitState = this.getCircuitBreaker().getState(); + const successRate = this.metrics.requestCount > 0 + ? this.metrics.successCount / this.metrics.requestCount + : 1; + + return { + healthy: circuitState.state === 'CLOSED' && successRate > 0.8, + circuitBreakerState: circuitState.state, + metrics: this.getMetrics(), + lastError: circuitState.lastFailure + }; + } + + /** + * Update provider configuration + */ + updateConfig(config: Partial): void { + this.config = { ...this.config, ...config }; + } + + /** + * Get current configuration (without sensitive data) + */ + getConfig(): Omit { + const { apiKey, ...safeConfig } = this.config; + return safeConfig; + } + + /** + * Providers share the named singleton breaker so factory-level routing and + * per-provider execution observe the same failure history. + */ + protected getCircuitBreaker(): CircuitBreaker { + if (this.circuitBreaker.name !== this.name) { + this.circuitBreaker = defaultCircuitBreakerManager.getBreaker(this.name); + } + + return this.circuitBreaker; + } + + /** + * Common model capability definitions + */ + protected getModelCapabilities(): Record { + // Override in subclasses to provide model-specific capabilities + return {}; + } + + /** + * Validate request before processing + */ + protected validateRequest(request: LLMRequest): void { + if (!request.messages || request.messages.length === 0) { + throw new ConfigurationError(this.name, 'Request must contain at least one message'); + } + + if (request.maxTokens && request.maxTokens < 1) { + throw new ConfigurationError(this.name, 'maxTokens must be greater than 0'); + } + + if (request.temperature && (request.temperature < 0 || request.temperature > 2)) { + throw new ConfigurationError(this.name, 'temperature must be between 0 and 2'); + } + + // Validate model if specified + if (request.model && !this.models.includes(request.model)) { + throw new ConfigurationError( + this.name, + `Model '${request.model}' not supported. Available models: ${this.models.join(', ')}` + ); + } + + if ((request.images?.length ?? 0) > 0 && this.supportsVision !== true) { + throw new ConfigurationError(this.name, `${this.name} does not support image input`); + } + + if (request.images) { + for (const image of request.images) { + this.validateImageInput(image); + } + } + } + + protected estimateTextTokens(request: LLMRequest): number { + return ( + (request.systemPrompt ? Math.ceil(request.systemPrompt.length / 4) : 0) + + request.messages.reduce((sum, msg) => sum + Math.ceil(msg.content.length / 4), 0) + ); + } + + protected validateImageInput(image: LLMImageInput): void { + if (!image.data && !image.url) { + throw new ConfigurationError(this.name, 'Image input must include data or url'); + } + + if (image.data && !image.mimeType) { + throw new ConfigurationError(this.name, 'Base64 image input must include mimeType'); + } + } + + protected getAIGatewayHeaders(request?: LLMRequest): Record { + const baseUrl = typeof this.config.baseUrl === 'string' ? this.config.baseUrl : ''; + const isGateway = this.cfGatewayActive || baseUrl.includes('gateway.ai.cloudflare.com'); + if (!isGateway || !request?.gatewayMetadata) { + return {}; + } + + const headers: Record = {}; + const metadata = { + ...(request.gatewayMetadata.customMetadata ?? {}), + ...(request.gatewayMetadata.requestId ? { requestId: request.gatewayMetadata.requestId } : {}), + ...(request.requestId ? { llmRequestId: request.requestId } : {}), + ...(request.tenantId ? { tenantId: request.tenantId } : {}), + }; + + if (Object.keys(metadata).length > 0) { + headers['cf-aig-metadata'] = JSON.stringify(metadata); + } + if (request.gatewayMetadata.cacheKey) { + headers['cf-aig-cache-key'] = request.gatewayMetadata.cacheKey; + } + if (typeof request.gatewayMetadata.cacheTtl === 'number') { + headers['cf-aig-cache-ttl'] = String(request.gatewayMetadata.cacheTtl); + } + if (typeof request.gatewayMetadata.skipCache === 'boolean') { + headers['cf-aig-skip-cache'] = String(request.gatewayMetadata.skipCache); + } + + return headers; + } + + /** + * cf-aig-* headers injected when this provider was configured via `cfGateway` + * (cfGatewayActive). Derived from `request.gatewayMetadata`; each header is + * only present when its source field is defined. + */ + protected getCfGatewayHeaders(request?: LLMRequest): Record { + if (!this.cfGatewayActive || !request?.gatewayMetadata) { + return {}; + } + + const headers: Record = {}; + const { cacheTtl, cacheKey, skipCache } = request.gatewayMetadata; + + if (typeof cacheTtl === 'number') { + headers['cf-aig-cache-ttl'] = String(cacheTtl); + } + if (typeof cacheKey === 'string') { + headers['cf-aig-cache-key'] = cacheKey; + } + if (typeof skipCache === 'boolean') { + headers['cf-aig-skip-cache'] = String(skipCache); + } + + return headers; + } + + /** + * Validate and sanitize tool calls returned by the provider. + * + * Ensures each tool call has the required fields (`id`, `type`, `function.name`, + * `function.arguments`) and that their types are correct. Malformed entries are + * dropped and logged rather than propagated to the caller. + * + * Returns `undefined` when the input is empty or all entries are invalid, so the + * result can be assigned directly to `response.toolCalls`. + */ + protected validateToolCalls(toolCalls: ToolCall[] | undefined): ToolCall[] | undefined { + if (!toolCalls || toolCalls.length === 0) { + return undefined; + } + + const valid: ToolCall[] = []; + + for (let i = 0; i < toolCalls.length; i++) { + const tc = toolCalls[i]; + + // Must be an object + if (tc == null || typeof tc !== 'object') { + this.logger.warn(`[${this.name}] Dropping tool_call[${i}]: not an object`); + continue; + } + + // id — must be a non-empty string + if (typeof tc.id !== 'string' || tc.id.length === 0) { + this.logger.warn(`[${this.name}] Dropping tool_call[${i}]: missing or empty id`); + continue; + } + + // type — must be 'function' + if (tc.type !== 'function') { + this.logger.warn(`[${this.name}] Dropping tool_call[${i}]: invalid type "${String(tc.type)}"`); + continue; + } + + // function — must be an object with name and arguments + if (tc.function == null || typeof tc.function !== 'object') { + this.logger.warn(`[${this.name}] Dropping tool_call[${i}]: missing function object`); + continue; + } + + if (typeof tc.function.name !== 'string' || tc.function.name.length === 0) { + this.logger.warn(`[${this.name}] Dropping tool_call[${i}]: missing or empty function.name`); + continue; + } + + if (typeof tc.function.arguments !== 'string') { + this.logger.warn(`[${this.name}] Dropping tool_call[${i}]: function.arguments is not a string`); + continue; + } + + valid.push({ + id: tc.id, + type: 'function', + function: { + name: tc.function.name, + arguments: tc.function.arguments, + }, + }); + } + + return valid.length > 0 ? valid : undefined; + } + + /** + * Calculate token usage cost + */ + protected calculateCost( + inputTokens: number, + outputTokens: number, + model: string + ): number { + const capabilities = this.getModelCapabilities()[model]; + if (!capabilities) return 0; + + const inputCost = (inputTokens / 1000) * capabilities.inputTokenCost; + const outputCost = (outputTokens / 1000) * capabilities.outputTokenCost; + + return inputCost + outputCost; + } + + protected parseOpenAICompatibleStreamUsage( + rawUsage: unknown, + model: string + ): TokenUsage | undefined { + if (!rawUsage || typeof rawUsage !== 'object' || Array.isArray(rawUsage)) return undefined; + + const usage = rawUsage as Record; + const inputTokens = usage['prompt_tokens']; + const outputTokens = usage['completion_tokens']; + const totalTokens = usage['total_tokens']; + + if ( + typeof inputTokens !== 'number' || + typeof outputTokens !== 'number' || + typeof totalTokens !== 'number' + ) { + return undefined; + } + + const parsed: TokenUsage = { + inputTokens, + outputTokens, + totalTokens, + cost: this.calculateCost(inputTokens, outputTokens, model), + }; + + const details = usage['prompt_tokens_details']; + if (details && typeof details === 'object' && !Array.isArray(details)) { + const cachedTokens = (details as Record)['cached_tokens']; + if (typeof cachedTokens === 'number') { + parsed.cachedInputTokens = cachedTokens; + } + } + + return parsed; + } + + /** + * Common response formatting + */ + protected buildResponse( + content: string, + usage: { inputTokens: number; outputTokens: number }, + model: string, + responseTime: number, + metadata?: Record + ): LLMResponse { + const cost = this.calculateCost(usage.inputTokens, usage.outputTokens, model); + + return { + message: content, + content, + usage: { + inputTokens: usage.inputTokens, + outputTokens: usage.outputTokens, + totalTokens: usage.inputTokens + usage.outputTokens, + cost + }, + model, + provider: this.name, + responseTime, + finishReason: 'stop', + metadata + }; + } + + /** + * Log request/response for debugging + */ + protected logRequest(request: LLMRequest, response?: LLMResponse, error?: Error): void { + const logData = { + provider: this.name, + model: request.model, + messageCount: request.messages.length, + requestId: request.requestId, + tenantId: request.tenantId, + success: !error, + responseTime: response?.responseTime, + usage: response?.usage, + error: error?.message + }; + + if (error) { + this.logger.error(`[${this.name}] Request failed:`, logData); + } else { + this.logger.debug(`[${this.name}] Request completed:`, logData); + } + } +} diff --git a/src/version.ts b/src/version.ts index 0edf0e3..ee3662d 100644 --- a/src/version.ts +++ b/src/version.ts @@ -1,2 +1,2 @@ // Generated by scripts/sync-version.mjs; do not edit manually. -export const VERSION = '1.17.2'; +export const VERSION = '1.19.0';