From f7e8f1809cf4d278b42b139d00d78a7025b8cd0c Mon Sep 17 00:00:00 2001 From: AlinsRan Date: Thu, 9 Jul 2026 08:03:53 +0800 Subject: [PATCH 1/8] feat(ai-proxy-multi): add semantic load-balancing algorithm MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a `semantic` balancer algorithm to `ai-proxy-multi` that picks an LLM instance by the semantic similarity between the request prompt and each instance's configured `examples`, instead of by weight or hash. Each non-catchall instance declares `examples` describing the prompts it should serve. On the request path the plugin embeds those examples and the incoming prompt via a configurable OpenAI-compatible embedding endpoint, computes cosine similarity, aggregates per-instance scores, and routes to the highest-scoring instance that clears its threshold. When no instance clears its threshold — or embedding fails for any reason — the request fails open to the `catchall` instance (else the first instance), so a request always has a target and never 500s on the routing path. Reference vectors are embedded once per config version and cached; only the prompt is embedded per request. No vector database is required. New config: `balancer.algorithm=semantic`, `balancer.threshold`, `balancer.aggregation`, `balancer.expose_scores`, per-instance `examples` / `threshold` / `catchall`, and an `embeddings` block. --- apisix/plugins/ai-providers/base.lua | 3 +- apisix/plugins/ai-proxy-multi.lua | 258 +++++++- apisix/plugins/ai-proxy/embedding.lua | 159 +++++ apisix/plugins/ai-proxy/schema.lua | 99 ++- apisix/plugins/ai-proxy/semantic.lua | 83 +++ apisix/plugins/ai-transport/http.lua | 12 +- docs/en/latest/plugins/ai-proxy-multi.md | 150 ++++- docs/zh/latest/plugins/ai-proxy-multi.md | 150 ++++- t/plugin/ai-proxy-semantic-routing.t | 731 +++++++++++++++++++++++ t/plugin/ai-proxy-semantic-schema.t | 408 +++++++++++++ t/plugin/ai-proxy-semantic.t | 67 +++ 11 files changed, 2110 insertions(+), 10 deletions(-) create mode 100644 apisix/plugins/ai-proxy/embedding.lua create mode 100644 apisix/plugins/ai-proxy/semantic.lua create mode 100644 t/plugin/ai-proxy-semantic-routing.t create mode 100644 t/plugin/ai-proxy-semantic-schema.t create mode 100644 t/plugin/ai-proxy-semantic.t diff --git a/apisix/plugins/ai-providers/base.lua b/apisix/plugins/ai-providers/base.lua index a992defbaabc..6b8c8e35c9c1 100644 --- a/apisix/plugins/ai-providers/base.lua +++ b/apisix/plugins/ai-providers/base.lua @@ -188,7 +188,8 @@ function _M.build_request(self, ctx, conf, request_body, opts) .. "includes a path", 400 end - local headers = transport_http.construct_forward_headers(auth.header or {}, ctx) + local headers = transport_http.construct_forward_headers(auth.header or {}, ctx, + opts.skip_client_headers) if opts.host_header then headers["Host"] = opts.host_header end diff --git a/apisix/plugins/ai-proxy-multi.lua b/apisix/plugins/ai-proxy-multi.lua index 9b8f2eab5485..ee233254d3fd 100644 --- a/apisix/plugins/ai-proxy-multi.lua +++ b/apisix/plugins/ai-proxy-multi.lua @@ -33,12 +33,17 @@ local ngx_now = ngx.now local require = require local pcall = pcall +local error = error +local tostring = tostring local ipairs = ipairs local type = type local string = string +local sub = string.sub local url = require("socket.url") local priority_balancer = require("apisix.balancer.priority") +local semantic = require("apisix.plugins.ai-proxy.semantic") +local embedding = require("apisix.plugins.ai-proxy.embedding") local endpoint_regex = "^(https?)://([^:/]+):?(%d*)/?.*$" local pickers = {} @@ -48,6 +53,16 @@ local lrucache_server_picker = core.lrucache.new({ local lrucache_health_status = core.lrucache.new({ ttl = 300, count = 256 }) +-- Keyed by route + conf version, so config changes invalidate immediately; +-- the long ttl just avoids re-embedding references on unchanged config. +local lrucache_semantic_vectors = core.lrucache.new({ + ttl = 3600, count = 256 +}) +-- The prompt is sent verbatim to a third-party embedding endpoint. Bound it: a +-- request body may be up to max_req_body_size (64MB by default), and an oversized +-- input would blow the embedding model's token limit, 400, and silently push every +-- large prompt to the catchall. Routing intent lives in the opening sentences. +local MAX_EMBED_PROMPT_BYTES = 8192 local plugin_name = "ai-proxy-multi" local _M = { @@ -156,7 +171,70 @@ function _M.check_schema(conf) end end - return ok + if algo == "semantic" then + if not conf.embeddings then + return false, "must configure `embeddings` when balancer algorithm is semantic" + end + -- Same scheme+host check the instance endpoints get: without it an + -- endpoint like "host/path" parses to a nil host and every request would + -- silently fail open, i.e. the route would never work with no signal. + local eendpoint = conf.embeddings.endpoint + if eendpoint then + local scheme, host = eendpoint:match(endpoint_regex) + if not scheme or not host then + return false, "invalid `embeddings.endpoint`" + end + end + if conf.embeddings.provider == "azure-openai" then + -- Azure carries the deployment in the URL and declares no default host + -- or path, so the endpoint must be the full embeddings URL. Without a + -- path every request would silently fail open to the catchall. + if not eendpoint then + return false, "must configure `embeddings.endpoint` when embeddings " .. + "provider is azure-openai" + end + local parsed = url.parse(eendpoint) + local epath = parsed and parsed.path + if not epath or epath == "" or epath == "/" then + return false, "`embeddings.endpoint` for azure-openai must include the " .. + "full deployment path, e.g. https://{resource}.openai.azure.com" .. + "/openai/deployments/{deployment}/embeddings?api-version=..." + end + end + local catchall_count = 0 + for _, instance in ipairs(conf.instances) do + if instance.catchall then + catchall_count = catchall_count + 1 + -- The catchall is the fallback target, never a ranking candidate. + -- Allowing examples on it would let it outrank a real match. + if instance.examples then + return false, "instance '" .. (instance.name or "?") .. + "': `catchall` instance must not configure `examples`; it is " .. + "the fallback target and does not take part in ranking" + end + else + local has_example = false + if instance.examples then + for _, ex in ipairs(instance.examples) do + if type(ex) == "string" and ex ~= "" then + has_example = true + break + end + end + end + if not has_example then + return false, "instance '" .. (instance.name or "?") .. + "': must configure non-empty `examples` for the semantic " .. + "algorithm unless `catchall` is set" + end + end + end + if catchall_count > 1 then + return false, "at most one instance may be marked `catchall`" + end + end + + return true end @@ -598,9 +676,185 @@ local function pick_target(ctx, conf, ups_tab) end +local function extract_last_user_message() + local body = core.request.get_json_request_body_table() + if not body or type(body.messages) ~= "table" then + return nil + end + for i = #body.messages, 1, -1 do + local m = body.messages[i] + if type(m) == "table" and m.role == "user" then + local content = m.content + if type(content) == "string" then + return content + elseif type(content) == "table" then + -- multimodal content: concatenate the text parts so routing + -- still works for {type=text|image_url,...} arrays. + local parts = {} + for _, p in ipairs(content) do + if type(p) == "table" and p.type == "text" + and type(p.text) == "string" then + parts[#parts + 1] = p.text + end + end + if #parts > 0 then + return table_concat(parts, " ") + end + end + end + end + return nil +end + + +-- Embed every instance's examples in one batch and group the normalized +-- reference vectors by instance name. Raises on embedding failure so the +-- lrucache below does not cache a bad result. +local function build_instance_vectors(conf) + local texts = {} + local owners = {} + for _, inst in ipairs(conf.instances) do + if inst.examples then + for _, ex in ipairs(inst.examples) do + texts[#texts + 1] = ex + owners[#texts] = inst.name + end + end + end + + local vecs, err = embedding.fetch(conf.embeddings, texts) + if not vecs then + error("failed to fetch reference embeddings: " .. tostring(err)) + end + + local by_instance = {} + for i, v in ipairs(vecs) do + local name = owners[i] + if name then + by_instance[name] = by_instance[name] or {} + core.table.insert(by_instance[name], semantic.normalize(v)) + end + end + return by_instance +end + + +-- Guaranteed fallback: catchall instance if configured, else the first +-- instance. Never fails, so a request always has a target. +local function semantic_fallback(conf) + for _, inst in ipairs(conf.instances) do + if inst.catchall then + return inst.name, inst + end + end + local inst = conf.instances[1] + return inst.name, inst +end + + +local function pick_semantic_instance(ctx, conf) + local version = plugin.conf_version(conf) + local ok, by_instance = pcall(lrucache_semantic_vectors, + ctx.matched_route.key .. "#semantic", version, + build_instance_vectors, conf) + if not ok or not by_instance then + core.log.warn("semantic routing: ", by_instance, ", falling back") + return semantic_fallback(conf) + end + + local prompt = extract_last_user_message() + if not prompt then + core.log.warn("semantic routing: no user message found, falling back") + return semantic_fallback(conf) + end + + -- pcall, like the reference path above: the embedding response is + -- provider-controlled, so a raise here must fall back rather than 500. + if #prompt > MAX_EMBED_PROMPT_BYTES then + prompt = sub(prompt, 1, MAX_EMBED_PROMPT_BYTES) + end + + -- pcall, like the reference path above: the embedding response is + -- provider-controlled, so a raise here must fall back rather than 500. + local fetched, qvecs, err = pcall(embedding.fetch, conf.embeddings, { prompt }) + if not fetched then + core.log.warn("semantic routing: query embedding error: ", qvecs, ", falling back") + return semantic_fallback(conf) + end + if not qvecs or not qvecs[1] then + core.log.warn("semantic routing: query embedding failed: ", err, ", falling back") + return semantic_fallback(conf) + end + local qvec = semantic.normalize(qvecs[1]) + local qdim = #qvec + + local ranked = {} + for _, inst in ipairs(conf.instances) do + local refs = by_instance[inst.name] + if refs then + local scores = {} + for _, rv in ipairs(refs) do + -- guard against dimension drift (e.g. embedding model changed): + -- mismatched vectors would make dot() error, so fail open instead. + if #rv ~= qdim then + core.log.warn("semantic routing: embedding dimension mismatch ", + "(query ", qdim, " vs reference ", #rv, "), falling back") + return semantic_fallback(conf) + end + scores[#scores + 1] = semantic.dot(qvec, rv) + end + core.table.insert(ranked, { + name = inst.name, + score = semantic.max(scores), + }) + end + end + core.table.sort(ranked, function(a, b) return a.score > b.score end) + + local expose_scores = conf.balancer.expose_scores + if expose_scores then + local parts = {} + for _, c in ipairs(ranked) do + parts[#parts + 1] = c.name .. ":" .. string.format("%.4f", c.score) + end + core.response.set_header("X-AI-Semantic-Scores", table_concat(parts, ",")) + end + + -- Highest score first; pick the first instance that clears its own threshold + -- (per-instance override, else the global balancer.threshold). + for _, cand in ipairs(ranked) do + local inst = get_instance_conf(conf.instances, cand.name) + local thr = inst.threshold or conf.balancer.threshold or 0 + if cand.score >= thr then + if expose_scores then + core.response.set_header("X-AI-Semantic-Route", cand.name) + end + core.log.info("semantic routing picked instance: ", cand.name, + ", score: ", cand.score) + return cand.name, inst + end + end + + if expose_scores then + core.response.set_header("X-AI-Semantic-Route", "fallback") + end + -- Only on the fallback path: surface why nothing matched, without requiring + -- expose_scores. Cheap, because this runs once per unmatched request. + local unmatched = {} + for _, c in ipairs(ranked) do + unmatched[#unmatched + 1] = c.name .. ":" .. string.format("%.4f", c.score) + end + core.log.warn("semantic routing: no instance cleared threshold (scores: ", + table_concat(unmatched, ","), "), falling back") + return semantic_fallback(conf) +end + + local function pick_ai_instance(ctx, conf, ups_tab) local instance_name, instance_conf, err - if #conf.instances == 1 then + if conf.balancer and conf.balancer.algorithm == "semantic" then + instance_name, instance_conf = pick_semantic_instance(ctx, conf) + elseif #conf.instances == 1 then instance_name = conf.instances[1].name instance_conf = conf.instances[1] else diff --git a/apisix/plugins/ai-proxy/embedding.lua b/apisix/plugins/ai-proxy/embedding.lua new file mode 100644 index 000000000000..f069312a8c20 --- /dev/null +++ b/apisix/plugins/ai-proxy/embedding.lua @@ -0,0 +1,159 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +-- Batch embedding client for the semantic balancer. Routes the request through +-- the shared ai-provider layer (apisix.plugins.ai-providers.*) so endpoint +-- resolution, authentication and HTTP transport match the chat path, instead of +-- re-implementing them here. One call embeds a list of texts (OpenAI-compatible +-- /embeddings: input is an array), returning a vector per text. Keeps semantic +-- routing free of a vector database. +local core = require("apisix.core") +local ipairs = ipairs +local type = type +local tostring = tostring +local require = require +local pcall = pcall +local INF = math.huge + +local _M = {} + +local EMBEDDINGS_PROTOCOL = "openai-embeddings" + + +-- The provider's embeddings capability supplies a default path when it has one +-- (openai: /v1/embeddings). Providers whose endpoint always carries its own path +-- (azure-openai) declare none, and build_request then requires `endpoint` to give +-- it. Which providers are allowed is gated by the `embeddings.provider` enum, so +-- a missing capability is not an error here. +local function embeddings_target_path(ai_provider) + local cap = ai_provider.capabilities and ai_provider.capabilities[EMBEDDINGS_PROTOCOL] + if not cap then + return nil + end + local path = cap.path + if type(path) == "function" then + path = path() + end + return path +end + + +-- Fetch embeddings for `texts` in a single batch request. +-- Returns an array of vectors index-aligned with `texts`, or nil + err. +function _M.fetch(conf, texts) + if not texts or #texts == 0 then + return {} + end + + if not conf.model or conf.model == "" then + return nil, "embedding model is not configured" + end + + local ok, ai_provider = pcall(require, "apisix.plugins.ai-providers." .. conf.provider) + if not ok then + return nil, "failed to load ai-provider: " .. tostring(conf.provider) + end + + local target_path = embeddings_target_path(ai_provider) + + -- The embedding call is a self-contained sidecar request; use a throwaway + -- ctx so it never reads or mutates the in-flight request's ai_* fields. + -- ai_request_body_changed = true tells build_request to send our + -- { model, input } body instead of reusing the client's request body. + local ctx = { ai_request_body_changed = true } + local extra_opts = { + endpoint = conf.endpoint, + auth = conf.auth, + target_path = target_path, + -- This call carries its own credentials; never forward the client's + -- headers (Authorization, Cookie, ...) to the embedding provider. + skip_client_headers = true, + } + local req_conf = { + ssl_verify = conf.ssl_verify ~= false, + timeout = conf.timeout or 10000, + keepalive = true, + } + + local status, raw_body, err = ai_provider:request(ctx, req_conf, + { model = conf.model, input = texts }, extra_opts) + -- The upstream body never reaches a log line or a returned error. It is + -- untrusted third-party content that may echo the prompt back, be arbitrarily + -- large, or forge log lines with newlines. Report the status and a bounded, + -- sanitized summary instead. + if status ~= 200 then + if err then + return nil, "embedding request failed: " .. err + end + return nil, "embedding endpoint returned status " .. tostring(status) + end + + -- A 200 body that decodes to a scalar, boolean or null must not be indexed + -- (`data.data` on a number raises). `derr` is cjson's own parse error, which + -- is bounded and carries no body content. + local data, derr = core.json.decode(raw_body) + if type(data) ~= "table" or type(data.data) ~= "table" then + return nil, "invalid embedding response: " .. (derr or "unexpected shape") + end + + local vectors = {} + for i, item in ipairs(data.data) do + -- `data` may be an array of non-objects; indexing those would raise and + -- turn a sidecar failure into a 500, so check before touching a field. + if type(item) ~= "table" then + return nil, "invalid embedding entry at index " .. (i - 1) + end + -- OpenAI returns an `index`; fall back to positional order. It must be a + -- usable table key: cjson decodes JSON `NaN` to a number, and `t[NaN]` + -- raises. A float or out-of-range index is equally unusable. + local idx = i + local raw_idx = item.index + if type(raw_idx) == "number" then + if raw_idx ~= raw_idx or raw_idx < 0 or raw_idx >= #texts + or raw_idx % 1 ~= 0 + then + return nil, "invalid embedding index at entry " .. (i - 1) + end + idx = raw_idx + 1 + end + local emb = item.embedding + if type(emb) ~= "table" or #emb == 0 then + return nil, "invalid embedding vector at index " .. (idx - 1) + end + for _, n in ipairs(emb) do + -- reject non-finite components: cjson decodes `1e999` to inf, which + -- would normalize to NaN and silently poison every similarity score. + if type(n) ~= "number" or n ~= n or n == INF or n == -INF then + return nil, "non-numeric embedding component at index " .. (idx - 1) + end + end + vectors[idx] = emb + end + + -- Every input position must have produced a vector; a hole means a + -- dropped/duplicated index we must not silently route on. Checking each + -- slot is reliable where `#vectors` is not on a sparse table. + for i = 1, #texts do + if vectors[i] == nil then + return nil, "missing embedding for input index " .. (i - 1) + end + end + return vectors +end + + +return _M diff --git a/apisix/plugins/ai-proxy/schema.lua b/apisix/plugins/ai-proxy/schema.lua index 5ffd6ef856fd..cdb9a544d97c 100644 --- a/apisix/plugins/ai-proxy/schema.lua +++ b/apisix/plugins/ai-proxy/schema.lua @@ -202,12 +202,77 @@ local ai_instance_schema = { active = schema_def.health_checker_active, }, required = {"active"} - } + }, + examples = { + type = "array", + minItems = 1, + maxItems = 64, + items = { type = "string", minLength = 1 }, + description = "Example utterances representing this instance's " + .. "intent; each is embedded into its own reference vector " + .. "for the semantic algorithm. Required for every instance " + .. "except the catchall, which must not set them.", + }, + threshold = { + type = "number", + minimum = -1, + maximum = 1, + description = "Per-instance minimum cosine similarity for the " + .. "semantic algorithm; overrides balancer.threshold.", + }, + catchall = { + type = "boolean", + description = "Marks this instance as the semantic fallback, " + .. "used when no instance clears its threshold. It takes no " + .. "part in ranking, so it must not configure examples.", + }, }, required = {"name", "provider", "auth", "weight"}, }, } +local embeddings_schema = { + type = "object", + properties = { + provider = { + type = "string", + enum = { "openai", "azure-openai" }, + description = "Embedding provider used by the semantic algorithm.", + }, + model = { + type = "string", + minLength = 1, + description = "Embedding model name, e.g. text-embedding-3-small.", + }, + endpoint = { + type = "string", + description = "Embedding API endpoint. Optional for openai (defaults " + .. "to the public API). Required for azure-openai, where it must " + .. "be the full URL, e.g. https://{resource}.openai.azure.com" + .. "/openai/deployments/{deployment}/embeddings?api-version=...", + }, + auth = { + type = "object", + properties = { + header = { type = "object", additionalProperties = { type = "string" } }, + query = { type = "object", additionalProperties = { type = "string" } }, + }, + -- header/query only. The sidecar call runs on a throwaway ctx, so the + -- ctx-dependent schemes (gcp, aws) cannot work here; reject them at + -- config time instead of failing open on every request. + additionalProperties = false, + }, + timeout = { + type = "integer", + minimum = 1, + default = 10000, + description = "Embedding request timeout in milliseconds.", + }, + ssl_verify = { type = "boolean", default = true }, + }, + required = { "provider", "model", "auth" }, +} + local logging_schema = { type = "object", properties = { @@ -303,7 +368,34 @@ _M.ai_proxy_multi_schema = { properties = { algorithm = { type = "string", - enum = { "chash", "roundrobin" }, + enum = { "chash", "roundrobin", "semantic" }, + description = "Load-balancing algorithm. Note: 'semantic' " + .. "picks an instance by prompt similarity and does not " + .. "participate in health checks or fallback_strategy / " + .. "retry — an upstream failure on the chosen instance is " + .. "returned to the client. It only falls back (to the " + .. "catchall, else the first instance) when no instance " + .. "clears its threshold or embedding fails.", + }, + threshold = { + type = "number", + minimum = -1, + maximum = 1, + default = 0, + description = "Global minimum cosine similarity for the " + .. "semantic algorithm. When no instance clears its " + .. "threshold the request falls back to the catchall " + .. "instance, or the first instance if none is set. " + .. "The default of 0 admits essentially any prompt, so " + .. "the catchall only ever runs if a threshold above 0 " + .. "is set.", + }, + expose_scores = { + type = "boolean", + default = false, + description = "When true, the semantic algorithm exposes " + .. "per-instance scores and the routing decision via " + .. "X-AI-Semantic-* response headers, for debugging.", }, hash_on = { type = "string", @@ -324,6 +416,7 @@ _M.ai_proxy_multi_schema = { default = { algorithm = "roundrobin" } }, instances = ai_instance_schema, + embeddings = embeddings_schema, logging = logging_schema, fallback_strategy = { anyOf = { @@ -419,6 +512,8 @@ _M.ai_proxy_multi_schema = { "instances.auth.gcp.service_account_json", "instances.auth.aws.secret_access_key", "instances.auth.aws.session_token", + "embeddings.auth.header", + "embeddings.auth.query", }, } diff --git a/apisix/plugins/ai-proxy/semantic.lua b/apisix/plugins/ai-proxy/semantic.lua new file mode 100644 index 000000000000..c58c8629dc5f --- /dev/null +++ b/apisix/plugins/ai-proxy/semantic.lua @@ -0,0 +1,83 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +-- Pure-lua vector math for semantic routing: no apisix.core dependency so it can +-- be unit-tested standalone. Vectors are plain arrays of numbers. +local sqrt = math.sqrt + +local _M = {} + + +-- Return the unit vector of `vec`. A zero vector normalizes to zeros (no NaN). +function _M.normalize(vec) + local sum = 0 + for i = 1, #vec do + sum = sum + vec[i] * vec[i] + end + local norm = sqrt(sum) + local out = {} + if norm == 0 then + for i = 1, #vec do + out[i] = 0 + end + return out + end + for i = 1, #vec do + out[i] = vec[i] / norm + end + return out +end + + +-- Dot product. On pre-normalized vectors this equals cosine similarity. +function _M.dot(a, b) + local sum = 0 + for i = 1, #a do + sum = sum + a[i] * b[i] + end + return sum +end + + +-- Cosine similarity in [-1, 1]. Normalizes both inputs; for hot paths prefer +-- pre-normalizing once and calling dot() directly. +function _M.cosine(a, b) + return _M.dot(_M.normalize(a), _M.normalize(b)) +end + + +-- Score an instance from its per-example similarities. +-- `examples` are alternative phrasings of one intent, so the question is whether +-- ANY of them matches: take the best. Averaging would dilute a dead-on example +-- with the instance's broader ones, and would make adding an example lower the +-- instance's score. +function _M.max(scores) + local n = #scores + if n == 0 then + return nil + end + local m = scores[1] + for i = 2, n do + if scores[i] > m then + m = scores[i] + end + end + return m +end + + +return _M diff --git a/apisix/plugins/ai-transport/http.lua b/apisix/plugins/ai-transport/http.lua index 5ea9d2194545..ee142061a32b 100644 --- a/apisix/plugins/ai-transport/http.lua +++ b/apisix/plugins/ai-transport/http.lua @@ -45,7 +45,11 @@ end --- Build forwarded headers from client request + extra headers. -- Copies client headers, merges ext_opts_headers (lowercased), -- forces Content-Type to application/json, removes host/content-length. -function _M.construct_forward_headers(ext_opts_headers, ctx) +-- When skip_client_headers is true the client's request headers are not +-- forwarded. Use it for self-contained sidecar calls (e.g. embeddings), which +-- carry their own credentials and must not leak the client's Authorization, +-- Cookie or other headers to a third-party endpoint. +function _M.construct_forward_headers(ext_opts_headers, ctx, skip_client_headers) local blacklist = { "host", "content-length", @@ -53,8 +57,10 @@ function _M.construct_forward_headers(ext_opts_headers, ctx) } local headers = {} - for k, v in pairs(core.request.headers(ctx) or {}) do - headers[str_lower(k)] = v + if not skip_client_headers then + for k, v in pairs(core.request.headers(ctx) or {}) do + headers[str_lower(k)] = v + end end for k, v in pairs(ext_opts_headers or {}) do headers[str_lower(k)] = v diff --git a/docs/en/latest/plugins/ai-proxy-multi.md b/docs/en/latest/plugins/ai-proxy-multi.md index a91b0f43b160..f76007dc002c 100644 --- a/docs/en/latest/plugins/ai-proxy-multi.md +++ b/docs/en/latest/plugins/ai-proxy-multi.md @@ -71,7 +71,9 @@ When an instance's `provider` is set to `bedrock`, the Plugin expects requests i | max_retries | integer | False | | greater or equal to 0 | Maximum number of fallback retries after the initial request fails. Bounds how many additional instances a single request tries, so it does not exhaust every configured instance. Only takes effect together with `fallback_strategy`. When unset, the Plugin retries until an instance succeeds or all are tried. | | retry_on_failure_within_ms | integer | False | | greater or equal to 1 | Only fall back to another instance when the upstream fails within this many milliseconds. Fast failures (such as connection errors or quick `429`/`5xx`) are retried, while a slow failure that takes longer than this is returned to the client directly to avoid doubling the wait time. Only takes effect together with `fallback_strategy`. When unset, the Plugin retries regardless of how long the failed attempt took. | | balancer | object | False | | | Load balancing configurations. | -| balancer.algorithm | string | False | roundrobin | [roundrobin, chash] | Load balancing algorithm. When set to `roundrobin`, weighted round robin algorithm is used. When set to `chash`, consistent hashing algorithm is used. | +| balancer.algorithm | string | False | roundrobin | [roundrobin, chash, semantic] | Load balancing algorithm. When set to `roundrobin`, weighted round robin algorithm is used. When set to `chash`, consistent hashing algorithm is used. When set to `semantic`, the Plugin picks an instance by the semantic similarity between the request prompt and each instance's `examples`. Note that `semantic` does not participate in health checks or `fallback_strategy` / retry — an upstream failure on the chosen instance is returned to the client; it only falls back (to the `catchall` instance, else the first instance) when no instance clears its threshold or embedding fails. | +| balancer.threshold | number | False | 0 | -1 to 1 | Used when `algorithm` is `semantic`. Global minimum cosine similarity an instance must reach to be selected. **The default of 0 admits essentially any prompt** (real embeddings are rarely dissimilar enough to score below 0), so the `catchall` only ever runs if you raise this above 0. When no instance clears its threshold, the request falls back to the `catchall` instance, or the first instance if none is set. | +| balancer.expose_scores | boolean | False | false | | Used when `algorithm` is `semantic`. If true, expose per-instance scores and the routing decision via `X-AI-Semantic-Scores` and `X-AI-Semantic-Route` response headers, for debugging. | | balancer.hash_on | string | False | | [vars, headers, cookie, consumer, vars_combinations] | Used when `type` is `chash`. Support hashing on [NGINX variables](https://nginx.org/en/docs/varindex.html), headers, cookie, consumer, or a combination of [NGINX variables](https://nginx.org/en/docs/varindex.html). | | balancer.key | string | False | | | Used when `type` is `chash`. When `hash_on` is set to `header` or `cookie`, `key` is required. When `hash_on` is set to `consumer`, `key` is not required as the consumer name will be used as the key automatically. | | instances | array[object] | True | | | LLM instance configurations. | @@ -95,6 +97,9 @@ When an instance's `provider` is set to `bedrock`, the Plugin expects requests i | instances.auth.aws.session_token | string | False | | minLength = 1 | AWS session token for temporary credentials (e.g., from STS assume-role). Encrypted at rest. | | instances.options | object | False | | | Model configurations. In addition to `model`, you can configure additional parameters and they will be forwarded to the upstream LLM service in the request body. For instance, if you are working with OpenAI, DeepSeek, or AIMLAPI, you can configure additional parameters such as `max_tokens`, `temperature`, `top_p`, and `stream`. See your LLM provider's API documentation for more available options. | | instances.options.model | string | False | | | Name of the LLM model, such as `gpt-4` or `gpt-3.5`. See your LLM provider's API documentation for more available models. For Bedrock, this can be a foundation model ID (e.g., `anthropic.claude-3-5-sonnet-20240620-v1:0`), a cross-region inference profile ID (e.g., `us.anthropic.claude-3-5-sonnet-20240620-v1:0`), or an application inference profile ARN (e.g., `arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/abc123`). | +| instances.examples | array[string] | False | | 1 to 64 items | Used when `algorithm` is `semantic`. Example utterances representing this instance's intent; each is embedded into its own reference vector. Required for every instance except the `catchall`, which must not set them. | +| instances.threshold | number | False | | -1 to 1 | Used when `algorithm` is `semantic`. Per-instance minimum cosine similarity; overrides `balancer.threshold`. | +| instances.catchall | boolean | False | | | Used when `algorithm` is `semantic`. Marks this instance as the semantic fallback, used when no instance clears its threshold. It takes no part in ranking, so it must not configure `examples`. At most one instance may be marked `catchall`. | | logging | object | False | | | Logging configurations. | | logging.summaries | boolean | False | false | | If true, log request LLM model, duration, request, and response tokens. | | logging.payloads | boolean | False | false | | If true, log request and response payload. | @@ -122,6 +127,15 @@ When an instance's `provider` is set to `bedrock`, the Plugin expects requests i | instances.checks.active.unhealthy.http_statuses | array[integer] | False | [429,404,500,501,502,503,504,505] | status code between 200 and 599 inclusive | An array of HTTP status codes that defines an unhealthy node. | | instances.checks.active.unhealthy.http_failures | integer | False | 5 | between 1 and 254 inclusive | Number of HTTP failures to define an unhealthy node. | | instances.checks.active.unhealthy.timeout | integer | False | 3 | between 1 and 254 inclusive | Number of probe timeouts to define an unhealthy node. | +| embeddings | object | False | | | Embedding service configuration. Required when `balancer.algorithm` is `semantic`. | +| embeddings.provider | string | True | | [openai, azure-openai] | Embedding provider used by the semantic algorithm. | +| embeddings.model | string | True | | | Embedding model name, e.g. `text-embedding-3-small`. | +| embeddings.endpoint | string | False | | | Embedding API endpoint. Optional for `openai` (defaults to the public API). Required for `azure-openai`, where it must be the **full** URL including the deployment path and API version, e.g. `https://{resource}.openai.azure.com/openai/deployments/{deployment}/embeddings?api-version=2024-02-01`. | +| embeddings.auth | object | True | | | Authentication configurations for the embedding service. | +| embeddings.auth.header | object | False | | | Authentication headers. Encrypted at rest. | +| embeddings.auth.query | object | False | | | Authentication query parameters. Encrypted at rest. | +| embeddings.timeout | integer | False | 10000 | minimum = 1 | Embedding request timeout in milliseconds. | +| embeddings.ssl_verify | boolean | False | true | | If true, verify the embedding service's certificate. | | timeout | integer | False | 30000 | greater than or equal to 1 | Request timeout in milliseconds when requesting the LLM service. Applied per socket operation (connect / send / read block); does not cap the total duration of a streaming response. | | max_req_body_size | integer | False | 67108864 | greater than or equal to 1 | Maximum request body size in bytes that the plugin reads into memory. Requests whose body exceeds this limit are rejected with `413`. Prevents unbounded memory buffering of large request bodies. | | max_stream_duration_ms | integer | False | | greater than or equal to 1 | Maximum wall-clock duration (in milliseconds) for a streaming AI response. If the upstream keeps sending data past this deadline, the gateway closes the connection. Unset means no cap. Use this to protect the gateway from upstream bugs that produce tokens indefinitely. When the limit is hit mid-stream, the downstream SSE stream is truncated (no protocol-specific terminator such as `[DONE]`, `message_stop`, or `response.completed`); well-behaved clients should treat a missing terminator as an incomplete response. | @@ -2974,3 +2988,137 @@ You should receive a response similar to the following if the request is forward ``` In the Kafka topic, you should also see a log entry corresponding to the request with the LLM summary and request/response payload. + +### Route by Semantic Similarity + +The following example demonstrates how you can use the `semantic` algorithm to pick an instance by the meaning of the request prompt, rather than by weight or hash. Each non-`catchall` instance declares `examples` that describe the kind of prompt it should handle; the Plugin embeds those examples and the incoming prompt with the configured embedding model, and forwards the request to the instance whose examples are most similar. + +Create a Route as such and update the LLM providers, embedding model, and API keys accordingly. The `code` instance handles programming prompts, the `translate` instance handles translation prompts, and the `default` instance is the `catchall` used when no instance clears the similarity threshold: + +```shell +curl "http://127.0.0.1:9180/apisix/admin/routes" -X PUT \ + -H "X-API-KEY: ${admin_key}" \ + -d '{ + "id": "ai-proxy-multi-route", + "uri": "/anything", + "methods": ["POST"], + "plugins": { + "ai-proxy-multi": { + "embeddings": { + "provider": "openai", + "model": "text-embedding-3-small", + "auth": { + "header": { + "Authorization": "Bearer '"$YOUR_EMBEDDING_API_KEY"'" + } + } + }, + "balancer": { + "algorithm": "semantic", + "threshold": 0.5 + }, + "instances": [ + { + "name": "code", + "provider": "openai", + "weight": 1, + "auth": { + "header": { + "Authorization": "Bearer '"$YOUR_LLM_API_KEY"'" + } + }, + "options": { + "model": "gpt-4o" + }, + "examples": [ + "write a python function", + "debug this code" + ] + }, + { + "name": "translate", + "provider": "openai", + "weight": 1, + "auth": { + "header": { + "Authorization": "Bearer '"$YOUR_LLM_API_KEY"'" + } + }, + "options": { + "model": "gpt-4o-mini" + }, + "examples": [ + "translate this sentence into English" + ] + }, + { + "name": "default", + "provider": "openai", + "weight": 1, + "auth": { + "header": { + "Authorization": "Bearer '"$YOUR_LLM_API_KEY"'" + } + }, + "options": { + "model": "gpt-4o-mini" + }, + "catchall": true + } + ] + } + } + }' +``` + +Send a request with a programming prompt: + +```shell +curl "http://127.0.0.1:9080/anything" -X POST \ + -H "Content-Type: application/json" \ + -d '{ + "model": "auto", + "messages": [ + { "role": "user", "content": "help me debug this python code" } + ] + }' +``` + +The request should be routed to the `code` instance and served by `gpt-4o`. A prompt unrelated to any instance's `examples` (for example, `what is the weather today`) clears no threshold and is routed to the `default` `catchall` instance instead. + +#### Debugging the routing decision + +Set `balancer.expose_scores` to `true` to have the Plugin report how each instance scored and which one it picked: + +```shell +curl -i "http://127.0.0.1:9080/anything" -X POST \ + -H "Content-Type: application/json" \ + -d '{"model":"auto","messages":[{"role":"user","content":"help me debug this python code"}]}' +``` + +```text +HTTP/1.1 200 OK +X-AI-Semantic-Route: code +X-AI-Semantic-Scores: code:0.8213,translate:0.1904 +``` + +`X-AI-Semantic-Scores` lists every instance that has `examples`, highest score first, so you can see not only the winner but how close the runners-up came. Instances without `examples` — that is, the `catchall` — carry no similarity score and therefore do not appear. + +When no instance clears its threshold, the pick becomes `fallback` while the scores still show how close each one got: + +```text +X-AI-Semantic-Route: fallback +X-AI-Semantic-Scores: code:0.3057,translate:0.2884 +``` + +Use this to calibrate `threshold`: pick a value between the scores of the prompts you want matched and the scores of the prompts you want to reach the `catchall`. + +The same information is available without exposing it to clients: whenever no instance clears its threshold, the Plugin logs the full score list at `warn` level. + +```text +semantic routing: no instance cleared threshold (scores: code:0.3057,translate:0.2884), falling back +``` + +If the headers are missing entirely while `expose_scores` is on, the embedding step itself failed and the request was served by the `catchall` before any score was computed; the reason is logged at `warn`. + +Because `expose_scores` reveals instance names to the caller, keep it off in production and rely on the log line instead. diff --git a/docs/zh/latest/plugins/ai-proxy-multi.md b/docs/zh/latest/plugins/ai-proxy-multi.md index 8f3ea99dc615..65eac5df0449 100644 --- a/docs/zh/latest/plugins/ai-proxy-multi.md +++ b/docs/zh/latest/plugins/ai-proxy-multi.md @@ -71,7 +71,9 @@ import TabItem from '@theme/TabItem'; | max_retries | integer | 否 | | 大于或等于 0 | 初始请求失败后允许的最大故障转移重试次数。用于限制单个请求最多尝试多少个额外实例，避免穷举所有已配置的实例。仅在配置 `fallback_strategy` 时生效。未设置时，插件会持续重试直到某个实例成功或所有实例都已尝试。 | | retry_on_failure_within_ms | integer | 否 | | 大于或等于 1 | 仅当上游在指定毫秒数内失败时才故障转移到其他实例。快速失败（如连接错误、快速返回的 `429`/`5xx`）会触发重试，而耗时超过该值的慢失败会直接将错误返回给客户端，避免客户端等待时间翻倍。仅在配置 `fallback_strategy` 时生效。未设置时，插件无论失败请求耗时多久都会重试。 | | balancer | object | 否 | | | 负载均衡配置。 | -| balancer.algorithm | string | 否 | roundrobin | [roundrobin, chash] | 负载均衡算法。设置为 `roundrobin` 时，使用加权轮询算法。设置为 `chash` 时，使用一致性哈希算法。 | +| balancer.algorithm | string | 否 | roundrobin | [roundrobin, chash, semantic] | 负载均衡算法。设置为 `roundrobin` 时，使用加权轮询算法。设置为 `chash` 时，使用一致性哈希算法。设置为 `semantic` 时，插件根据请求提示词与各实例 `examples` 之间的语义相似度选择实例。注意：`semantic` 不参与健康检查，也不参与 `fallback_strategy` / 重试——被选中实例的上游失败会直接返回给客户端；只有当没有实例达到其阈值或嵌入请求失败时，才会回退（回退到 `catchall` 实例，若未配置则回退到第一个实例）。 | +| balancer.threshold | number | 否 | 0 | -1 到 1 | 当 `algorithm` 为 `semantic` 时使用。实例被选中所需达到的全局最小余弦相似度。**默认值 0 几乎会接纳任何提示词**（真实嵌入向量很少会不相似到得分低于 0），因此只有把它调到 0 以上，`catchall` 才会真正生效。当没有实例达到其阈值时，请求回退到 `catchall` 实例；若未配置 `catchall`，则回退到第一个实例。 | +| balancer.expose_scores | boolean | 否 | false | | 当 `algorithm` 为 `semantic` 时使用。若为 true，通过 `X-AI-Semantic-Scores` 和 `X-AI-Semantic-Route` 响应头暴露各实例的分数和路由决策，用于调试。 | | balancer.hash_on | string | 否 | | [vars, headers, cookie, consumer, vars_combinations] | 当 `type` 为 `chash` 时使用。支持基于 [NGINX 变量](https://nginx.org/en/docs/varindex.html)、标头、cookie、消费者或 [NGINX 变量](https://nginx.org/en/docs/varindex.html)组合进行哈希。 | | balancer.key | string | 否 | | | 当 `type` 为 `chash` 时使用。当 `hash_on` 设置为 `header` 或 `cookie` 时，需要 `key`。当 `hash_on` 设置为 `consumer` 时，不需要 `key`，因为消费者名称将自动用作键。 | | instances | array[object] | 是 | | | LLM 实例配置。 | @@ -101,6 +103,9 @@ import TabItem from '@theme/TabItem'; | instances.override.llm_options.max_tokens | integer | 否 | | ≥ 1 | 最大输出 token 数。APISIX 会自动将该值映射为各上游服务商对应的字段名。始终强制覆盖客户端值。 | | instances.override.request_body | object | 否 | | | 按目标协议的请求体覆盖配置。请参阅 `ai-proxy` 文档中的[按协议的请求体覆盖](./ai-proxy.md#per-protocol-request-body-override)。 | | instances.override.request_body_force_override | boolean | 否 | false | | 为 `false`（默认）时，客户端请求体中的字段优先，`instances.override.request_body` 仅补充缺失字段。为 `true` 时，`instances.override.request_body` 的值强制覆盖客户端请求体中的同名字段。不影响 `instances.override.llm_options`。 | +| instances.examples | array[string] | 否 | | 1 到 64 项 | 当 `algorithm` 为 `semantic` 时使用。代表该实例意图的示例语句；每一条都会被嵌入为独立的参考向量。除 `catchall` 实例外均为必填，而 `catchall` 实例不得配置该字段。 | +| instances.threshold | number | 否 | | -1 到 1 | 当 `algorithm` 为 `semantic` 时使用。该实例的最小余弦相似度，覆盖 `balancer.threshold`。 | +| instances.catchall | boolean | 否 | | | 当 `algorithm` 为 `semantic` 时使用。将该实例标记为语义回退实例，当没有实例达到其阈值时使用。它不参与排名，因此不得配置 `examples`。最多只能有一个实例被标记为 `catchall`。 | | logging | object | 否 | | | 日志配置。不影响 `error.log`。 | | logging.summaries | boolean | 否 | false | | 如果为 true，记录请求 LLM 模型、持续时间、请求和响应令牌。 | | logging.payloads | boolean | 否 | false | | 如果为 true，记录请求和响应负载。 | @@ -124,6 +129,15 @@ import TabItem from '@theme/TabItem'; | instances.checks.active.unhealthy.http_statuses | array[integer] | 否 | [429,404,500,501,502,503,504,505] | 200 到 599 之间的状态码（包含） | 定义不健康节点的 HTTP 状态码数组。 | | instances.checks.active.unhealthy.http_failures | integer | 否 | 5 | 1 到 254（包含） | 定义不健康节点的 HTTP 失败次数。 | | instances.checks.active.unhealthy.timeout | integer | 否 | 3 | 1 到 254（包含） | 定义不健康节点的探测超时次数。 | +| embeddings | object | 否 | | | 嵌入服务配置。当 `balancer.algorithm` 为 `semantic` 时必填。 | +| embeddings.provider | string | 是 | | [openai, azure-openai] | 语义算法使用的嵌入服务提供商。 | +| embeddings.model | string | 是 | | | 嵌入模型名称，例如 `text-embedding-3-small`。 | +| embeddings.endpoint | string | 否 | | | 嵌入 API 端点。对于 `openai` 可选（默认使用公共 API）。对于 `azure-openai` 必填，且必须是**完整** URL，包含 deployment 路径和 API 版本，例如 `https://{resource}.openai.azure.com/openai/deployments/{deployment}/embeddings?api-version=2024-02-01`。 | +| embeddings.auth | object | 是 | | | 嵌入服务的认证配置。 | +| embeddings.auth.header | object | 否 | | | 认证请求头。加密存储。 | +| embeddings.auth.query | object | 否 | | | 认证查询参数。加密存储。 | +| embeddings.timeout | integer | 否 | 10000 | 最小值为 1 | 嵌入请求的超时时间（毫秒）。 | +| embeddings.ssl_verify | boolean | 否 | true | | 如果为 true，验证嵌入服务的证书。 | | timeout | integer | 否 | 30000 | 大于或等于 1 | 请求 LLM 服务时的请求超时时间（毫秒）。应用于单次 socket 操作（连接 / 发送 / 读取块），不限制流式响应的总时长。 | | max_stream_duration_ms | integer | 否 | | 大于或等于 1 | 流式 AI 响应的总墙钟时长上限（毫秒）。若上游在此时间后仍持续发送数据，网关将关闭连接。未设置时不限制。用于防护上游持续输出 token 导致网关 CPU 被打满的异常情况。中途触发上限时，下游 SSE 流会被截断（不再发送协议特定的终止标记，例如 `[DONE]`、`message_stop` 或 `response.completed`），客户端应将缺失的终止标记视为响应未完成。 | | max_response_bytes | integer | 否 | | 大于或等于 1 | 单次 AI 响应（流式或非流式）允许从上游读取的最大总字节数。超出时关闭连接。非流式响应若存在 `Content-Length`，在读取 body 之前预检；否则（chunked 传输）与流式响应一样在接收字节的过程中增量检查。未设置时不限制。 | @@ -2779,3 +2793,137 @@ nginx_config: ``` 访问日志条目显示请求类型为 `ai_chat`，Apisix 上游响应时间为 `5765` 毫秒，首次令牌时间为 `2858` 毫秒，请求的 LLM 模型为 `gpt-4`。LLM 模型为 `gpt-4`，提示令牌使用量为 `23`，完成令牌使用量为 `8`。 + +### 按语义相似度路由 + +以下示例演示了如何使用 `semantic` 算法，根据请求提示词的语义而非权重或哈希来选择实例。每个非 `catchall` 实例通过 `examples` 声明它应当处理的提示词类型；插件使用配置的嵌入模型将这些示例和传入的提示词嵌入为向量，并将请求转发给示例与之最相似的实例。 + +创建如下路由，并相应地更新 LLM 提供商、嵌入模型和 API 密钥。`code` 实例处理编程类提示词，`translate` 实例处理翻译类提示词，`default` 实例是 `catchall`，当没有实例达到相似度阈值时使用： + +```shell +curl "http://127.0.0.1:9180/apisix/admin/routes" -X PUT \ + -H "X-API-KEY: ${admin_key}" \ + -d '{ + "id": "ai-proxy-multi-route", + "uri": "/anything", + "methods": ["POST"], + "plugins": { + "ai-proxy-multi": { + "embeddings": { + "provider": "openai", + "model": "text-embedding-3-small", + "auth": { + "header": { + "Authorization": "Bearer '"$YOUR_EMBEDDING_API_KEY"'" + } + } + }, + "balancer": { + "algorithm": "semantic", + "threshold": 0.5 + }, + "instances": [ + { + "name": "code", + "provider": "openai", + "weight": 1, + "auth": { + "header": { + "Authorization": "Bearer '"$YOUR_LLM_API_KEY"'" + } + }, + "options": { + "model": "gpt-4o" + }, + "examples": [ + "write a python function", + "debug this code" + ] + }, + { + "name": "translate", + "provider": "openai", + "weight": 1, + "auth": { + "header": { + "Authorization": "Bearer '"$YOUR_LLM_API_KEY"'" + } + }, + "options": { + "model": "gpt-4o-mini" + }, + "examples": [ + "translate this sentence into English" + ] + }, + { + "name": "default", + "provider": "openai", + "weight": 1, + "auth": { + "header": { + "Authorization": "Bearer '"$YOUR_LLM_API_KEY"'" + } + }, + "options": { + "model": "gpt-4o-mini" + }, + "catchall": true + } + ] + } + } + }' +``` + +发送一个编程类提示词的请求： + +```shell +curl "http://127.0.0.1:9080/anything" -X POST \ + -H "Content-Type: application/json" \ + -d '{ + "model": "auto", + "messages": [ + { "role": "user", "content": "help me debug this python code" } + ] + }' +``` + +该请求应被路由到 `code` 实例，由 `gpt-4o` 处理。如果提示词与任何实例的 `examples` 都不相关（例如 `what is the weather today`），则不会达到任何阈值，请求会被路由到 `default` 这个 `catchall` 实例。 + +#### 调试路由决策 + +将 `balancer.expose_scores` 设置为 `true`，插件会在响应中报告各实例的得分以及最终选中的实例： + +```shell +curl -i "http://127.0.0.1:9080/anything" -X POST \ + -H "Content-Type: application/json" \ + -d '{"model":"auto","messages":[{"role":"user","content":"help me debug this python code"}]}' +``` + +```text +HTTP/1.1 200 OK +X-AI-Semantic-Route: code +X-AI-Semantic-Scores: code:0.8213,translate:0.1904 +``` + +`X-AI-Semantic-Scores` 按得分从高到低列出**所有配置了 `examples` 的实例**，因此你不仅能看到胜出者，也能看到其他实例差了多少。没有配置 `examples` 的实例（即 `catchall`）没有相似度得分，因此不会出现在其中。 + +当没有任何实例达到阈值时，选中结果变为 `fallback`，而得分仍会显示各实例的接近程度： + +```text +X-AI-Semantic-Route: fallback +X-AI-Semantic-Scores: code:0.3057,translate:0.2884 +``` + +据此可以校准 `threshold`：取一个介于「希望匹配的提示词得分」与「希望落到 `catchall` 的提示词得分」之间的值。 + +同样的信息也可以不暴露给客户端：每当没有实例达到阈值时，插件都会以 `warn` 级别记录完整的得分列表。 + +```text +semantic routing: no instance cleared threshold (scores: code:0.3057,translate:0.2884), falling back +``` + +如果开启了 `expose_scores` 却完全没有这两个响应头，说明嵌入请求本身失败了，请求在计算任何得分之前就已回退到 `catchall`，失败原因会以 `warn` 级别记录。 + +由于 `expose_scores` 会向调用方暴露实例名称，生产环境应保持关闭，改用日志排查。 diff --git a/t/plugin/ai-proxy-semantic-routing.t b/t/plugin/ai-proxy-semantic-routing.t new file mode 100644 index 000000000000..49df719b7207 --- /dev/null +++ b/t/plugin/ai-proxy-semantic-routing.t @@ -0,0 +1,731 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +use t::APISIX 'no_plan'; + +log_level("info"); +repeat_each(1); +no_long_string(); +no_root_location(); +no_shuffle(); + +add_block_preprocessor(sub { + my ($block) = @_; + + if (!defined $block->request) { + $block->set_value("request", "GET /t"); + } + + my $http_config = $block->http_config // <<_EOC_; + # Mock embedding endpoint: returns a 2-D vector keyed on keywords so + # tests can steer which instance a prompt matches. + server { + server_name mock_embedding; + listen 6797; + default_type 'application/json'; + location /v1/embeddings { + content_by_lua_block { + local json = require("cjson.safe") + ngx.req.read_body() + -- the embedding call must not carry the client's headers + ngx.log(ngx.WARN, "embed-recv-cookie:", + ngx.var.http_cookie or "none") + local body = json.decode(ngx.req.get_body_data()) or {} + for _, t in ipairs(body.input or {}) do + if string.lower(t):find("servererror") then + -- an error body that echoes the prompt back, as some + -- providers do -- it must never reach our logs + ngx.status = 500 + ngx.say('{"error":"bad input: ', t, ' SENSITIVE-ECHO"}') + return + elseif string.lower(t):find("scalarbody") then + -- 200 with a bare JSON null: decodes to a non-table, + -- which must not be indexed + ngx.status = 200 + ngx.say("null") + return + elseif string.lower(t):find("scalarentry") then + -- 200 whose data array holds non-objects: indexing + -- item.index on a number would raise + ngx.status = 200 + ngx.say('{"data":[1,2,3]}') + return + end + end + local function vec(text) + text = string.lower(text or "") + if text:find("code") or text:find("python") or text:find("debug") then + return {1, 0} + elseif text:find("translate") or text:find("summar") then + return {0, 1} + end + return {0.6, 0.6} + end + local data = {} + for i, t in ipairs(body.input or {}) do + if string.lower(t):find("malformed") then + -- emit a JSON null embedding (decodes to cjson.null) + -- to exercise fail-open on malformed data + data[i] = { index = i - 1, embedding = json.null } + elseif string.lower(t):find("dimmismatch") then + -- 3-D vector vs the 2-D reference vectors + data[i] = { index = i - 1, embedding = {1, 0, 0} } + else + data[i] = { index = i - 1, embedding = vec(t) } + end + end + ngx.status = 200 + ngx.say(json.encode({ data = data })) + } + } + } + # Mock Azure OpenAI embeddings: the deployment and api-version live in + # the URL, exactly as a real Azure endpoint requires. + server { + server_name mock_azure_embedding; + listen 6799; + default_type 'application/json'; + location /openai/deployments/emb/embeddings { + content_by_lua_block { + local json = require("cjson.safe") + ngx.req.read_body() + -- prove the request reached the full Azure path with its query + -- (an nginx arg variable cannot express the hyphen) + local args = ngx.req.get_uri_args() + ngx.log(ngx.WARN, "azure-embed-hit api-version=", + args["api-version"] or "none") + local body = json.decode(ngx.req.get_body_data()) or {} + local data = {} + for i, t in ipairs(body.input or {}) do + local v = {0.6, 0.6} + if string.lower(t):find("code") or string.lower(t):find("python") then + v = {1, 0} + end + data[i] = { index = i - 1, embedding = v } + end + ngx.status = 200 + ngx.say(json.encode({ data = data })) + } + } + } + # Mock LLM: echoes the model it received so tests can tell which + # instance was selected. + server { + server_name mock_llm; + listen 6798; + default_type 'application/json'; + location /v1/chat/completions { + content_by_lua_block { + local json = require("cjson.safe") + ngx.req.read_body() + local body = json.decode(ngx.req.get_body_data()) or {} + ngx.status = 200 + ngx.print(body.model or "unknown") + } + } + } +_EOC_ + $block->set_value("http_config", $http_config); + + my $user_yaml_config = <<_EOC_; +plugins: + - ai-proxy-multi +_EOC_ + $block->set_value("extra_yaml_config", $user_yaml_config); +}); + +run_tests(); + +__DATA__ + +=== TEST 1: configure a semantic route (code / translate / catchall) +--- config + location /t { + content_by_lua_block { + -- Build the config as a Lua table and core.json.encode it. A long + -- inline JSON `[[ ]]` block that crosses nginx's config buffer + -- boundary is misparsed by ngx_lua ("missing the closing long + -- bracket"), so we keep the inlined Lua short instead. + local core = require("apisix.core") + local t = require("lib.test_admin").test + local function inst(name, model, examples, catchall) + local i = { + name = name, provider = "openai", weight = 1, + auth = { header = { Authorization = "Bearer token" } }, + options = { model = model }, + override = { + endpoint = "http://127.0.0.1:6798/v1/chat/completions", + }, + } + if examples then i.examples = examples end + if catchall then i.catchall = true end + return i + end + local conf = { + uri = "/anything", + plugins = { + ["ai-proxy-multi"] = { + embeddings = { + provider = "openai", + model = "text-embedding-3-small", + endpoint = "http://127.0.0.1:6797/v1/embeddings", + auth = { header = { Authorization = "Bearer token" } }, + ssl_verify = false, + }, + balancer = { algorithm = "semantic", threshold = 0.9 }, + instances = { + inst("code", "model-code", {"write python code"}), + inst("cheap", "model-cheap", {"translate this text"}), + inst("default", "model-fallback", nil, true), + }, + ssl_verify = false, + }, + }, + } + local code, body = t('/apisix/admin/routes/1', ngx.HTTP_PUT, + core.json.encode(conf)) + if code >= 300 then + ngx.status = code + ngx.say(body) + return + end + ngx.say("passed") + } + } +--- response_body +passed +--- no_error_log +[error] + + + +=== TEST 2: coding prompt routes to the code model +--- request +POST /anything +{"model":"auto","messages":[{"role":"user","content":"help me debug this python code"}]} +--- more_headers +Content-Type: application/json +--- response_body chomp +model-code +--- no_error_log +[error] + + + +=== TEST 3: translation prompt routes to the cheap model +--- request +POST /anything +{"model":"auto","messages":[{"role":"user","content":"please translate this to english"}]} +--- more_headers +Content-Type: application/json +--- response_body chomp +model-cheap +--- no_error_log +[error] + + + +=== TEST 4: unrelated prompt clears no threshold, falls back to catchall +--- request +POST /anything +{"model":"auto","messages":[{"role":"user","content":"what is the weather today"}]} +--- more_headers +Content-Type: application/json +--- response_body chomp +model-fallback +--- error_log eval +qr/no instance cleared threshold \(scores: code:0\.\d+,cheap:0\.\d+\)/ +--- no_error_log +[error] + + + +=== TEST 5: malformed embedding fails open to catchall, not the matching instance +--- request +POST /anything +{"model":"auto","messages":[{"role":"user","content":"debug this python code, malformed"}]} +--- more_headers +Content-Type: application/json +--- error_code: 200 +--- response_body chomp +model-fallback +--- error_log +semantic routing: query embedding failed +--- no_error_log +[error] + + + +=== TEST 6: reconfigure the route with expose_scores enabled +--- config + location /t { + content_by_lua_block { + local core = require("apisix.core") + local t = require("lib.test_admin").test + local function inst(name, model, examples, catchall) + local i = { + name = name, provider = "openai", weight = 1, + auth = { header = { Authorization = "Bearer token" } }, + options = { model = model }, + override = { + endpoint = "http://127.0.0.1:6798/v1/chat/completions", + }, + } + if examples then i.examples = examples end + if catchall then i.catchall = true end + return i + end + local conf = { + uri = "/anything", + plugins = { + ["ai-proxy-multi"] = { + embeddings = { + provider = "openai", + model = "text-embedding-3-small", + endpoint = "http://127.0.0.1:6797/v1/embeddings", + auth = { header = { Authorization = "Bearer token" } }, + ssl_verify = false, + }, + balancer = { + algorithm = "semantic", threshold = 0.9, + expose_scores = true, + }, + instances = { + inst("code", "model-code", {"write python code"}), + inst("cheap", "model-cheap", {"translate this text"}), + inst("default", "model-fallback", nil, true), + }, + ssl_verify = false, + }, + }, + } + local code, body = t('/apisix/admin/routes/1', ngx.HTTP_PUT, + core.json.encode(conf)) + if code >= 300 then + ngx.status = code + ngx.say(body) + return + end + ngx.say("passed") + } + } +--- response_body +passed +--- no_error_log +[error] + + + +=== TEST 7: debug exposes per-instance scores and the pick via headers +--- request +POST /anything +{"model":"auto","messages":[{"role":"user","content":"help me debug this python code"}]} +--- more_headers +Content-Type: application/json +--- response_body chomp +model-code +--- response_headers_like +X-AI-Semantic-Route: code +X-AI-Semantic-Scores: code:1\.\d+,cheap:0\.\d+ +--- no_error_log +[error] + + + +=== TEST 8: embedding dimension mismatch fails open to catchall +--- request +POST /anything +{"model":"auto","messages":[{"role":"user","content":"dimmismatch python code"}]} +--- more_headers +Content-Type: application/json +--- error_code: 200 +--- response_body chomp +model-fallback +--- error_log +embedding dimension mismatch +--- no_error_log +[error] + + + +=== TEST 9: multimodal content array extracts text parts for routing +--- request +POST /anything +{"model":"auto","messages":[{"role":"user","content":[ +{"type":"text","text":"write python code"}, +{"type":"image_url","image_url":{"url":"http://x/y.png"}} +]}]} +--- more_headers +Content-Type: application/json +--- response_body chomp +model-code +--- no_error_log +[error] + + + +=== TEST 10: client headers are not forwarded to the embedding endpoint +--- request +POST /anything +{"model":"auto","messages":[{"role":"user","content":"help me debug this python code"}]} +--- more_headers +Content-Type: application/json +Cookie: session=super-secret +--- response_body chomp +model-code +--- error_log +embed-recv-cookie:none +--- no_error_log +embed-recv-cookie:session=super-secret + + + +=== TEST 11: non-200 fails open to catchall, without logging the upstream body +--- request +POST /anything +{"model":"auto","messages":[{"role":"user","content":"servererror python code"}]} +--- more_headers +Content-Type: application/json +--- error_code: 200 +--- response_body chomp +model-fallback +--- error_log +embedding endpoint returned status 500 +--- no_error_log +SENSITIVE-ECHO + + + +=== TEST 12: 200 with a non-table body fails open instead of raising +--- request +POST /anything +{"model":"auto","messages":[{"role":"user","content":"scalarbody python code"}]} +--- more_headers +Content-Type: application/json +--- error_code: 200 +--- response_body chomp +model-fallback +--- error_log +invalid embedding response +--- no_error_log +[error] + + + +=== TEST 13: 200 whose data array holds non-objects fails open instead of raising +--- request +POST /anything +{"model":"auto","messages":[{"role":"user","content":"scalarentry python code"}]} +--- more_headers +Content-Type: application/json +--- error_code: 200 +--- response_body chomp +model-fallback +--- error_log +invalid embedding entry at index 0 +--- no_error_log +[error] + + + +=== TEST 14: reference embedding failure fails open to catchall +--- config + location /t { + content_by_lua_block { + local core = require("apisix.core") + local t = require("lib.test_admin").test + local function inst(name, model, examples, catchall) + local i = { + name = name, provider = "openai", weight = 1, + auth = { header = { Authorization = "Bearer token" } }, + options = { model = model }, + override = { + endpoint = "http://127.0.0.1:6798/v1/chat/completions", + }, + } + if examples then i.examples = examples end + if catchall then i.catchall = true end + return i + end + local conf = { + uri = "/anything", + plugins = { + ["ai-proxy-multi"] = { + embeddings = { + provider = "openai", + model = "text-embedding-3-small", + endpoint = "http://127.0.0.1:6797/v1/embeddings", + auth = { header = { Authorization = "Bearer token" } }, + ssl_verify = false, + }, + balancer = { algorithm = "semantic", threshold = 0.9 }, + instances = { + -- embedding this instance's examples makes the mock + -- fail the whole reference batch + inst("code", "model-code", {"servererror example"}), + inst("default", "model-fallback", nil, true), + }, + ssl_verify = false, + }, + }, + } + local code, body = t('/apisix/admin/routes/1', ngx.HTTP_PUT, + core.json.encode(conf)) + if code >= 300 then + ngx.status = code + ngx.say(body) + return + end + ngx.say("passed") + } + } +--- response_body +passed +--- no_error_log +[error] + + + +=== TEST 15: request on the broken-reference route still routes to catchall +--- request +POST /anything +{"model":"auto","messages":[{"role":"user","content":"help me debug this python code"}]} +--- more_headers +Content-Type: application/json +--- error_code: 200 +--- response_body chomp +model-fallback +--- error_log +failed to fetch reference embeddings +--- no_error_log +[error] + + + +=== TEST 16: azure-openai embeddings route via the full deployment URL +--- config + location /t { + content_by_lua_block { + local core = require("apisix.core") + local t = require("lib.test_admin").test + local function inst(name, model, examples, catchall) + local i = { + name = name, provider = "openai", weight = 1, + auth = { header = { Authorization = "Bearer token" } }, + options = { model = model }, + override = { + endpoint = "http://127.0.0.1:6798/v1/chat/completions", + }, + } + if examples then i.examples = examples end + if catchall then i.catchall = true end + return i + end + local conf = { + uri = "/anything", + plugins = { + ["ai-proxy-multi"] = { + embeddings = { + provider = "azure-openai", + model = "text-embedding-3-small", + endpoint = "http://127.0.0.1:6799/openai/deployments/emb" + .. "/embeddings?api-version=2024-02-01", + auth = { header = { ["api-key"] = "azure-key" } }, + ssl_verify = false, + }, + balancer = { algorithm = "semantic", threshold = 0.9 }, + instances = { + inst("code", "model-code", {"write python code"}), + inst("default", "model-fallback", nil, true), + }, + ssl_verify = false, + }, + }, + } + local code, body = t('/apisix/admin/routes/1', ngx.HTTP_PUT, + core.json.encode(conf)) + if code >= 300 then + ngx.status = code + ngx.say(body) + return + end + ngx.say("passed") + } + } +--- response_body +passed +--- no_error_log +[error] + + + +=== TEST 17: azure embeddings request reaches the deployment path with api-version +--- request +POST /anything +{"model":"auto","messages":[{"role":"user","content":"help me debug this python code"}]} +--- more_headers +Content-Type: application/json +--- response_body chomp +model-code +--- error_log +azure-embed-hit api-version=2024-02-01 +--- no_error_log +[error] + + + +=== TEST 18: no catchall configured -- fallback is the first instance +--- config + location /t { + content_by_lua_block { + local core = require("apisix.core") + local t = require("lib.test_admin").test + local function inst(name, model, examples) + return { + name = name, provider = "openai", weight = 1, + auth = { header = { Authorization = "Bearer token" } }, + options = { model = model }, + override = { + endpoint = "http://127.0.0.1:6798/v1/chat/completions", + }, + examples = examples, + } + end + local conf = { + uri = "/anything", + plugins = { + ["ai-proxy-multi"] = { + embeddings = { + provider = "openai", + model = "text-embedding-3-small", + endpoint = "http://127.0.0.1:6797/v1/embeddings", + auth = { header = { Authorization = "Bearer token" } }, + ssl_verify = false, + }, + balancer = { algorithm = "semantic", threshold = 0.9 }, + instances = { + inst("first", "model-first", {"write python code"}), + inst("second", "model-second", {"translate this text"}), + }, + ssl_verify = false, + }, + }, + } + local code, body = t('/apisix/admin/routes/1', ngx.HTTP_PUT, + core.json.encode(conf)) + if code >= 300 then + ngx.status = code + ngx.say(body) + return + end + ngx.say("passed") + } + } +--- response_body +passed +--- no_error_log +[error] + + + +=== TEST 19: unmatched prompt falls back to the first instance +--- request +POST /anything +{"model":"auto","messages":[{"role":"user","content":"what is the weather today"}]} +--- more_headers +Content-Type: application/json +--- response_body chomp +model-first +--- error_log +no instance cleared threshold +--- no_error_log +[error] + + + +=== TEST 20: per-instance threshold overrides the global one +--- config + location /t { + content_by_lua_block { + local core = require("apisix.core") + local t = require("lib.test_admin").test + local conf = { + uri = "/anything", + plugins = { + ["ai-proxy-multi"] = { + embeddings = { + provider = "openai", + model = "text-embedding-3-small", + endpoint = "http://127.0.0.1:6797/v1/embeddings", + auth = { header = { Authorization = "Bearer token" } }, + ssl_verify = false, + }, + -- the global threshold is permissive (0.1); the code + -- instance raises its own bar to 0.9, so a prompt scoring + -- ~0.707 clears the global one but not the instance's + balancer = { algorithm = "semantic", threshold = 0.1 }, + instances = { + { + name = "code", provider = "openai", weight = 1, + auth = { header = { Authorization = "Bearer token" } }, + options = { model = "model-code" }, + override = { + endpoint = "http://127.0.0.1:6798/v1/chat/completions", + }, + examples = {"write python code"}, + threshold = 0.9, + }, + { + name = "default", provider = "openai", weight = 1, + auth = { header = { Authorization = "Bearer token" } }, + options = { model = "model-fallback" }, + override = { + endpoint = "http://127.0.0.1:6798/v1/chat/completions", + }, + catchall = true, + }, + }, + ssl_verify = false, + }, + }, + } + local code, body = t('/apisix/admin/routes/1', ngx.HTTP_PUT, + core.json.encode(conf)) + if code >= 300 then + ngx.status = code + ngx.say(body) + return + end + ngx.say("passed") + } + } +--- response_body +passed +--- no_error_log +[error] + + + +=== TEST 21: a prompt below the per-instance threshold reaches the catchall +--- request +POST /anything +{"model":"auto","messages":[{"role":"user","content":"what is the weather today"}]} +--- more_headers +Content-Type: application/json +--- response_body chomp +model-fallback +--- no_error_log +[error] diff --git a/t/plugin/ai-proxy-semantic-schema.t b/t/plugin/ai-proxy-semantic-schema.t new file mode 100644 index 000000000000..dd2a4aded983 --- /dev/null +++ b/t/plugin/ai-proxy-semantic-schema.t @@ -0,0 +1,408 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +use t::APISIX 'no_plan'; + +log_level("info"); +repeat_each(1); +no_long_string(); +no_root_location(); +no_shuffle(); + +add_block_preprocessor(sub { + my ($block) = @_; + if (!defined $block->request) { + $block->set_value("request", "GET /t"); + } +}); + +run_tests(); + +__DATA__ + +=== TEST 1: valid semantic config is accepted +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', ngx.HTTP_PUT, [[{ + "uri": "/anything", + "plugins": { + "ai-proxy-multi": { + "embeddings": { + "provider": "openai", + "model": "text-embedding-3-small", + "auth": { "header": { "Authorization": "Bearer sk-emb" } } + }, + "balancer": { + "algorithm": "semantic", "threshold": 0.75 + }, + "instances": [ + { + "name": "code", "provider": "openai", "weight": 1, + "auth": { "header": { "Authorization": "Bearer token" } }, + "options": { "model": "gpt-4o" }, + "examples": ["write a python function", "debug this code"], + "threshold": 0.85 + }, + { + "name": "default", "provider": "openai", "weight": 1, + "auth": { "header": { "Authorization": "Bearer token" } }, + "options": { "model": "gpt-4o-mini" }, + "catchall": true + } + ], + "ssl_verify": false + } + } + }]]) + if code >= 300 then + ngx.status = code + ngx.say(body) + return + end + ngx.say("passed") + } + } +--- response_body +passed +--- no_error_log +[error] + + + +=== TEST 2: semantic requires examples on non-catchall instance +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', ngx.HTTP_PUT, [[{ + "uri": "/anything", + "plugins": { + "ai-proxy-multi": { + "embeddings": { + "provider": "openai", "model": "text-embedding-3-small", + "auth": { "header": { "Authorization": "Bearer sk-emb" } } + }, + "balancer": { "algorithm": "semantic" }, + "instances": [ + { + "name": "code", "provider": "openai", "weight": 1, + "auth": { "header": { "Authorization": "Bearer token" } }, + "options": { "model": "gpt-4o" } + } + ], + "ssl_verify": false + } + } + }]]) + ngx.status = code + ngx.say(body) + } + } +--- error_code: 400 +--- response_body_like: examples + + + +=== TEST 3: semantic requires embeddings config +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', ngx.HTTP_PUT, [[{ + "uri": "/anything", + "plugins": { + "ai-proxy-multi": { + "balancer": { "algorithm": "semantic" }, + "instances": [ + { + "name": "code", "provider": "openai", "weight": 1, + "auth": { "header": { "Authorization": "Bearer token" } }, + "options": { "model": "gpt-4o" }, + "examples": ["write code"] + } + ], + "ssl_verify": false + } + } + }]]) + ngx.status = code + ngx.say(body) + } + } +--- error_code: 400 +--- response_body_like: embeddings + + + +=== TEST 4: at most one catchall instance +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', ngx.HTTP_PUT, [[{ + "uri": "/anything", + "plugins": { + "ai-proxy-multi": { + "embeddings": { + "provider": "openai", "model": "text-embedding-3-small", + "auth": { "header": { "Authorization": "Bearer sk-emb" } } + }, + "balancer": { "algorithm": "semantic" }, + "instances": [ + { + "name": "a", "provider": "openai", "weight": 1, + "auth": { "header": { "Authorization": "Bearer token" } }, + "options": { "model": "gpt-4o-mini" }, + "catchall": true + }, + { + "name": "b", "provider": "openai", "weight": 1, + "auth": { "header": { "Authorization": "Bearer token" } }, + "options": { "model": "gpt-4o" }, + "catchall": true + } + ], + "ssl_verify": false + } + } + }]]) + ngx.status = code + ngx.say(body) + } + } +--- error_code: 400 +--- response_body_like: catchall + + + +=== TEST 5: azure-openai embeddings require an endpoint +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', ngx.HTTP_PUT, [[{ + "uri": "/anything", + "plugins": { + "ai-proxy-multi": { + "embeddings": { + "provider": "azure-openai", "model": "text-embedding-3-small", + "auth": { "header": { "api-key": "sk-emb" } } + }, + "balancer": { "algorithm": "semantic" }, + "instances": [ + { + "name": "a", "provider": "openai", "weight": 1, + "auth": { "header": { "Authorization": "Bearer token" } }, + "options": { "model": "gpt-4o" }, + "examples": ["write code"] + } + ], + "ssl_verify": false + } + } + }]]) + ngx.status = code + ngx.say(body) + } + } +--- error_code: 400 +--- response_body_like: endpoint + + + +=== TEST 6: embeddings.model is required +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', ngx.HTTP_PUT, [[{ + "uri": "/anything", + "plugins": { + "ai-proxy-multi": { + "embeddings": { + "provider": "openai", + "auth": { "header": { "Authorization": "Bearer sk-emb" } } + }, + "balancer": { "algorithm": "semantic" }, + "instances": [ + { + "name": "a", "provider": "openai", "weight": 1, + "auth": { "header": { "Authorization": "Bearer token" } }, + "options": { "model": "gpt-4o" }, + "examples": ["write code"] + } + ], + "ssl_verify": false + } + } + }]]) + ngx.status = code + ngx.say(body) + } + } +--- error_code: 400 +--- response_body_like: model + + + +=== TEST 7: catchall instance must not configure examples +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', ngx.HTTP_PUT, [[{ + "uri": "/anything", + "plugins": { + "ai-proxy-multi": { + "embeddings": { + "provider": "openai", "model": "text-embedding-3-small", + "auth": { "header": { "Authorization": "Bearer sk-emb" } } + }, + "balancer": { "algorithm": "semantic" }, + "instances": [ + { + "name": "a", "provider": "openai", "weight": 1, + "auth": { "header": { "Authorization": "Bearer token" } }, + "options": { "model": "gpt-4o" }, + "examples": ["write code"] + }, + { + "name": "fallback", "provider": "openai", "weight": 1, + "auth": { "header": { "Authorization": "Bearer token" } }, + "options": { "model": "gpt-4o-mini" }, + "catchall": true, + "examples": ["anything else"] + } + ], + "ssl_verify": false + } + } + }]]) + ngx.status = code + ngx.say(body) + } + } +--- error_code: 400 +--- response_body_like: must not configure + + + +=== TEST 8: azure-openai embeddings endpoint must include the deployment path +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', ngx.HTTP_PUT, [[{ + "uri": "/anything", + "plugins": { + "ai-proxy-multi": { + "embeddings": { + "provider": "azure-openai", "model": "text-embedding-3-small", + "endpoint": "https://my.openai.azure.com", + "auth": { "header": { "api-key": "sk-emb" } } + }, + "balancer": { "algorithm": "semantic" }, + "instances": [ + { + "name": "a", "provider": "openai", "weight": 1, + "auth": { "header": { "Authorization": "Bearer token" } }, + "options": { "model": "gpt-4o" }, + "examples": ["write code"] + } + ], + "ssl_verify": false + } + } + }]]) + ngx.status = code + ngx.say(body) + } + } +--- error_code: 400 +--- response_body_like: full deployment path + + + +=== TEST 9: embeddings.auth rejects ctx-dependent schemes +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', ngx.HTTP_PUT, [[{ + "uri": "/anything", + "plugins": { + "ai-proxy-multi": { + "embeddings": { + "provider": "openai", "model": "text-embedding-3-small", + "auth": { "gcp": { "service_account_json": "{}" } } + }, + "balancer": { "algorithm": "semantic" }, + "instances": [ + { + "name": "a", "provider": "openai", "weight": 1, + "auth": { "header": { "Authorization": "Bearer token" } }, + "options": { "model": "gpt-4o" }, + "examples": ["write code"] + } + ], + "ssl_verify": false + } + } + }]]) + ngx.status = code + ngx.say(body) + } + } +--- error_code: 400 +--- response_body_like: gcp + + + +=== TEST 10: embeddings.endpoint must carry a scheme and host +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', ngx.HTTP_PUT, [[{ + "uri": "/anything", + "plugins": { + "ai-proxy-multi": { + "embeddings": { + "provider": "openai", "model": "text-embedding-3-small", + "endpoint": "my.embeddings.host/v1/embeddings", + "auth": { "header": { "Authorization": "Bearer sk" } } + }, + "balancer": { "algorithm": "semantic" }, + "instances": [ + { + "name": "a", "provider": "openai", "weight": 1, + "auth": { "header": { "Authorization": "Bearer token" } }, + "options": { "model": "gpt-4o" }, + "examples": ["write code"] + } + ], + "ssl_verify": false + } + } + }]]) + ngx.status = code + ngx.say(body) + } + } +--- error_code: 400 +--- response_body_like: invalid `embeddings.endpoint` diff --git a/t/plugin/ai-proxy-semantic.t b/t/plugin/ai-proxy-semantic.t new file mode 100644 index 000000000000..5fe9e0fa65a7 --- /dev/null +++ b/t/plugin/ai-proxy-semantic.t @@ -0,0 +1,67 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +use t::APISIX 'no_plan'; + +log_level("info"); +repeat_each(1); +no_long_string(); +no_root_location(); +no_shuffle(); + +run_tests(); + +__DATA__ + +=== TEST 1: semantic math helpers (normalize / dot / cosine / max) +--- config + location /t { + content_by_lua_block { + local s = require("apisix.plugins.ai-proxy.semantic") + local function approx(a, b) return math.abs(a - b) < 1e-9 end + + -- normalize + local n = s.normalize({3, 4}) + assert(approx(n[1], 0.6) and approx(n[2], 0.8), "normalize unit vector") + local z = s.normalize({0, 0}) + assert(approx(z[1], 0) and approx(z[2], 0), "normalize zero vector") + + -- cosine + assert(approx(s.cosine({1, 0}, {1, 0}), 1.0), "cosine identical") + assert(approx(s.cosine({1, 0}, {0, 1}), 0.0), "cosine orthogonal") + assert(approx(s.cosine({1, 1}, {2, 2}), 1.0), "cosine same direction") + assert(approx(s.cosine({1, 0}, {-1, 0}), -1.0), "cosine opposite") + + -- dot on normalized == cosine + local a = s.normalize({1, 2, 3}) + assert(approx(s.dot(a, a), 1.0), "dot of normalized identical") + + -- max: an instance scores by its best-matching example, so one + -- dead-on example is not diluted by the instance's broader ones + assert(approx(s.max({0.7, 0.85}), 0.85), "max picks the best") + assert(approx(s.max({1.0, 0.5, 0.4}), 1.0), "max is not diluted") + assert(approx(s.max({0.5}), 0.5), "max of a single score") + assert(s.max({}) == nil, "max of empty is nil") + + ngx.say("passed") + } + } +--- request +GET /t +--- response_body +passed +--- no_error_log +[error] From b7e61ca84e9f912bd586319c0bedd1957acd171c Mon Sep 17 00:00:00 2001 From: AlinsRan Date: Wed, 15 Jul 2026 13:16:56 +0800 Subject: [PATCH 2/8] fix(ai-proxy-multi): read semantic prompt via protocol adapter extract_last_user_message read body.messages directly, so the semantic balancer silently fell back for every non-Chat client protocol: OpenAI Responses carries the prompt in body.input, Anthropic/Bedrock in structured content blocks. Read the last user turn through ai-protocols.get_messages() so all supported protocols are normalized. Add Responses and Anthropic routing regression tests. --- apisix/plugins/ai-proxy-multi.lua | 25 +++-- t/plugin/ai-proxy-semantic-routing.t | 146 +++++++++++++++++++++++++++ 2 files changed, 163 insertions(+), 8 deletions(-) diff --git a/apisix/plugins/ai-proxy-multi.lua b/apisix/plugins/ai-proxy-multi.lua index ee233254d3fd..6d0408c1fc56 100644 --- a/apisix/plugins/ai-proxy-multi.lua +++ b/apisix/plugins/ai-proxy-multi.lua @@ -44,6 +44,7 @@ local url = require("socket.url") local priority_balancer = require("apisix.balancer.priority") local semantic = require("apisix.plugins.ai-proxy.semantic") local embedding = require("apisix.plugins.ai-proxy.embedding") +local ai_protocols = require("apisix.plugins.ai-protocols") local endpoint_regex = "^(https?)://([^:/]+):?(%d*)/?.*$" local pickers = {} @@ -676,20 +677,28 @@ local function pick_target(ctx, conf, ups_tab) end -local function extract_last_user_message() +local function extract_last_user_message(ctx) local body = core.request.get_json_request_body_table() - if not body or type(body.messages) ~= "table" then + if not body then return nil end - for i = #body.messages, 1, -1 do - local m = body.messages[i] + -- Normalize through the protocol adapter instead of reading body.messages + -- directly, so the prompt is found for every supported client protocol: + -- OpenAI Responses carries it in body.input, Anthropic/Bedrock in structured + -- content blocks. get_messages returns canonical {role, content} entries. + local messages = ai_protocols.get_messages(body, ctx) + for i = #messages, 1, -1 do + local m = messages[i] if type(m) == "table" and m.role == "user" then local content = m.content if type(content) == "string" then - return content + if content ~= "" then + return content + end elseif type(content) == "table" then - -- multimodal content: concatenate the text parts so routing - -- still works for {type=text|image_url,...} arrays. + -- Some adapters return multimodal content unflattened + -- ({type=text|image_url,...}); concatenate the text parts so + -- routing still works. local parts = {} for _, p in ipairs(content) do if type(p) == "table" and p.type == "text" @@ -762,7 +771,7 @@ local function pick_semantic_instance(ctx, conf) return semantic_fallback(conf) end - local prompt = extract_last_user_message() + local prompt = extract_last_user_message(ctx) if not prompt then core.log.warn("semantic routing: no user message found, falling back") return semantic_fallback(conf) diff --git a/t/plugin/ai-proxy-semantic-routing.t b/t/plugin/ai-proxy-semantic-routing.t index 49df719b7207..45785bfb8ad8 100644 --- a/t/plugin/ai-proxy-semantic-routing.t +++ b/t/plugin/ai-proxy-semantic-routing.t @@ -729,3 +729,149 @@ Content-Type: application/json model-fallback --- no_error_log [error] + + + +=== TEST 22: configure a semantic route on the Responses API path +--- config + location /t { + content_by_lua_block { + local core = require("apisix.core") + local t = require("lib.test_admin").test + local function inst(name, model, examples, catchall) + local i = { + name = name, provider = "openai", weight = 1, + auth = { header = { Authorization = "Bearer token" } }, + options = { model = model }, + override = { + endpoint = "http://127.0.0.1:6798/v1/chat/completions", + }, + } + if examples then i.examples = examples end + if catchall then i.catchall = true end + return i + end + local conf = { + uri = "/llm/v1/responses", + plugins = { + ["ai-proxy-multi"] = { + embeddings = { + provider = "openai", + model = "text-embedding-3-small", + endpoint = "http://127.0.0.1:6797/v1/embeddings", + auth = { header = { Authorization = "Bearer token" } }, + ssl_verify = false, + }, + balancer = { + algorithm = "semantic", threshold = 0.9, + expose_scores = true, + }, + instances = { + inst("code", "model-code", {"write python code"}), + inst("cheap", "model-cheap", {"translate this text"}), + inst("default", "model-fallback", nil, true), + }, + ssl_verify = false, + }, + }, + } + local code, body = t('/apisix/admin/routes/2', ngx.HTTP_PUT, + core.json.encode(conf)) + if code >= 300 then + ngx.status = code + ngx.say(body) + return + end + ngx.say("passed") + } + } +--- response_body +passed +--- no_error_log +[error] + + + +=== TEST 23: a Responses request (body.input) is routed, not fallen back +--- request +POST /llm/v1/responses +{"model":"auto","input":"help me debug this python code"} +--- more_headers +Content-Type: application/json +--- response_headers_like +X-AI-Semantic-Route: code +X-AI-Semantic-Scores: code:1\.\d+,cheap:0\.\d+ +--- no_error_log +[error] + + + +=== TEST 24: configure a semantic route on the Anthropic Messages path +--- config + location /t { + content_by_lua_block { + local core = require("apisix.core") + local t = require("lib.test_admin").test + local function inst(name, model, examples, catchall) + local i = { + name = name, provider = "openai", weight = 1, + auth = { header = { Authorization = "Bearer token" } }, + options = { model = model }, + override = { + endpoint = "http://127.0.0.1:6798/v1/chat/completions", + }, + } + if examples then i.examples = examples end + if catchall then i.catchall = true end + return i + end + local conf = { + uri = "/llm/v1/messages", + plugins = { + ["ai-proxy-multi"] = { + embeddings = { + provider = "openai", + model = "text-embedding-3-small", + endpoint = "http://127.0.0.1:6797/v1/embeddings", + auth = { header = { Authorization = "Bearer token" } }, + ssl_verify = false, + }, + balancer = { + algorithm = "semantic", threshold = 0.9, + expose_scores = true, + }, + instances = { + inst("code", "model-code", {"write python code"}), + inst("cheap", "model-cheap", {"translate this text"}), + inst("default", "model-fallback", nil, true), + }, + ssl_verify = false, + }, + }, + } + local code, body = t('/apisix/admin/routes/3', ngx.HTTP_PUT, + core.json.encode(conf)) + if code >= 300 then + ngx.status = code + ngx.say(body) + return + end + ngx.say("passed") + } + } +--- response_body +passed +--- no_error_log +[error] + + + +=== TEST 25: an Anthropic request (body.system + content) is routed by the last user turn +--- request +POST /llm/v1/messages +{"model":"auto","system":"you are a helpful assistant","messages":[{"role":"user","content":"help me debug this python code"}]} +--- more_headers +Content-Type: application/json +--- response_headers_like +X-AI-Semantic-Route: code +X-AI-Semantic-Scores: code:1\.\d+,cheap:0\.\d+ From 56a9af5e7b7f3c1743f682e687b1bbd78be9e506 Mon Sep 17 00:00:00 2001 From: AlinsRan Date: Wed, 15 Jul 2026 17:45:54 +0800 Subject: [PATCH 3/8] refactor(ai-proxy-multi): group semantic options under semantic_opts Address review feedback on the semantic balancer config surface: - Group all semantic-only options under a single `semantic_opts` object (embeddings, threshold, fallback, debugging) instead of scattering them across balancer/instances/top-level, which cut the cross-field schema validation. - Replace the per-instance `catchall` boolean with `semantic_opts.fallback`, which names the fallback instance. It needs no `examples` and a real ranked instance can also serve as the fallback; drops the '<=1 catchall' and 'catchall must not set examples' checks. - Rename `expose_scores` to `debugging`: the name is not tied to scores, so future debug signals can reuse it. Also decouple the HTTP transport from ctx: construct_forward_headers now takes an explicit client_headers table (nil for internal requests) instead of ctx + a skip flag, so the transport carries no downstream-request coupling and the caller decides whether client headers are forwarded. Docs (en + zh) and tests updated. --- apisix/plugins/ai-providers/base.lua | 13 +- apisix/plugins/ai-proxy-multi.lua | 82 +++++----- apisix/plugins/ai-proxy/schema.lua | 78 +++++---- apisix/plugins/ai-transport/http.lua | 20 +-- docs/en/latest/plugins/ai-proxy-multi.md | 75 ++++----- docs/zh/latest/plugins/ai-proxy-multi.md | 75 ++++----- t/plugin/ai-proxy-semantic-routing.t | 192 +++++++++++++---------- t/plugin/ai-proxy-semantic-schema.t | 136 ++++++++-------- 8 files changed, 367 insertions(+), 304 deletions(-) diff --git a/apisix/plugins/ai-providers/base.lua b/apisix/plugins/ai-providers/base.lua index 6b8c8e35c9c1..7c1416c5b17d 100644 --- a/apisix/plugins/ai-providers/base.lua +++ b/apisix/plugins/ai-providers/base.lua @@ -188,8 +188,17 @@ function _M.build_request(self, ctx, conf, request_body, opts) .. "includes a path", 400 end - local headers = transport_http.construct_forward_headers(auth.header or {}, ctx, - opts.skip_client_headers) + -- Forward the downstream request's headers only on the proxy path. A + -- self-contained internal request (skip_client_headers, e.g. the embedding + -- sidecar call) carries its own credentials and must not leak the client's + -- Authorization/Cookie to a third-party endpoint. Resolving them here keeps + -- the transport free of any ctx dependency. + local client_headers + if not opts.skip_client_headers then + client_headers = core.request.headers(ctx) + end + local headers = transport_http.construct_forward_headers(auth.header or {}, + client_headers) if opts.host_header then headers["Host"] = opts.host_header end diff --git a/apisix/plugins/ai-proxy-multi.lua b/apisix/plugins/ai-proxy-multi.lua index 6d0408c1fc56..c0955ae1940d 100644 --- a/apisix/plugins/ai-proxy-multi.lua +++ b/apisix/plugins/ai-proxy-multi.lua @@ -173,46 +173,48 @@ function _M.check_schema(conf) end if algo == "semantic" then - if not conf.embeddings then - return false, "must configure `embeddings` when balancer algorithm is semantic" + local semantic_opts = conf.semantic_opts + if not semantic_opts or not semantic_opts.embeddings then + return false, "must configure `semantic_opts.embeddings` when balancer " .. + "algorithm is semantic" end + local embeddings = semantic_opts.embeddings -- Same scheme+host check the instance endpoints get: without it an -- endpoint like "host/path" parses to a nil host and every request would -- silently fail open, i.e. the route would never work with no signal. - local eendpoint = conf.embeddings.endpoint + local eendpoint = embeddings.endpoint if eendpoint then local scheme, host = eendpoint:match(endpoint_regex) if not scheme or not host then - return false, "invalid `embeddings.endpoint`" + return false, "invalid `semantic_opts.embeddings.endpoint`" end end - if conf.embeddings.provider == "azure-openai" then + if embeddings.provider == "azure-openai" then -- Azure carries the deployment in the URL and declares no default host -- or path, so the endpoint must be the full embeddings URL. Without a - -- path every request would silently fail open to the catchall. + -- path every request would silently fail open to the fallback. if not eendpoint then - return false, "must configure `embeddings.endpoint` when embeddings " .. - "provider is azure-openai" + return false, "must configure `semantic_opts.embeddings.endpoint` " .. + "when embeddings provider is azure-openai" end local parsed = url.parse(eendpoint) local epath = parsed and parsed.path if not epath or epath == "" or epath == "/" then - return false, "`embeddings.endpoint` for azure-openai must include the " .. - "full deployment path, e.g. https://{resource}.openai.azure.com" .. + return false, "`semantic_opts.embeddings.endpoint` for azure-openai " .. + "must include the full deployment path, e.g. " .. + "https://{resource}.openai.azure.com" .. "/openai/deployments/{deployment}/embeddings?api-version=..." end end - local catchall_count = 0 + -- The `fallback` instance, if named, is the only one exempt from the + -- examples requirement: it is reached by fallback, not by ranking. It must + -- name an instance that actually exists. + local fallback = semantic_opts.fallback + local fallback_found = false for _, instance in ipairs(conf.instances) do - if instance.catchall then - catchall_count = catchall_count + 1 - -- The catchall is the fallback target, never a ranking candidate. - -- Allowing examples on it would let it outrank a real match. - if instance.examples then - return false, "instance '" .. (instance.name or "?") .. - "': `catchall` instance must not configure `examples`; it is " .. - "the fallback target and does not take part in ranking" - end + local is_fallback = fallback and instance.name == fallback + if is_fallback then + fallback_found = true else local has_example = false if instance.examples then @@ -226,12 +228,13 @@ function _M.check_schema(conf) if not has_example then return false, "instance '" .. (instance.name or "?") .. "': must configure non-empty `examples` for the semantic " .. - "algorithm unless `catchall` is set" + "algorithm unless it is named by `semantic_opts.fallback`" end end end - if catchall_count > 1 then - return false, "at most one instance may be marked `catchall`" + if fallback and not fallback_found then + return false, "`semantic_opts.fallback` names unknown instance '" .. + fallback .. "'" end end @@ -731,7 +734,7 @@ local function build_instance_vectors(conf) end end - local vecs, err = embedding.fetch(conf.embeddings, texts) + local vecs, err = embedding.fetch(conf.semantic_opts.embeddings, texts) if not vecs then error("failed to fetch reference embeddings: " .. tostring(err)) end @@ -748,12 +751,16 @@ local function build_instance_vectors(conf) end --- Guaranteed fallback: catchall instance if configured, else the first --- instance. Never fails, so a request always has a target. +-- Guaranteed fallback: the instance named by semantic_opts.fallback if +-- configured, else the first instance. Never fails, so a request always has a +-- target. local function semantic_fallback(conf) - for _, inst in ipairs(conf.instances) do - if inst.catchall then - return inst.name, inst + local fallback = conf.semantic_opts and conf.semantic_opts.fallback + if fallback then + for _, inst in ipairs(conf.instances) do + if inst.name == fallback then + return inst.name, inst + end end end local inst = conf.instances[1] @@ -785,7 +792,8 @@ local function pick_semantic_instance(ctx, conf) -- pcall, like the reference path above: the embedding response is -- provider-controlled, so a raise here must fall back rather than 500. - local fetched, qvecs, err = pcall(embedding.fetch, conf.embeddings, { prompt }) + local fetched, qvecs, err = pcall(embedding.fetch, conf.semantic_opts.embeddings, + { prompt }) if not fetched then core.log.warn("semantic routing: query embedding error: ", qvecs, ", falling back") return semantic_fallback(conf) @@ -820,8 +828,8 @@ local function pick_semantic_instance(ctx, conf) end core.table.sort(ranked, function(a, b) return a.score > b.score end) - local expose_scores = conf.balancer.expose_scores - if expose_scores then + local debugging = conf.semantic_opts.debugging + if debugging then local parts = {} for _, c in ipairs(ranked) do parts[#parts + 1] = c.name .. ":" .. string.format("%.4f", c.score) @@ -830,12 +838,12 @@ local function pick_semantic_instance(ctx, conf) end -- Highest score first; pick the first instance that clears its own threshold - -- (per-instance override, else the global balancer.threshold). + -- (per-instance override, else the global semantic_opts.threshold). for _, cand in ipairs(ranked) do local inst = get_instance_conf(conf.instances, cand.name) - local thr = inst.threshold or conf.balancer.threshold or 0 + local thr = inst.threshold or conf.semantic_opts.threshold or 0 if cand.score >= thr then - if expose_scores then + if debugging then core.response.set_header("X-AI-Semantic-Route", cand.name) end core.log.info("semantic routing picked instance: ", cand.name, @@ -844,11 +852,11 @@ local function pick_semantic_instance(ctx, conf) end end - if expose_scores then + if debugging then core.response.set_header("X-AI-Semantic-Route", "fallback") end -- Only on the fallback path: surface why nothing matched, without requiring - -- expose_scores. Cheap, because this runs once per unmatched request. + -- debugging. Cheap, because this runs once per unmatched request. local unmatched = {} for _, c in ipairs(ranked) do unmatched[#unmatched + 1] = c.name .. ":" .. string.format("%.4f", c.score) diff --git a/apisix/plugins/ai-proxy/schema.lua b/apisix/plugins/ai-proxy/schema.lua index cdb9a544d97c..ad1a0518d80e 100644 --- a/apisix/plugins/ai-proxy/schema.lua +++ b/apisix/plugins/ai-proxy/schema.lua @@ -211,20 +211,14 @@ local ai_instance_schema = { description = "Example utterances representing this instance's " .. "intent; each is embedded into its own reference vector " .. "for the semantic algorithm. Required for every instance " - .. "except the catchall, which must not set them.", + .. "except the one named by semantic_opts.fallback.", }, threshold = { type = "number", minimum = -1, maximum = 1, description = "Per-instance minimum cosine similarity for the " - .. "semantic algorithm; overrides balancer.threshold.", - }, - catchall = { - type = "boolean", - description = "Marks this instance as the semantic fallback, " - .. "used when no instance clears its threshold. It takes no " - .. "part in ranking, so it must not configure examples.", + .. "semantic algorithm; overrides semantic_opts.threshold.", }, }, required = {"name", "provider", "auth", "weight"}, @@ -273,6 +267,43 @@ local embeddings_schema = { required = { "provider", "model", "auth" }, } +-- All options specific to the semantic balancer live here, so they are grouped +-- in one place instead of being scattered across balancer/instances/top-level. +local semantic_opts_schema = { + type = "object", + properties = { + embeddings = embeddings_schema, + threshold = { + type = "number", + minimum = -1, + maximum = 1, + default = 0, + description = "Global minimum cosine similarity for the semantic " + .. "algorithm. When no instance clears its threshold the request " + .. "falls back to the `fallback` instance, or the first instance " + .. "if none is named. The default of 0 admits essentially any " + .. "prompt, so the fallback only ever runs if a threshold above 0 " + .. "is set.", + }, + fallback = { + type = "string", + minLength = 1, + description = "Name of the instance to route to when no instance " + .. "clears its threshold or the embedding request fails. Unlike a " + .. "ranked instance it needs no `examples`. Defaults to the first " + .. "instance when unset.", + }, + debugging = { + type = "boolean", + default = false, + description = "When true, the semantic algorithm exposes per-instance " + .. "scores and the routing decision via X-AI-Semantic-* response " + .. "headers, for debugging.", + }, + }, + required = { "embeddings" }, +} + local logging_schema = { type = "object", properties = { @@ -374,28 +405,9 @@ _M.ai_proxy_multi_schema = { .. "participate in health checks or fallback_strategy / " .. "retry — an upstream failure on the chosen instance is " .. "returned to the client. It only falls back (to the " - .. "catchall, else the first instance) when no instance " - .. "clears its threshold or embedding fails.", - }, - threshold = { - type = "number", - minimum = -1, - maximum = 1, - default = 0, - description = "Global minimum cosine similarity for the " - .. "semantic algorithm. When no instance clears its " - .. "threshold the request falls back to the catchall " - .. "instance, or the first instance if none is set. " - .. "The default of 0 admits essentially any prompt, so " - .. "the catchall only ever runs if a threshold above 0 " - .. "is set.", - }, - expose_scores = { - type = "boolean", - default = false, - description = "When true, the semantic algorithm exposes " - .. "per-instance scores and the routing decision via " - .. "X-AI-Semantic-* response headers, for debugging.", + .. "semantic_opts.fallback, else the first instance) when " + .. "no instance clears its threshold or embedding fails. " + .. "Configure it under `semantic_opts`.", }, hash_on = { type = "string", @@ -416,7 +428,7 @@ _M.ai_proxy_multi_schema = { default = { algorithm = "roundrobin" } }, instances = ai_instance_schema, - embeddings = embeddings_schema, + semantic_opts = semantic_opts_schema, logging = logging_schema, fallback_strategy = { anyOf = { @@ -512,8 +524,8 @@ _M.ai_proxy_multi_schema = { "instances.auth.gcp.service_account_json", "instances.auth.aws.secret_access_key", "instances.auth.aws.session_token", - "embeddings.auth.header", - "embeddings.auth.query", + "semantic_opts.embeddings.auth.header", + "semantic_opts.embeddings.auth.query", }, } diff --git a/apisix/plugins/ai-transport/http.lua b/apisix/plugins/ai-transport/http.lua index ee142061a32b..4cbc61b84a1f 100644 --- a/apisix/plugins/ai-transport/http.lua +++ b/apisix/plugins/ai-transport/http.lua @@ -43,13 +43,15 @@ end --- Build forwarded headers from client request + extra headers. --- Copies client headers, merges ext_opts_headers (lowercased), +-- Copies `client_headers`, merges ext_opts_headers (lowercased), -- forces Content-Type to application/json, removes host/content-length. --- When skip_client_headers is true the client's request headers are not --- forwarded. Use it for self-contained sidecar calls (e.g. embeddings), which --- carry their own credentials and must not leak the client's Authorization, --- Cookie or other headers to a third-party endpoint. -function _M.construct_forward_headers(ext_opts_headers, ctx, skip_client_headers) +-- `client_headers` is the downstream request's headers to forward (proxy path), +-- or nil for a self-contained internal request (e.g. the embedding sidecar call), +-- which carries its own credentials and must not leak the client's Authorization, +-- Cookie or other headers to a third-party endpoint. Passing them in explicitly +-- keeps the transport free of any `ctx` dependency: the caller — which legitimately +-- holds ctx — decides whether client headers are forwarded. +function _M.construct_forward_headers(ext_opts_headers, client_headers) local blacklist = { "host", "content-length", @@ -57,10 +59,8 @@ function _M.construct_forward_headers(ext_opts_headers, ctx, skip_client_headers } local headers = {} - if not skip_client_headers then - for k, v in pairs(core.request.headers(ctx) or {}) do - headers[str_lower(k)] = v - end + for k, v in pairs(client_headers or {}) do + headers[str_lower(k)] = v end for k, v in pairs(ext_opts_headers or {}) do headers[str_lower(k)] = v diff --git a/docs/en/latest/plugins/ai-proxy-multi.md b/docs/en/latest/plugins/ai-proxy-multi.md index f76007dc002c..c43035371a8f 100644 --- a/docs/en/latest/plugins/ai-proxy-multi.md +++ b/docs/en/latest/plugins/ai-proxy-multi.md @@ -71,9 +71,7 @@ When an instance's `provider` is set to `bedrock`, the Plugin expects requests i | max_retries | integer | False | | greater or equal to 0 | Maximum number of fallback retries after the initial request fails. Bounds how many additional instances a single request tries, so it does not exhaust every configured instance. Only takes effect together with `fallback_strategy`. When unset, the Plugin retries until an instance succeeds or all are tried. | | retry_on_failure_within_ms | integer | False | | greater or equal to 1 | Only fall back to another instance when the upstream fails within this many milliseconds. Fast failures (such as connection errors or quick `429`/`5xx`) are retried, while a slow failure that takes longer than this is returned to the client directly to avoid doubling the wait time. Only takes effect together with `fallback_strategy`. When unset, the Plugin retries regardless of how long the failed attempt took. | | balancer | object | False | | | Load balancing configurations. | -| balancer.algorithm | string | False | roundrobin | [roundrobin, chash, semantic] | Load balancing algorithm. When set to `roundrobin`, weighted round robin algorithm is used. When set to `chash`, consistent hashing algorithm is used. When set to `semantic`, the Plugin picks an instance by the semantic similarity between the request prompt and each instance's `examples`. Note that `semantic` does not participate in health checks or `fallback_strategy` / retry — an upstream failure on the chosen instance is returned to the client; it only falls back (to the `catchall` instance, else the first instance) when no instance clears its threshold or embedding fails. | -| balancer.threshold | number | False | 0 | -1 to 1 | Used when `algorithm` is `semantic`. Global minimum cosine similarity an instance must reach to be selected. **The default of 0 admits essentially any prompt** (real embeddings are rarely dissimilar enough to score below 0), so the `catchall` only ever runs if you raise this above 0. When no instance clears its threshold, the request falls back to the `catchall` instance, or the first instance if none is set. | -| balancer.expose_scores | boolean | False | false | | Used when `algorithm` is `semantic`. If true, expose per-instance scores and the routing decision via `X-AI-Semantic-Scores` and `X-AI-Semantic-Route` response headers, for debugging. | +| balancer.algorithm | string | False | roundrobin | [roundrobin, chash, semantic] | Load balancing algorithm. When set to `roundrobin`, weighted round robin algorithm is used. When set to `chash`, consistent hashing algorithm is used. When set to `semantic`, the Plugin picks an instance by the semantic similarity between the request prompt and each instance's `examples`, and its options are configured under `semantic_opts`. Note that `semantic` does not participate in health checks or `fallback_strategy` / retry — an upstream failure on the chosen instance is returned to the client; it only falls back (to the `semantic_opts.fallback` instance, else the first instance) when no instance clears its threshold or embedding fails. | | balancer.hash_on | string | False | | [vars, headers, cookie, consumer, vars_combinations] | Used when `type` is `chash`. Support hashing on [NGINX variables](https://nginx.org/en/docs/varindex.html), headers, cookie, consumer, or a combination of [NGINX variables](https://nginx.org/en/docs/varindex.html). | | balancer.key | string | False | | | Used when `type` is `chash`. When `hash_on` is set to `header` or `cookie`, `key` is required. When `hash_on` is set to `consumer`, `key` is not required as the consumer name will be used as the key automatically. | | instances | array[object] | True | | | LLM instance configurations. | @@ -97,9 +95,8 @@ When an instance's `provider` is set to `bedrock`, the Plugin expects requests i | instances.auth.aws.session_token | string | False | | minLength = 1 | AWS session token for temporary credentials (e.g., from STS assume-role). Encrypted at rest. | | instances.options | object | False | | | Model configurations. In addition to `model`, you can configure additional parameters and they will be forwarded to the upstream LLM service in the request body. For instance, if you are working with OpenAI, DeepSeek, or AIMLAPI, you can configure additional parameters such as `max_tokens`, `temperature`, `top_p`, and `stream`. See your LLM provider's API documentation for more available options. | | instances.options.model | string | False | | | Name of the LLM model, such as `gpt-4` or `gpt-3.5`. See your LLM provider's API documentation for more available models. For Bedrock, this can be a foundation model ID (e.g., `anthropic.claude-3-5-sonnet-20240620-v1:0`), a cross-region inference profile ID (e.g., `us.anthropic.claude-3-5-sonnet-20240620-v1:0`), or an application inference profile ARN (e.g., `arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/abc123`). | -| instances.examples | array[string] | False | | 1 to 64 items | Used when `algorithm` is `semantic`. Example utterances representing this instance's intent; each is embedded into its own reference vector. Required for every instance except the `catchall`, which must not set them. | -| instances.threshold | number | False | | -1 to 1 | Used when `algorithm` is `semantic`. Per-instance minimum cosine similarity; overrides `balancer.threshold`. | -| instances.catchall | boolean | False | | | Used when `algorithm` is `semantic`. Marks this instance as the semantic fallback, used when no instance clears its threshold. It takes no part in ranking, so it must not configure `examples`. At most one instance may be marked `catchall`. | +| instances.examples | array[string] | False | | 1 to 64 items | Used when `algorithm` is `semantic`. Example utterances representing this instance's intent; each is embedded into its own reference vector. Required for every instance except the one named by `semantic_opts.fallback`. | +| instances.threshold | number | False | | -1 to 1 | Used when `algorithm` is `semantic`. Per-instance minimum cosine similarity; overrides `semantic_opts.threshold`. | | logging | object | False | | | Logging configurations. | | logging.summaries | boolean | False | false | | If true, log request LLM model, duration, request, and response tokens. | | logging.payloads | boolean | False | false | | If true, log request and response payload. | @@ -127,15 +124,19 @@ When an instance's `provider` is set to `bedrock`, the Plugin expects requests i | instances.checks.active.unhealthy.http_statuses | array[integer] | False | [429,404,500,501,502,503,504,505] | status code between 200 and 599 inclusive | An array of HTTP status codes that defines an unhealthy node. | | instances.checks.active.unhealthy.http_failures | integer | False | 5 | between 1 and 254 inclusive | Number of HTTP failures to define an unhealthy node. | | instances.checks.active.unhealthy.timeout | integer | False | 3 | between 1 and 254 inclusive | Number of probe timeouts to define an unhealthy node. | -| embeddings | object | False | | | Embedding service configuration. Required when `balancer.algorithm` is `semantic`. | -| embeddings.provider | string | True | | [openai, azure-openai] | Embedding provider used by the semantic algorithm. | -| embeddings.model | string | True | | | Embedding model name, e.g. `text-embedding-3-small`. | -| embeddings.endpoint | string | False | | | Embedding API endpoint. Optional for `openai` (defaults to the public API). Required for `azure-openai`, where it must be the **full** URL including the deployment path and API version, e.g. `https://{resource}.openai.azure.com/openai/deployments/{deployment}/embeddings?api-version=2024-02-01`. | -| embeddings.auth | object | True | | | Authentication configurations for the embedding service. | -| embeddings.auth.header | object | False | | | Authentication headers. Encrypted at rest. | -| embeddings.auth.query | object | False | | | Authentication query parameters. Encrypted at rest. | -| embeddings.timeout | integer | False | 10000 | minimum = 1 | Embedding request timeout in milliseconds. | -| embeddings.ssl_verify | boolean | False | true | | If true, verify the embedding service's certificate. | +| semantic_opts | object | False | | | Options for the `semantic` balancer, grouped in one place. Required when `balancer.algorithm` is `semantic`. | +| semantic_opts.threshold | number | False | 0 | -1 to 1 | Global minimum cosine similarity an instance must reach to be selected. **The default of 0 admits essentially any prompt** (real embeddings are rarely dissimilar enough to score below 0), so the fallback only ever runs if you raise this above 0. When no instance clears its threshold, the request falls back to the `fallback` instance, or the first instance if none is named. | +| semantic_opts.fallback | string | False | | | Name of the instance to route to when no instance clears its threshold or the embedding request fails. Unlike a ranked instance it needs no `examples`. Defaults to the first instance when unset. | +| semantic_opts.debugging | boolean | False | false | | If true, expose per-instance scores and the routing decision via `X-AI-Semantic-Scores` and `X-AI-Semantic-Route` response headers, for debugging. | +| semantic_opts.embeddings | object | True | | | Embedding service configuration used to score prompts against each instance's `examples`. | +| semantic_opts.embeddings.provider | string | True | | [openai, azure-openai] | Embedding provider used by the semantic algorithm. | +| semantic_opts.embeddings.model | string | True | | | Embedding model name, e.g. `text-embedding-3-small`. | +| semantic_opts.embeddings.endpoint | string | False | | | Embedding API endpoint. Optional for `openai` (defaults to the public API). Required for `azure-openai`, where it must be the **full** URL including the deployment path and API version, e.g. `https://{resource}.openai.azure.com/openai/deployments/{deployment}/embeddings?api-version=2024-02-01`. | +| semantic_opts.embeddings.auth | object | True | | | Authentication configurations for the embedding service. | +| semantic_opts.embeddings.auth.header | object | False | | | Authentication headers. Encrypted at rest. | +| semantic_opts.embeddings.auth.query | object | False | | | Authentication query parameters. Encrypted at rest. | +| semantic_opts.embeddings.timeout | integer | False | 10000 | minimum = 1 | Embedding request timeout in milliseconds. | +| semantic_opts.embeddings.ssl_verify | boolean | False | true | | If true, verify the embedding service's certificate. | | timeout | integer | False | 30000 | greater than or equal to 1 | Request timeout in milliseconds when requesting the LLM service. Applied per socket operation (connect / send / read block); does not cap the total duration of a streaming response. | | max_req_body_size | integer | False | 67108864 | greater than or equal to 1 | Maximum request body size in bytes that the plugin reads into memory. Requests whose body exceeds this limit are rejected with `413`. Prevents unbounded memory buffering of large request bodies. | | max_stream_duration_ms | integer | False | | greater than or equal to 1 | Maximum wall-clock duration (in milliseconds) for a streaming AI response. If the upstream keeps sending data past this deadline, the gateway closes the connection. Unset means no cap. Use this to protect the gateway from upstream bugs that produce tokens indefinitely. When the limit is hit mid-stream, the downstream SSE stream is truncated (no protocol-specific terminator such as `[DONE]`, `message_stop`, or `response.completed`); well-behaved clients should treat a missing terminator as an incomplete response. | @@ -2991,9 +2992,9 @@ In the Kafka topic, you should also see a log entry corresponding to the request ### Route by Semantic Similarity -The following example demonstrates how you can use the `semantic` algorithm to pick an instance by the meaning of the request prompt, rather than by weight or hash. Each non-`catchall` instance declares `examples` that describe the kind of prompt it should handle; the Plugin embeds those examples and the incoming prompt with the configured embedding model, and forwards the request to the instance whose examples are most similar. +The following example demonstrates how you can use the `semantic` algorithm to pick an instance by the meaning of the request prompt, rather than by weight or hash. Each ranked instance declares `examples` that describe the kind of prompt it should handle; the Plugin embeds those examples and the incoming prompt with the configured embedding model, and forwards the request to the instance whose examples are most similar. -Create a Route as such and update the LLM providers, embedding model, and API keys accordingly. The `code` instance handles programming prompts, the `translate` instance handles translation prompts, and the `default` instance is the `catchall` used when no instance clears the similarity threshold: +Create a Route as such and update the LLM providers, embedding model, and API keys accordingly. The `code` instance handles programming prompts, the `translate` instance handles translation prompts, and the `default` instance is named as the `semantic_opts.fallback`, used when no instance clears the similarity threshold: ```shell curl "http://127.0.0.1:9180/apisix/admin/routes" -X PUT \ @@ -3004,18 +3005,21 @@ curl "http://127.0.0.1:9180/apisix/admin/routes" -X PUT \ "methods": ["POST"], "plugins": { "ai-proxy-multi": { - "embeddings": { - "provider": "openai", - "model": "text-embedding-3-small", - "auth": { - "header": { - "Authorization": "Bearer '"$YOUR_EMBEDDING_API_KEY"'" - } - } - }, "balancer": { - "algorithm": "semantic", - "threshold": 0.5 + "algorithm": "semantic" + }, + "semantic_opts": { + "embeddings": { + "provider": "openai", + "model": "text-embedding-3-small", + "auth": { + "header": { + "Authorization": "Bearer '"$YOUR_EMBEDDING_API_KEY"'" + } + } + }, + "threshold": 0.5, + "fallback": "default" }, "instances": [ { @@ -3062,8 +3066,7 @@ curl "http://127.0.0.1:9180/apisix/admin/routes" -X PUT \ }, "options": { "model": "gpt-4o-mini" - }, - "catchall": true + } } ] } @@ -3084,11 +3087,11 @@ curl "http://127.0.0.1:9080/anything" -X POST \ }' ``` -The request should be routed to the `code` instance and served by `gpt-4o`. A prompt unrelated to any instance's `examples` (for example, `what is the weather today`) clears no threshold and is routed to the `default` `catchall` instance instead. +The request should be routed to the `code` instance and served by `gpt-4o`. A prompt unrelated to any instance's `examples` (for example, `what is the weather today`) clears no threshold and is routed to the `default` fallback instance instead. #### Debugging the routing decision -Set `balancer.expose_scores` to `true` to have the Plugin report how each instance scored and which one it picked: +Set `semantic_opts.debugging` to `true` to have the Plugin report how each instance scored and which one it picked: ```shell curl -i "http://127.0.0.1:9080/anything" -X POST \ @@ -3102,7 +3105,7 @@ X-AI-Semantic-Route: code X-AI-Semantic-Scores: code:0.8213,translate:0.1904 ``` -`X-AI-Semantic-Scores` lists every instance that has `examples`, highest score first, so you can see not only the winner but how close the runners-up came. Instances without `examples` — that is, the `catchall` — carry no similarity score and therefore do not appear. +`X-AI-Semantic-Scores` lists every instance that has `examples`, highest score first, so you can see not only the winner but how close the runners-up came. Instances without `examples` — such as a pure fallback — carry no similarity score and therefore do not appear. When no instance clears its threshold, the pick becomes `fallback` while the scores still show how close each one got: @@ -3111,7 +3114,7 @@ X-AI-Semantic-Route: fallback X-AI-Semantic-Scores: code:0.3057,translate:0.2884 ``` -Use this to calibrate `threshold`: pick a value between the scores of the prompts you want matched and the scores of the prompts you want to reach the `catchall`. +Use this to calibrate `threshold`: pick a value between the scores of the prompts you want matched and the scores of the prompts you want to reach the fallback. The same information is available without exposing it to clients: whenever no instance clears its threshold, the Plugin logs the full score list at `warn` level. @@ -3119,6 +3122,6 @@ The same information is available without exposing it to clients: whenever no in semantic routing: no instance cleared threshold (scores: code:0.3057,translate:0.2884), falling back ``` -If the headers are missing entirely while `expose_scores` is on, the embedding step itself failed and the request was served by the `catchall` before any score was computed; the reason is logged at `warn`. +If the headers are missing entirely while `debugging` is on, the embedding step itself failed and the request was served by the fallback before any score was computed; the reason is logged at `warn`. -Because `expose_scores` reveals instance names to the caller, keep it off in production and rely on the log line instead. +Because `debugging` reveals instance names to the caller, keep it off in production and rely on the log line instead. diff --git a/docs/zh/latest/plugins/ai-proxy-multi.md b/docs/zh/latest/plugins/ai-proxy-multi.md index 65eac5df0449..f351b0764a83 100644 --- a/docs/zh/latest/plugins/ai-proxy-multi.md +++ b/docs/zh/latest/plugins/ai-proxy-multi.md @@ -71,9 +71,7 @@ import TabItem from '@theme/TabItem'; | max_retries | integer | 否 | | 大于或等于 0 | 初始请求失败后允许的最大故障转移重试次数。用于限制单个请求最多尝试多少个额外实例，避免穷举所有已配置的实例。仅在配置 `fallback_strategy` 时生效。未设置时，插件会持续重试直到某个实例成功或所有实例都已尝试。 | | retry_on_failure_within_ms | integer | 否 | | 大于或等于 1 | 仅当上游在指定毫秒数内失败时才故障转移到其他实例。快速失败（如连接错误、快速返回的 `429`/`5xx`）会触发重试，而耗时超过该值的慢失败会直接将错误返回给客户端，避免客户端等待时间翻倍。仅在配置 `fallback_strategy` 时生效。未设置时，插件无论失败请求耗时多久都会重试。 | | balancer | object | 否 | | | 负载均衡配置。 | -| balancer.algorithm | string | 否 | roundrobin | [roundrobin, chash, semantic] | 负载均衡算法。设置为 `roundrobin` 时，使用加权轮询算法。设置为 `chash` 时，使用一致性哈希算法。设置为 `semantic` 时，插件根据请求提示词与各实例 `examples` 之间的语义相似度选择实例。注意：`semantic` 不参与健康检查，也不参与 `fallback_strategy` / 重试——被选中实例的上游失败会直接返回给客户端；只有当没有实例达到其阈值或嵌入请求失败时，才会回退（回退到 `catchall` 实例，若未配置则回退到第一个实例）。 | -| balancer.threshold | number | 否 | 0 | -1 到 1 | 当 `algorithm` 为 `semantic` 时使用。实例被选中所需达到的全局最小余弦相似度。**默认值 0 几乎会接纳任何提示词**（真实嵌入向量很少会不相似到得分低于 0），因此只有把它调到 0 以上，`catchall` 才会真正生效。当没有实例达到其阈值时，请求回退到 `catchall` 实例；若未配置 `catchall`，则回退到第一个实例。 | -| balancer.expose_scores | boolean | 否 | false | | 当 `algorithm` 为 `semantic` 时使用。若为 true，通过 `X-AI-Semantic-Scores` 和 `X-AI-Semantic-Route` 响应头暴露各实例的分数和路由决策，用于调试。 | +| balancer.algorithm | string | 否 | roundrobin | [roundrobin, chash, semantic] | 负载均衡算法。设置为 `roundrobin` 时，使用加权轮询算法。设置为 `chash` 时，使用一致性哈希算法。设置为 `semantic` 时，插件根据请求提示词与各实例 `examples` 之间的语义相似度选择实例，其相关选项配置在 `semantic_opts` 下。注意：`semantic` 不参与健康检查，也不参与 `fallback_strategy` / 重试——被选中实例的上游失败会直接返回给客户端；只有当没有实例达到其阈值或嵌入请求失败时，才会回退（回退到 `semantic_opts.fallback` 实例，若未配置则回退到第一个实例）。 | | balancer.hash_on | string | 否 | | [vars, headers, cookie, consumer, vars_combinations] | 当 `type` 为 `chash` 时使用。支持基于 [NGINX 变量](https://nginx.org/en/docs/varindex.html)、标头、cookie、消费者或 [NGINX 变量](https://nginx.org/en/docs/varindex.html)组合进行哈希。 | | balancer.key | string | 否 | | | 当 `type` 为 `chash` 时使用。当 `hash_on` 设置为 `header` 或 `cookie` 时，需要 `key`。当 `hash_on` 设置为 `consumer` 时，不需要 `key`，因为消费者名称将自动用作键。 | | instances | array[object] | 是 | | | LLM 实例配置。 | @@ -103,9 +101,8 @@ import TabItem from '@theme/TabItem'; | instances.override.llm_options.max_tokens | integer | 否 | | ≥ 1 | 最大输出 token 数。APISIX 会自动将该值映射为各上游服务商对应的字段名。始终强制覆盖客户端值。 | | instances.override.request_body | object | 否 | | | 按目标协议的请求体覆盖配置。请参阅 `ai-proxy` 文档中的[按协议的请求体覆盖](./ai-proxy.md#per-protocol-request-body-override)。 | | instances.override.request_body_force_override | boolean | 否 | false | | 为 `false`（默认）时，客户端请求体中的字段优先，`instances.override.request_body` 仅补充缺失字段。为 `true` 时，`instances.override.request_body` 的值强制覆盖客户端请求体中的同名字段。不影响 `instances.override.llm_options`。 | -| instances.examples | array[string] | 否 | | 1 到 64 项 | 当 `algorithm` 为 `semantic` 时使用。代表该实例意图的示例语句；每一条都会被嵌入为独立的参考向量。除 `catchall` 实例外均为必填，而 `catchall` 实例不得配置该字段。 | -| instances.threshold | number | 否 | | -1 到 1 | 当 `algorithm` 为 `semantic` 时使用。该实例的最小余弦相似度，覆盖 `balancer.threshold`。 | -| instances.catchall | boolean | 否 | | | 当 `algorithm` 为 `semantic` 时使用。将该实例标记为语义回退实例，当没有实例达到其阈值时使用。它不参与排名，因此不得配置 `examples`。最多只能有一个实例被标记为 `catchall`。 | +| instances.examples | array[string] | 否 | | 1 到 64 项 | 当 `algorithm` 为 `semantic` 时使用。代表该实例意图的示例语句；每一条都会被嵌入为独立的参考向量。除被 `semantic_opts.fallback` 指定的实例外均为必填。 | +| instances.threshold | number | 否 | | -1 到 1 | 当 `algorithm` 为 `semantic` 时使用。该实例的最小余弦相似度，覆盖 `semantic_opts.threshold`。 | | logging | object | 否 | | | 日志配置。不影响 `error.log`。 | | logging.summaries | boolean | 否 | false | | 如果为 true，记录请求 LLM 模型、持续时间、请求和响应令牌。 | | logging.payloads | boolean | 否 | false | | 如果为 true，记录请求和响应负载。 | @@ -129,15 +126,19 @@ import TabItem from '@theme/TabItem'; | instances.checks.active.unhealthy.http_statuses | array[integer] | 否 | [429,404,500,501,502,503,504,505] | 200 到 599 之间的状态码（包含） | 定义不健康节点的 HTTP 状态码数组。 | | instances.checks.active.unhealthy.http_failures | integer | 否 | 5 | 1 到 254（包含） | 定义不健康节点的 HTTP 失败次数。 | | instances.checks.active.unhealthy.timeout | integer | 否 | 3 | 1 到 254（包含） | 定义不健康节点的探测超时次数。 | -| embeddings | object | 否 | | | 嵌入服务配置。当 `balancer.algorithm` 为 `semantic` 时必填。 | -| embeddings.provider | string | 是 | | [openai, azure-openai] | 语义算法使用的嵌入服务提供商。 | -| embeddings.model | string | 是 | | | 嵌入模型名称，例如 `text-embedding-3-small`。 | -| embeddings.endpoint | string | 否 | | | 嵌入 API 端点。对于 `openai` 可选（默认使用公共 API）。对于 `azure-openai` 必填，且必须是**完整** URL，包含 deployment 路径和 API 版本，例如 `https://{resource}.openai.azure.com/openai/deployments/{deployment}/embeddings?api-version=2024-02-01`。 | -| embeddings.auth | object | 是 | | | 嵌入服务的认证配置。 | -| embeddings.auth.header | object | 否 | | | 认证请求头。加密存储。 | -| embeddings.auth.query | object | 否 | | | 认证查询参数。加密存储。 | -| embeddings.timeout | integer | 否 | 10000 | 最小值为 1 | 嵌入请求的超时时间（毫秒）。 | -| embeddings.ssl_verify | boolean | 否 | true | | 如果为 true，验证嵌入服务的证书。 | +| semantic_opts | object | 否 | | | `semantic` 均衡器的选项，集中配置在一处。当 `balancer.algorithm` 为 `semantic` 时必填。 | +| semantic_opts.threshold | number | 否 | 0 | -1 到 1 | 实例被选中所需达到的全局最小余弦相似度。**默认值 0 几乎会接纳任何提示词**（真实嵌入向量很少会不相似到得分低于 0），因此只有把它调到 0 以上，回退实例才会真正生效。当没有实例达到其阈值时，请求回退到 `fallback` 实例；若未指定，则回退到第一个实例。 | +| semantic_opts.fallback | string | 否 | | | 当没有实例达到其阈值或嵌入请求失败时，路由到的实例名称。与参与排名的实例不同，它无需配置 `examples`。未设置时默认回退到第一个实例。 | +| semantic_opts.debugging | boolean | 否 | false | | 若为 true，通过 `X-AI-Semantic-Scores` 和 `X-AI-Semantic-Route` 响应头暴露各实例的分数和路由决策，用于调试。 | +| semantic_opts.embeddings | object | 是 | | | 嵌入服务配置，用于将提示词与各实例的 `examples` 进行相似度打分。 | +| semantic_opts.embeddings.provider | string | 是 | | [openai, azure-openai] | 语义算法使用的嵌入服务提供商。 | +| semantic_opts.embeddings.model | string | 是 | | | 嵌入模型名称，例如 `text-embedding-3-small`。 | +| semantic_opts.embeddings.endpoint | string | 否 | | | 嵌入 API 端点。对于 `openai` 可选（默认使用公共 API）。对于 `azure-openai` 必填，且必须是**完整** URL，包含 deployment 路径和 API 版本，例如 `https://{resource}.openai.azure.com/openai/deployments/{deployment}/embeddings?api-version=2024-02-01`。 | +| semantic_opts.embeddings.auth | object | 是 | | | 嵌入服务的认证配置。 | +| semantic_opts.embeddings.auth.header | object | 否 | | | 认证请求头。加密存储。 | +| semantic_opts.embeddings.auth.query | object | 否 | | | 认证查询参数。加密存储。 | +| semantic_opts.embeddings.timeout | integer | 否 | 10000 | 最小值为 1 | 嵌入请求的超时时间（毫秒）。 | +| semantic_opts.embeddings.ssl_verify | boolean | 否 | true | | 如果为 true，验证嵌入服务的证书。 | | timeout | integer | 否 | 30000 | 大于或等于 1 | 请求 LLM 服务时的请求超时时间（毫秒）。应用于单次 socket 操作（连接 / 发送 / 读取块），不限制流式响应的总时长。 | | max_stream_duration_ms | integer | 否 | | 大于或等于 1 | 流式 AI 响应的总墙钟时长上限（毫秒）。若上游在此时间后仍持续发送数据，网关将关闭连接。未设置时不限制。用于防护上游持续输出 token 导致网关 CPU 被打满的异常情况。中途触发上限时，下游 SSE 流会被截断（不再发送协议特定的终止标记，例如 `[DONE]`、`message_stop` 或 `response.completed`），客户端应将缺失的终止标记视为响应未完成。 | | max_response_bytes | integer | 否 | | 大于或等于 1 | 单次 AI 响应（流式或非流式）允许从上游读取的最大总字节数。超出时关闭连接。非流式响应若存在 `Content-Length`，在读取 body 之前预检；否则（chunked 传输）与流式响应一样在接收字节的过程中增量检查。未设置时不限制。 | @@ -2796,9 +2797,9 @@ nginx_config: ### 按语义相似度路由 -以下示例演示了如何使用 `semantic` 算法，根据请求提示词的语义而非权重或哈希来选择实例。每个非 `catchall` 实例通过 `examples` 声明它应当处理的提示词类型；插件使用配置的嵌入模型将这些示例和传入的提示词嵌入为向量，并将请求转发给示例与之最相似的实例。 +以下示例演示了如何使用 `semantic` 算法，根据请求提示词的语义而非权重或哈希来选择实例。每个参与排名的实例通过 `examples` 声明它应当处理的提示词类型；插件使用配置的嵌入模型将这些示例和传入的提示词嵌入为向量，并将请求转发给示例与之最相似的实例。 -创建如下路由，并相应地更新 LLM 提供商、嵌入模型和 API 密钥。`code` 实例处理编程类提示词，`translate` 实例处理翻译类提示词，`default` 实例是 `catchall`，当没有实例达到相似度阈值时使用： +创建如下路由，并相应地更新 LLM 提供商、嵌入模型和 API 密钥。`code` 实例处理编程类提示词，`translate` 实例处理翻译类提示词，`default` 实例被指定为 `semantic_opts.fallback`，当没有实例达到相似度阈值时使用： ```shell curl "http://127.0.0.1:9180/apisix/admin/routes" -X PUT \ @@ -2809,18 +2810,21 @@ curl "http://127.0.0.1:9180/apisix/admin/routes" -X PUT \ "methods": ["POST"], "plugins": { "ai-proxy-multi": { - "embeddings": { - "provider": "openai", - "model": "text-embedding-3-small", - "auth": { - "header": { - "Authorization": "Bearer '"$YOUR_EMBEDDING_API_KEY"'" - } - } - }, "balancer": { - "algorithm": "semantic", - "threshold": 0.5 + "algorithm": "semantic" + }, + "semantic_opts": { + "embeddings": { + "provider": "openai", + "model": "text-embedding-3-small", + "auth": { + "header": { + "Authorization": "Bearer '"$YOUR_EMBEDDING_API_KEY"'" + } + } + }, + "threshold": 0.5, + "fallback": "default" }, "instances": [ { @@ -2867,8 +2871,7 @@ curl "http://127.0.0.1:9180/apisix/admin/routes" -X PUT \ }, "options": { "model": "gpt-4o-mini" - }, - "catchall": true + } } ] } @@ -2889,11 +2892,11 @@ curl "http://127.0.0.1:9080/anything" -X POST \ }' ``` -该请求应被路由到 `code` 实例，由 `gpt-4o` 处理。如果提示词与任何实例的 `examples` 都不相关（例如 `what is the weather today`），则不会达到任何阈值，请求会被路由到 `default` 这个 `catchall` 实例。 +该请求应被路由到 `code` 实例，由 `gpt-4o` 处理。如果提示词与任何实例的 `examples` 都不相关（例如 `what is the weather today`），则不会达到任何阈值，请求会被路由到 `default` 这个回退实例。 #### 调试路由决策 -将 `balancer.expose_scores` 设置为 `true`，插件会在响应中报告各实例的得分以及最终选中的实例： +将 `semantic_opts.debugging` 设置为 `true`，插件会在响应中报告各实例的得分以及最终选中的实例： ```shell curl -i "http://127.0.0.1:9080/anything" -X POST \ @@ -2907,7 +2910,7 @@ X-AI-Semantic-Route: code X-AI-Semantic-Scores: code:0.8213,translate:0.1904 ``` -`X-AI-Semantic-Scores` 按得分从高到低列出**所有配置了 `examples` 的实例**，因此你不仅能看到胜出者，也能看到其他实例差了多少。没有配置 `examples` 的实例（即 `catchall`）没有相似度得分，因此不会出现在其中。 +`X-AI-Semantic-Scores` 按得分从高到低列出**所有配置了 `examples` 的实例**，因此你不仅能看到胜出者，也能看到其他实例差了多少。没有配置 `examples` 的实例（例如纯回退实例）没有相似度得分，因此不会出现在其中。 当没有任何实例达到阈值时，选中结果变为 `fallback`，而得分仍会显示各实例的接近程度： @@ -2916,7 +2919,7 @@ X-AI-Semantic-Route: fallback X-AI-Semantic-Scores: code:0.3057,translate:0.2884 ``` -据此可以校准 `threshold`：取一个介于「希望匹配的提示词得分」与「希望落到 `catchall` 的提示词得分」之间的值。 +据此可以校准 `threshold`：取一个介于「希望匹配的提示词得分」与「希望落到回退实例的提示词得分」之间的值。 同样的信息也可以不暴露给客户端：每当没有实例达到阈值时，插件都会以 `warn` 级别记录完整的得分列表。 @@ -2924,6 +2927,6 @@ X-AI-Semantic-Scores: code:0.3057,translate:0.2884 semantic routing: no instance cleared threshold (scores: code:0.3057,translate:0.2884), falling back ``` -如果开启了 `expose_scores` 却完全没有这两个响应头，说明嵌入请求本身失败了，请求在计算任何得分之前就已回退到 `catchall`，失败原因会以 `warn` 级别记录。 +如果开启了 `debugging` 却完全没有这两个响应头，说明嵌入请求本身失败了，请求在计算任何得分之前就已回退到回退实例，失败原因会以 `warn` 级别记录。 -由于 `expose_scores` 会向调用方暴露实例名称，生产环境应保持关闭，改用日志排查。 +由于 `debugging` 会向调用方暴露实例名称，生产环境应保持关闭，改用日志排查。 diff --git a/t/plugin/ai-proxy-semantic-routing.t b/t/plugin/ai-proxy-semantic-routing.t index 45785bfb8ad8..5ef15b922abc 100644 --- a/t/plugin/ai-proxy-semantic-routing.t +++ b/t/plugin/ai-proxy-semantic-routing.t @@ -161,7 +161,7 @@ __DATA__ -- bracket"), so we keep the inlined Lua short instead. local core = require("apisix.core") local t = require("lib.test_admin").test - local function inst(name, model, examples, catchall) + local function inst(name, model, examples) local i = { name = name, provider = "openai", weight = 1, auth = { header = { Authorization = "Bearer token" } }, @@ -171,25 +171,28 @@ __DATA__ }, } if examples then i.examples = examples end - if catchall then i.catchall = true end return i end local conf = { uri = "/anything", plugins = { ["ai-proxy-multi"] = { - embeddings = { - provider = "openai", - model = "text-embedding-3-small", - endpoint = "http://127.0.0.1:6797/v1/embeddings", - auth = { header = { Authorization = "Bearer token" } }, - ssl_verify = false, + balancer = { algorithm = "semantic" }, + semantic_opts = { + embeddings = { + provider = "openai", + model = "text-embedding-3-small", + endpoint = "http://127.0.0.1:6797/v1/embeddings", + auth = { header = { Authorization = "Bearer token" } }, + ssl_verify = false, + }, + threshold = 0.9, + fallback = "default", }, - balancer = { algorithm = "semantic", threshold = 0.9 }, instances = { inst("code", "model-code", {"write python code"}), inst("cheap", "model-cheap", {"translate this text"}), - inst("default", "model-fallback", nil, true), + inst("default", "model-fallback"), }, ssl_verify = false, }, @@ -269,13 +272,13 @@ semantic routing: query embedding failed -=== TEST 6: reconfigure the route with expose_scores enabled +=== TEST 6: reconfigure the route with debugging enabled --- config location /t { content_by_lua_block { local core = require("apisix.core") local t = require("lib.test_admin").test - local function inst(name, model, examples, catchall) + local function inst(name, model, examples) local i = { name = name, provider = "openai", weight = 1, auth = { header = { Authorization = "Bearer token" } }, @@ -285,28 +288,29 @@ semantic routing: query embedding failed }, } if examples then i.examples = examples end - if catchall then i.catchall = true end return i end local conf = { uri = "/anything", plugins = { ["ai-proxy-multi"] = { - embeddings = { - provider = "openai", - model = "text-embedding-3-small", - endpoint = "http://127.0.0.1:6797/v1/embeddings", - auth = { header = { Authorization = "Bearer token" } }, - ssl_verify = false, - }, - balancer = { - algorithm = "semantic", threshold = 0.9, - expose_scores = true, + balancer = { algorithm = "semantic" }, + semantic_opts = { + embeddings = { + provider = "openai", + model = "text-embedding-3-small", + endpoint = "http://127.0.0.1:6797/v1/embeddings", + auth = { header = { Authorization = "Bearer token" } }, + ssl_verify = false, + }, + threshold = 0.9, + fallback = "default", + debugging = true, }, instances = { inst("code", "model-code", {"write python code"}), inst("cheap", "model-cheap", {"translate this text"}), - inst("default", "model-fallback", nil, true), + inst("default", "model-fallback"), }, ssl_verify = false, }, @@ -447,7 +451,7 @@ invalid embedding entry at index 0 content_by_lua_block { local core = require("apisix.core") local t = require("lib.test_admin").test - local function inst(name, model, examples, catchall) + local function inst(name, model, examples) local i = { name = name, provider = "openai", weight = 1, auth = { header = { Authorization = "Bearer token" } }, @@ -457,26 +461,29 @@ invalid embedding entry at index 0 }, } if examples then i.examples = examples end - if catchall then i.catchall = true end return i end local conf = { uri = "/anything", plugins = { ["ai-proxy-multi"] = { - embeddings = { - provider = "openai", - model = "text-embedding-3-small", - endpoint = "http://127.0.0.1:6797/v1/embeddings", - auth = { header = { Authorization = "Bearer token" } }, - ssl_verify = false, + balancer = { algorithm = "semantic" }, + semantic_opts = { + embeddings = { + provider = "openai", + model = "text-embedding-3-small", + endpoint = "http://127.0.0.1:6797/v1/embeddings", + auth = { header = { Authorization = "Bearer token" } }, + ssl_verify = false, + }, + threshold = 0.9, + fallback = "default", }, - balancer = { algorithm = "semantic", threshold = 0.9 }, instances = { -- embedding this instance's examples makes the mock -- fail the whole reference batch inst("code", "model-code", {"servererror example"}), - inst("default", "model-fallback", nil, true), + inst("default", "model-fallback"), }, ssl_verify = false, }, @@ -521,7 +528,7 @@ failed to fetch reference embeddings content_by_lua_block { local core = require("apisix.core") local t = require("lib.test_admin").test - local function inst(name, model, examples, catchall) + local function inst(name, model, examples) local i = { name = name, provider = "openai", weight = 1, auth = { header = { Authorization = "Bearer token" } }, @@ -531,25 +538,28 @@ failed to fetch reference embeddings }, } if examples then i.examples = examples end - if catchall then i.catchall = true end return i end local conf = { uri = "/anything", plugins = { ["ai-proxy-multi"] = { - embeddings = { - provider = "azure-openai", - model = "text-embedding-3-small", - endpoint = "http://127.0.0.1:6799/openai/deployments/emb" - .. "/embeddings?api-version=2024-02-01", - auth = { header = { ["api-key"] = "azure-key" } }, - ssl_verify = false, + balancer = { algorithm = "semantic" }, + semantic_opts = { + embeddings = { + provider = "azure-openai", + model = "text-embedding-3-small", + endpoint = "http://127.0.0.1:6799/openai/deployments/emb" + .. "/embeddings?api-version=2024-02-01", + auth = { header = { ["api-key"] = "azure-key" } }, + ssl_verify = false, + }, + threshold = 0.9, + fallback = "default", }, - balancer = { algorithm = "semantic", threshold = 0.9 }, instances = { inst("code", "model-code", {"write python code"}), - inst("default", "model-fallback", nil, true), + inst("default", "model-fallback"), }, ssl_verify = false, }, @@ -608,14 +618,17 @@ azure-embed-hit api-version=2024-02-01 uri = "/anything", plugins = { ["ai-proxy-multi"] = { - embeddings = { - provider = "openai", - model = "text-embedding-3-small", - endpoint = "http://127.0.0.1:6797/v1/embeddings", - auth = { header = { Authorization = "Bearer token" } }, - ssl_verify = false, + balancer = { algorithm = "semantic" }, + semantic_opts = { + embeddings = { + provider = "openai", + model = "text-embedding-3-small", + endpoint = "http://127.0.0.1:6797/v1/embeddings", + auth = { header = { Authorization = "Bearer token" } }, + ssl_verify = false, + }, + threshold = 0.9, }, - balancer = { algorithm = "semantic", threshold = 0.9 }, instances = { inst("first", "model-first", {"write python code"}), inst("second", "model-second", {"translate this text"}), @@ -666,17 +679,21 @@ no instance cleared threshold uri = "/anything", plugins = { ["ai-proxy-multi"] = { - embeddings = { - provider = "openai", - model = "text-embedding-3-small", - endpoint = "http://127.0.0.1:6797/v1/embeddings", - auth = { header = { Authorization = "Bearer token" } }, - ssl_verify = false, - }, -- the global threshold is permissive (0.1); the code -- instance raises its own bar to 0.9, so a prompt scoring -- ~0.707 clears the global one but not the instance's - balancer = { algorithm = "semantic", threshold = 0.1 }, + balancer = { algorithm = "semantic" }, + semantic_opts = { + embeddings = { + provider = "openai", + model = "text-embedding-3-small", + endpoint = "http://127.0.0.1:6797/v1/embeddings", + auth = { header = { Authorization = "Bearer token" } }, + ssl_verify = false, + }, + threshold = 0.1, + fallback = "default", + }, instances = { { name = "code", provider = "openai", weight = 1, @@ -695,7 +712,6 @@ no instance cleared threshold override = { endpoint = "http://127.0.0.1:6798/v1/chat/completions", }, - catchall = true, }, }, ssl_verify = false, @@ -738,7 +754,7 @@ model-fallback content_by_lua_block { local core = require("apisix.core") local t = require("lib.test_admin").test - local function inst(name, model, examples, catchall) + local function inst(name, model, examples) local i = { name = name, provider = "openai", weight = 1, auth = { header = { Authorization = "Bearer token" } }, @@ -748,28 +764,29 @@ model-fallback }, } if examples then i.examples = examples end - if catchall then i.catchall = true end return i end local conf = { uri = "/llm/v1/responses", plugins = { ["ai-proxy-multi"] = { - embeddings = { - provider = "openai", - model = "text-embedding-3-small", - endpoint = "http://127.0.0.1:6797/v1/embeddings", - auth = { header = { Authorization = "Bearer token" } }, - ssl_verify = false, - }, - balancer = { - algorithm = "semantic", threshold = 0.9, - expose_scores = true, + balancer = { algorithm = "semantic" }, + semantic_opts = { + embeddings = { + provider = "openai", + model = "text-embedding-3-small", + endpoint = "http://127.0.0.1:6797/v1/embeddings", + auth = { header = { Authorization = "Bearer token" } }, + ssl_verify = false, + }, + threshold = 0.9, + fallback = "default", + debugging = true, }, instances = { inst("code", "model-code", {"write python code"}), inst("cheap", "model-cheap", {"translate this text"}), - inst("default", "model-fallback", nil, true), + inst("default", "model-fallback"), }, ssl_verify = false, }, @@ -812,7 +829,7 @@ X-AI-Semantic-Scores: code:1\.\d+,cheap:0\.\d+ content_by_lua_block { local core = require("apisix.core") local t = require("lib.test_admin").test - local function inst(name, model, examples, catchall) + local function inst(name, model, examples) local i = { name = name, provider = "openai", weight = 1, auth = { header = { Authorization = "Bearer token" } }, @@ -822,28 +839,29 @@ X-AI-Semantic-Scores: code:1\.\d+,cheap:0\.\d+ }, } if examples then i.examples = examples end - if catchall then i.catchall = true end return i end local conf = { uri = "/llm/v1/messages", plugins = { ["ai-proxy-multi"] = { - embeddings = { - provider = "openai", - model = "text-embedding-3-small", - endpoint = "http://127.0.0.1:6797/v1/embeddings", - auth = { header = { Authorization = "Bearer token" } }, - ssl_verify = false, - }, - balancer = { - algorithm = "semantic", threshold = 0.9, - expose_scores = true, + balancer = { algorithm = "semantic" }, + semantic_opts = { + embeddings = { + provider = "openai", + model = "text-embedding-3-small", + endpoint = "http://127.0.0.1:6797/v1/embeddings", + auth = { header = { Authorization = "Bearer token" } }, + ssl_verify = false, + }, + threshold = 0.9, + fallback = "default", + debugging = true, }, instances = { inst("code", "model-code", {"write python code"}), inst("cheap", "model-cheap", {"translate this text"}), - inst("default", "model-fallback", nil, true), + inst("default", "model-fallback"), }, ssl_verify = false, }, diff --git a/t/plugin/ai-proxy-semantic-schema.t b/t/plugin/ai-proxy-semantic-schema.t index dd2a4aded983..734dfd5c2ec1 100644 --- a/t/plugin/ai-proxy-semantic-schema.t +++ b/t/plugin/ai-proxy-semantic-schema.t @@ -42,13 +42,15 @@ __DATA__ "uri": "/anything", "plugins": { "ai-proxy-multi": { - "embeddings": { - "provider": "openai", - "model": "text-embedding-3-small", - "auth": { "header": { "Authorization": "Bearer sk-emb" } } - }, - "balancer": { - "algorithm": "semantic", "threshold": 0.75 + "balancer": { "algorithm": "semantic" }, + "semantic_opts": { + "embeddings": { + "provider": "openai", + "model": "text-embedding-3-small", + "auth": { "header": { "Authorization": "Bearer sk-emb" } } + }, + "threshold": 0.75, + "fallback": "default" }, "instances": [ { @@ -61,8 +63,7 @@ __DATA__ { "name": "default", "provider": "openai", "weight": 1, "auth": { "header": { "Authorization": "Bearer token" } }, - "options": { "model": "gpt-4o-mini" }, - "catchall": true + "options": { "model": "gpt-4o-mini" } } ], "ssl_verify": false @@ -84,7 +85,7 @@ passed -=== TEST 2: semantic requires examples on non-catchall instance +=== TEST 2: semantic requires examples on a non-fallback instance --- config location /t { content_by_lua_block { @@ -93,11 +94,13 @@ passed "uri": "/anything", "plugins": { "ai-proxy-multi": { - "embeddings": { - "provider": "openai", "model": "text-embedding-3-small", - "auth": { "header": { "Authorization": "Bearer sk-emb" } } - }, "balancer": { "algorithm": "semantic" }, + "semantic_opts": { + "embeddings": { + "provider": "openai", "model": "text-embedding-3-small", + "auth": { "header": { "Authorization": "Bearer sk-emb" } } + } + }, "instances": [ { "name": "code", "provider": "openai", "weight": 1, @@ -118,7 +121,7 @@ passed -=== TEST 3: semantic requires embeddings config +=== TEST 3: semantic requires semantic_opts.embeddings config --- config location /t { content_by_lua_block { @@ -149,7 +152,7 @@ passed -=== TEST 4: at most one catchall instance +=== TEST 4: semantic_opts.fallback must name an existing instance --- config location /t { content_by_lua_block { @@ -158,23 +161,20 @@ passed "uri": "/anything", "plugins": { "ai-proxy-multi": { - "embeddings": { - "provider": "openai", "model": "text-embedding-3-small", - "auth": { "header": { "Authorization": "Bearer sk-emb" } } - }, "balancer": { "algorithm": "semantic" }, + "semantic_opts": { + "embeddings": { + "provider": "openai", "model": "text-embedding-3-small", + "auth": { "header": { "Authorization": "Bearer sk-emb" } } + }, + "fallback": "nope" + }, "instances": [ { "name": "a", "provider": "openai", "weight": 1, "auth": { "header": { "Authorization": "Bearer token" } }, - "options": { "model": "gpt-4o-mini" }, - "catchall": true - }, - { - "name": "b", "provider": "openai", "weight": 1, - "auth": { "header": { "Authorization": "Bearer token" } }, "options": { "model": "gpt-4o" }, - "catchall": true + "examples": ["write code"] } ], "ssl_verify": false @@ -186,7 +186,7 @@ passed } } --- error_code: 400 ---- response_body_like: catchall +--- response_body_like: unknown instance @@ -199,11 +199,13 @@ passed "uri": "/anything", "plugins": { "ai-proxy-multi": { - "embeddings": { - "provider": "azure-openai", "model": "text-embedding-3-small", - "auth": { "header": { "api-key": "sk-emb" } } - }, "balancer": { "algorithm": "semantic" }, + "semantic_opts": { + "embeddings": { + "provider": "azure-openai", "model": "text-embedding-3-small", + "auth": { "header": { "api-key": "sk-emb" } } + } + }, "instances": [ { "name": "a", "provider": "openai", "weight": 1, @@ -234,11 +236,13 @@ passed "uri": "/anything", "plugins": { "ai-proxy-multi": { - "embeddings": { - "provider": "openai", - "auth": { "header": { "Authorization": "Bearer sk-emb" } } - }, "balancer": { "algorithm": "semantic" }, + "semantic_opts": { + "embeddings": { + "provider": "openai", + "auth": { "header": { "Authorization": "Bearer sk-emb" } } + } + }, "instances": [ { "name": "a", "provider": "openai", "weight": 1, @@ -260,7 +264,7 @@ passed -=== TEST 7: catchall instance must not configure examples +=== TEST 7: only the named fallback is exempt from the examples requirement --- config location /t { content_by_lua_block { @@ -269,24 +273,24 @@ passed "uri": "/anything", "plugins": { "ai-proxy-multi": { - "embeddings": { - "provider": "openai", "model": "text-embedding-3-small", - "auth": { "header": { "Authorization": "Bearer sk-emb" } } - }, "balancer": { "algorithm": "semantic" }, + "semantic_opts": { + "embeddings": { + "provider": "openai", "model": "text-embedding-3-small", + "auth": { "header": { "Authorization": "Bearer sk-emb" } } + }, + "fallback": "default" + }, "instances": [ { "name": "a", "provider": "openai", "weight": 1, "auth": { "header": { "Authorization": "Bearer token" } }, - "options": { "model": "gpt-4o" }, - "examples": ["write code"] + "options": { "model": "gpt-4o" } }, { - "name": "fallback", "provider": "openai", "weight": 1, + "name": "default", "provider": "openai", "weight": 1, "auth": { "header": { "Authorization": "Bearer token" } }, - "options": { "model": "gpt-4o-mini" }, - "catchall": true, - "examples": ["anything else"] + "options": { "model": "gpt-4o-mini" } } ], "ssl_verify": false @@ -298,7 +302,7 @@ passed } } --- error_code: 400 ---- response_body_like: must not configure +--- response_body_like: examples @@ -311,12 +315,14 @@ passed "uri": "/anything", "plugins": { "ai-proxy-multi": { - "embeddings": { - "provider": "azure-openai", "model": "text-embedding-3-small", - "endpoint": "https://my.openai.azure.com", - "auth": { "header": { "api-key": "sk-emb" } } - }, "balancer": { "algorithm": "semantic" }, + "semantic_opts": { + "embeddings": { + "provider": "azure-openai", "model": "text-embedding-3-small", + "endpoint": "https://my.openai.azure.com", + "auth": { "header": { "api-key": "sk-emb" } } + } + }, "instances": [ { "name": "a", "provider": "openai", "weight": 1, @@ -347,11 +353,13 @@ passed "uri": "/anything", "plugins": { "ai-proxy-multi": { - "embeddings": { - "provider": "openai", "model": "text-embedding-3-small", - "auth": { "gcp": { "service_account_json": "{}" } } - }, "balancer": { "algorithm": "semantic" }, + "semantic_opts": { + "embeddings": { + "provider": "openai", "model": "text-embedding-3-small", + "auth": { "gcp": { "service_account_json": "{}" } } + } + }, "instances": [ { "name": "a", "provider": "openai", "weight": 1, @@ -382,12 +390,14 @@ passed "uri": "/anything", "plugins": { "ai-proxy-multi": { - "embeddings": { - "provider": "openai", "model": "text-embedding-3-small", - "endpoint": "my.embeddings.host/v1/embeddings", - "auth": { "header": { "Authorization": "Bearer sk" } } - }, "balancer": { "algorithm": "semantic" }, + "semantic_opts": { + "embeddings": { + "provider": "openai", "model": "text-embedding-3-small", + "endpoint": "my.embeddings.host/v1/embeddings", + "auth": { "header": { "Authorization": "Bearer sk" } } + } + }, "instances": [ { "name": "a", "provider": "openai", "weight": 1, @@ -405,4 +415,4 @@ passed } } --- error_code: 400 ---- response_body_like: invalid `embeddings.endpoint` +--- response_body_like: semantic_opts.embeddings.endpoint From df4a0ee1316be88e0847e10d6c42e5377408fed3 Mon Sep 17 00:00:00 2001 From: AlinsRan Date: Thu, 16 Jul 2026 06:59:39 +0800 Subject: [PATCH 4/8] test(ai-proxy-multi): align semantic test naming with semantic_opts Post-refactor cleanup from code review: - Update the stale 'catchall' comment to 'fallback'. - Rename test titles that still said 'catchall' to match the semantic_opts.fallback model (bodies were already migrated). - Add a schema test asserting the named fallback instance may also carry examples and participate in ranking. --- apisix/plugins/ai-proxy-multi.lua | 2 +- t/plugin/ai-proxy-semantic-routing.t | 18 +++++----- t/plugin/ai-proxy-semantic-schema.t | 50 ++++++++++++++++++++++++++++ 3 files changed, 60 insertions(+), 10 deletions(-) diff --git a/apisix/plugins/ai-proxy-multi.lua b/apisix/plugins/ai-proxy-multi.lua index c0955ae1940d..a349d106e9aa 100644 --- a/apisix/plugins/ai-proxy-multi.lua +++ b/apisix/plugins/ai-proxy-multi.lua @@ -62,7 +62,7 @@ local lrucache_semantic_vectors = core.lrucache.new({ -- The prompt is sent verbatim to a third-party embedding endpoint. Bound it: a -- request body may be up to max_req_body_size (64MB by default), and an oversized -- input would blow the embedding model's token limit, 400, and silently push every --- large prompt to the catchall. Routing intent lives in the opening sentences. +-- large prompt to the fallback. Routing intent lives in the opening sentences. local MAX_EMBED_PROMPT_BYTES = 8192 local plugin_name = "ai-proxy-multi" diff --git a/t/plugin/ai-proxy-semantic-routing.t b/t/plugin/ai-proxy-semantic-routing.t index 5ef15b922abc..3dd3ebb9ee79 100644 --- a/t/plugin/ai-proxy-semantic-routing.t +++ b/t/plugin/ai-proxy-semantic-routing.t @@ -151,7 +151,7 @@ run_tests(); __DATA__ -=== TEST 1: configure a semantic route (code / translate / catchall) +=== TEST 1: configure a semantic route (code / translate / fallback) --- config location /t { content_by_lua_block { @@ -241,7 +241,7 @@ model-cheap -=== TEST 4: unrelated prompt clears no threshold, falls back to catchall +=== TEST 4: unrelated prompt clears no threshold, falls back to the fallback instance --- request POST /anything {"model":"auto","messages":[{"role":"user","content":"what is the weather today"}]} @@ -256,7 +256,7 @@ qr/no instance cleared threshold \(scores: code:0\.\d+,cheap:0\.\d+\)/ -=== TEST 5: malformed embedding fails open to catchall, not the matching instance +=== TEST 5: malformed embedding fails open to the fallback, not the matching instance --- request POST /anything {"model":"auto","messages":[{"role":"user","content":"debug this python code, malformed"}]} @@ -349,7 +349,7 @@ X-AI-Semantic-Scores: code:1\.\d+,cheap:0\.\d+ -=== TEST 8: embedding dimension mismatch fails open to catchall +=== TEST 8: embedding dimension mismatch fails open to the fallback --- request POST /anything {"model":"auto","messages":[{"role":"user","content":"dimmismatch python code"}]} @@ -397,7 +397,7 @@ embed-recv-cookie:session=super-secret -=== TEST 11: non-200 fails open to catchall, without logging the upstream body +=== TEST 11: non-200 fails open to the fallback, without logging the upstream body --- request POST /anything {"model":"auto","messages":[{"role":"user","content":"servererror python code"}]} @@ -445,7 +445,7 @@ invalid embedding entry at index 0 -=== TEST 14: reference embedding failure fails open to catchall +=== TEST 14: reference embedding failure fails open to the fallback --- config location /t { content_by_lua_block { @@ -506,7 +506,7 @@ passed -=== TEST 15: request on the broken-reference route still routes to catchall +=== TEST 15: request on the broken-reference route still routes to the fallback --- request POST /anything {"model":"auto","messages":[{"role":"user","content":"help me debug this python code"}]} @@ -597,7 +597,7 @@ azure-embed-hit api-version=2024-02-01 -=== TEST 18: no catchall configured -- fallback is the first instance +=== TEST 18: no fallback configured -- fallback is the first instance --- config location /t { content_by_lua_block { @@ -735,7 +735,7 @@ passed -=== TEST 21: a prompt below the per-instance threshold reaches the catchall +=== TEST 21: a prompt below the per-instance threshold reaches the fallback --- request POST /anything {"model":"auto","messages":[{"role":"user","content":"what is the weather today"}]} diff --git a/t/plugin/ai-proxy-semantic-schema.t b/t/plugin/ai-proxy-semantic-schema.t index 734dfd5c2ec1..3be5041817ec 100644 --- a/t/plugin/ai-proxy-semantic-schema.t +++ b/t/plugin/ai-proxy-semantic-schema.t @@ -416,3 +416,53 @@ passed } --- error_code: 400 --- response_body_like: semantic_opts.embeddings.endpoint + + + +=== TEST 11: the fallback instance may also carry examples and rank +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', ngx.HTTP_PUT, [[{ + "uri": "/anything", + "plugins": { + "ai-proxy-multi": { + "balancer": { "algorithm": "semantic" }, + "semantic_opts": { + "embeddings": { + "provider": "openai", "model": "text-embedding-3-small", + "auth": { "header": { "Authorization": "Bearer sk-emb" } } + }, + "fallback": "generalist" + }, + "instances": [ + { + "name": "code", "provider": "openai", "weight": 1, + "auth": { "header": { "Authorization": "Bearer token" } }, + "options": { "model": "gpt-4o" }, + "examples": ["write a python function"] + }, + { + "name": "generalist", "provider": "openai", "weight": 1, + "auth": { "header": { "Authorization": "Bearer token" } }, + "options": { "model": "gpt-4o-mini" }, + "examples": ["answer a general question"] + } + ], + "ssl_verify": false + } + } + }]]) + if code >= 300 then + ngx.status = code + ngx.say(body) + return + end + ngx.say("passed") + } + } +--- response_body +passed +--- no_error_log +[error] From ff31a4405fa408a28633ab51a123b2328acd4fb7 Mon Sep 17 00:00:00 2001 From: AlinsRan Date: Thu, 16 Jul 2026 10:27:23 +0800 Subject: [PATCH 5/8] refactor(ai-proxy-multi): lower default embedding timeout, cover non-finite guard Pre-merge review follow-ups: - Lower the embeddings.timeout default from 10s to 3s. The query prompt is embedded synchronously on every request, so a 10s default meant an embedding-endpoint outage blocked each request 10s before failing open to the fallback. Document the per-request dependency on the field. - Add a routing test asserting a non-finite (1e999 -> inf) embedding component fails open to the fallback instead of poisoning the scores. - Remove a duplicated, misplaced pcall comment above the prompt truncation block. --- apisix/plugins/ai-proxy-multi.lua | 2 - apisix/plugins/ai-proxy/embedding.lua | 2 +- apisix/plugins/ai-proxy/schema.lua | 8 ++- docs/en/latest/plugins/ai-proxy-multi.md | 2 +- docs/zh/latest/plugins/ai-proxy-multi.md | 2 +- t/plugin/ai-proxy-semantic-routing.t | 85 ++++++++++++++++++++++++ 6 files changed, 94 insertions(+), 7 deletions(-) diff --git a/apisix/plugins/ai-proxy-multi.lua b/apisix/plugins/ai-proxy-multi.lua index a349d106e9aa..955cbb83007f 100644 --- a/apisix/plugins/ai-proxy-multi.lua +++ b/apisix/plugins/ai-proxy-multi.lua @@ -784,8 +784,6 @@ local function pick_semantic_instance(ctx, conf) return semantic_fallback(conf) end - -- pcall, like the reference path above: the embedding response is - -- provider-controlled, so a raise here must fall back rather than 500. if #prompt > MAX_EMBED_PROMPT_BYTES then prompt = sub(prompt, 1, MAX_EMBED_PROMPT_BYTES) end diff --git a/apisix/plugins/ai-proxy/embedding.lua b/apisix/plugins/ai-proxy/embedding.lua index f069312a8c20..ba93c8c696eb 100644 --- a/apisix/plugins/ai-proxy/embedding.lua +++ b/apisix/plugins/ai-proxy/embedding.lua @@ -85,7 +85,7 @@ function _M.fetch(conf, texts) } local req_conf = { ssl_verify = conf.ssl_verify ~= false, - timeout = conf.timeout or 10000, + timeout = conf.timeout or 3000, keepalive = true, } diff --git a/apisix/plugins/ai-proxy/schema.lua b/apisix/plugins/ai-proxy/schema.lua index ad1a0518d80e..574a7d35a42e 100644 --- a/apisix/plugins/ai-proxy/schema.lua +++ b/apisix/plugins/ai-proxy/schema.lua @@ -259,8 +259,12 @@ local embeddings_schema = { timeout = { type = "integer", minimum = 1, - default = 10000, - description = "Embedding request timeout in milliseconds.", + default = 3000, + description = "Embedding request timeout in milliseconds. The query " + .. "prompt is embedded synchronously on every request, so this " + .. "bounds the latency added to each request when the embedding " + .. "endpoint is slow or down (the request then fails open to the " + .. "fallback).", }, ssl_verify = { type = "boolean", default = true }, }, diff --git a/docs/en/latest/plugins/ai-proxy-multi.md b/docs/en/latest/plugins/ai-proxy-multi.md index c43035371a8f..9f0523303eda 100644 --- a/docs/en/latest/plugins/ai-proxy-multi.md +++ b/docs/en/latest/plugins/ai-proxy-multi.md @@ -135,7 +135,7 @@ When an instance's `provider` is set to `bedrock`, the Plugin expects requests i | semantic_opts.embeddings.auth | object | True | | | Authentication configurations for the embedding service. | | semantic_opts.embeddings.auth.header | object | False | | | Authentication headers. Encrypted at rest. | | semantic_opts.embeddings.auth.query | object | False | | | Authentication query parameters. Encrypted at rest. | -| semantic_opts.embeddings.timeout | integer | False | 10000 | minimum = 1 | Embedding request timeout in milliseconds. | +| semantic_opts.embeddings.timeout | integer | False | 3000 | minimum = 1 | Embedding request timeout in milliseconds. The query prompt is embedded synchronously on every request, so this bounds the latency added to each request when the embedding endpoint is slow or down (the request then fails open to the fallback). | | semantic_opts.embeddings.ssl_verify | boolean | False | true | | If true, verify the embedding service's certificate. | | timeout | integer | False | 30000 | greater than or equal to 1 | Request timeout in milliseconds when requesting the LLM service. Applied per socket operation (connect / send / read block); does not cap the total duration of a streaming response. | | max_req_body_size | integer | False | 67108864 | greater than or equal to 1 | Maximum request body size in bytes that the plugin reads into memory. Requests whose body exceeds this limit are rejected with `413`. Prevents unbounded memory buffering of large request bodies. | diff --git a/docs/zh/latest/plugins/ai-proxy-multi.md b/docs/zh/latest/plugins/ai-proxy-multi.md index f351b0764a83..4b0bfd5214c5 100644 --- a/docs/zh/latest/plugins/ai-proxy-multi.md +++ b/docs/zh/latest/plugins/ai-proxy-multi.md @@ -137,7 +137,7 @@ import TabItem from '@theme/TabItem'; | semantic_opts.embeddings.auth | object | 是 | | | 嵌入服务的认证配置。 | | semantic_opts.embeddings.auth.header | object | 否 | | | 认证请求头。加密存储。 | | semantic_opts.embeddings.auth.query | object | 否 | | | 认证查询参数。加密存储。 | -| semantic_opts.embeddings.timeout | integer | 否 | 10000 | 最小值为 1 | 嵌入请求的超时时间（毫秒）。 | +| semantic_opts.embeddings.timeout | integer | 否 | 3000 | 最小值为 1 | 嵌入请求的超时时间（毫秒）。每个请求都会同步嵌入查询提示词，因此该值限定了当嵌入端点变慢或不可用时每个请求增加的延迟上限（随后请求会 fail-open 到回退实例）。 | | semantic_opts.embeddings.ssl_verify | boolean | 否 | true | | 如果为 true，验证嵌入服务的证书。 | | timeout | integer | 否 | 30000 | 大于或等于 1 | 请求 LLM 服务时的请求超时时间（毫秒）。应用于单次 socket 操作（连接 / 发送 / 读取块），不限制流式响应的总时长。 | | max_stream_duration_ms | integer | 否 | | 大于或等于 1 | 流式 AI 响应的总墙钟时长上限（毫秒）。若上游在此时间后仍持续发送数据，网关将关闭连接。未设置时不限制。用于防护上游持续输出 token 导致网关 CPU 被打满的异常情况。中途触发上限时，下游 SSE 流会被截断（不再发送协议特定的终止标记，例如 `[DONE]`、`message_stop` 或 `response.completed`），客户端应将缺失的终止标记视为响应未完成。 | diff --git a/t/plugin/ai-proxy-semantic-routing.t b/t/plugin/ai-proxy-semantic-routing.t index 3dd3ebb9ee79..5c29bec17e6e 100644 --- a/t/plugin/ai-proxy-semantic-routing.t +++ b/t/plugin/ai-proxy-semantic-routing.t @@ -63,6 +63,13 @@ add_block_preprocessor(sub { ngx.status = 200 ngx.say('{"data":[1,2,3]}') return + elseif string.lower(t):find("naninf") then + -- 200 with a non-finite embedding component + -- (1e999 decodes to inf): must be rejected rather + -- than normalized to NaN and poisoning every score + ngx.status = 200 + ngx.say('{"data":[{"index":0,"embedding":[1e999,0]}]}') + return end end local function vec(text) @@ -893,3 +900,81 @@ Content-Type: application/json --- response_headers_like X-AI-Semantic-Route: code X-AI-Semantic-Scores: code:1\.\d+,cheap:0\.\d+ + + + +=== TEST 26: configure a route to exercise the non-finite-component guard +--- config + location /t { + content_by_lua_block { + local core = require("apisix.core") + local t = require("lib.test_admin").test + local conf = { + uri = "/anything", + plugins = { + ["ai-proxy-multi"] = { + balancer = { algorithm = "semantic" }, + semantic_opts = { + embeddings = { + provider = "openai", + model = "text-embedding-3-small", + endpoint = "http://127.0.0.1:6797/v1/embeddings", + auth = { header = { Authorization = "Bearer token" } }, + ssl_verify = false, + }, + threshold = 0.9, + fallback = "default", + }, + instances = { + { + name = "code", provider = "openai", weight = 1, + auth = { header = { Authorization = "Bearer token" } }, + options = { model = "model-code" }, + override = { + endpoint = "http://127.0.0.1:6798/v1/chat/completions", + }, + examples = {"write python code"}, + }, + { + name = "default", provider = "openai", weight = 1, + auth = { header = { Authorization = "Bearer token" } }, + options = { model = "model-fallback" }, + override = { + endpoint = "http://127.0.0.1:6798/v1/chat/completions", + }, + }, + }, + ssl_verify = false, + }, + }, + } + local code, body = t('/apisix/admin/routes/1', ngx.HTTP_PUT, + core.json.encode(conf)) + if code >= 300 then + ngx.status = code + ngx.say(body) + return + end + ngx.say("passed") + } + } +--- response_body +passed +--- no_error_log +[error] + + + +=== TEST 27: a non-finite embedding component fails open instead of poisoning scores +--- request +POST /anything +{"model":"auto","messages":[{"role":"user","content":"naninf please route me"}]} +--- more_headers +Content-Type: application/json +--- error_code: 200 +--- response_body chomp +model-fallback +--- error_log +non-numeric embedding component +--- no_error_log +[error] From bb9588dce8da7d4cbcbe55e0c90a4692a5c57ab1 Mon Sep 17 00:00:00 2001 From: AlinsRan Date: Thu, 16 Jul 2026 14:19:43 +0800 Subject: [PATCH 6/8] refactor(ai-proxy-multi): keep transport change minimal for this PR MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per review (nic-6443): scope the transport touch in this feature PR down to just the skip_client_headers guard that stops the embedding sidecar call from leaking the client's Authorization/Cookie to a third-party endpoint. The deeper cleanup — making the HTTP transport and build_request fully ctx-free and removing the skip_client_headers flag entirely — is a cross-cutting refactor of the shared chat path and will land as its own PR (motivated by fixing the same pre-existing header leak in ai-request-rewrite), which this feature can then build on. --- apisix/plugins/ai-providers/base.lua | 13 ++----------- apisix/plugins/ai-transport/http.lua | 20 ++++++++++---------- 2 files changed, 12 insertions(+), 21 deletions(-) diff --git a/apisix/plugins/ai-providers/base.lua b/apisix/plugins/ai-providers/base.lua index 7c1416c5b17d..6b8c8e35c9c1 100644 --- a/apisix/plugins/ai-providers/base.lua +++ b/apisix/plugins/ai-providers/base.lua @@ -188,17 +188,8 @@ function _M.build_request(self, ctx, conf, request_body, opts) .. "includes a path", 400 end - -- Forward the downstream request's headers only on the proxy path. A - -- self-contained internal request (skip_client_headers, e.g. the embedding - -- sidecar call) carries its own credentials and must not leak the client's - -- Authorization/Cookie to a third-party endpoint. Resolving them here keeps - -- the transport free of any ctx dependency. - local client_headers - if not opts.skip_client_headers then - client_headers = core.request.headers(ctx) - end - local headers = transport_http.construct_forward_headers(auth.header or {}, - client_headers) + local headers = transport_http.construct_forward_headers(auth.header or {}, ctx, + opts.skip_client_headers) if opts.host_header then headers["Host"] = opts.host_header end diff --git a/apisix/plugins/ai-transport/http.lua b/apisix/plugins/ai-transport/http.lua index 4cbc61b84a1f..ee142061a32b 100644 --- a/apisix/plugins/ai-transport/http.lua +++ b/apisix/plugins/ai-transport/http.lua @@ -43,15 +43,13 @@ end --- Build forwarded headers from client request + extra headers. --- Copies `client_headers`, merges ext_opts_headers (lowercased), +-- Copies client headers, merges ext_opts_headers (lowercased), -- forces Content-Type to application/json, removes host/content-length. --- `client_headers` is the downstream request's headers to forward (proxy path), --- or nil for a self-contained internal request (e.g. the embedding sidecar call), --- which carries its own credentials and must not leak the client's Authorization, --- Cookie or other headers to a third-party endpoint. Passing them in explicitly --- keeps the transport free of any `ctx` dependency: the caller — which legitimately --- holds ctx — decides whether client headers are forwarded. -function _M.construct_forward_headers(ext_opts_headers, client_headers) +-- When skip_client_headers is true the client's request headers are not +-- forwarded. Use it for self-contained sidecar calls (e.g. embeddings), which +-- carry their own credentials and must not leak the client's Authorization, +-- Cookie or other headers to a third-party endpoint. +function _M.construct_forward_headers(ext_opts_headers, ctx, skip_client_headers) local blacklist = { "host", "content-length", @@ -59,8 +57,10 @@ function _M.construct_forward_headers(ext_opts_headers, client_headers) } local headers = {} - for k, v in pairs(client_headers or {}) do - headers[str_lower(k)] = v + if not skip_client_headers then + for k, v in pairs(core.request.headers(ctx) or {}) do + headers[str_lower(k)] = v + end end for k, v in pairs(ext_opts_headers or {}) do headers[str_lower(k)] = v From 225c38213952e19a112bd5a18913ac40186bd8f3 Mon Sep 17 00:00:00 2001 From: AlinsRan Date: Fri, 24 Jul 2026 10:44:21 +0800 Subject: [PATCH 7/8] feat(ai-proxy-multi): address semantic routing review feedback - Require non-empty `examples` on every semantic instance, including the one named by `semantic_opts.fallback`: the fallback is a normal ranked instance that competes on similarity and only additionally serves as the target when nothing clears its threshold, so it is no longer exempt. - Rename the debug response header `X-AI-Semantic-Route` to `X-AI-Semantic-Picked-Instance`, which names what it carries. - Bound the embedding prompt by ~2 bytes/token against the model's ~8192 token limit (16 KiB) instead of an arbitrary 8 KiB cap. Docs (en/zh) and tests updated accordingly. --- apisix/plugins/ai-proxy-multi.lua | 46 +++++++++--------- apisix/plugins/ai-proxy/schema.lua | 9 ++-- docs/en/latest/plugins/ai-proxy-multi.md | 26 +++++----- docs/zh/latest/plugins/ai-proxy-multi.md | 26 +++++----- t/plugin/ai-proxy-semantic-routing.t | 61 +++++++++++++++--------- t/plugin/ai-proxy-semantic-schema.t | 8 ++-- 6 files changed, 103 insertions(+), 73 deletions(-) diff --git a/apisix/plugins/ai-proxy-multi.lua b/apisix/plugins/ai-proxy-multi.lua index 955cbb83007f..2746a935742f 100644 --- a/apisix/plugins/ai-proxy-multi.lua +++ b/apisix/plugins/ai-proxy-multi.lua @@ -62,8 +62,11 @@ local lrucache_semantic_vectors = core.lrucache.new({ -- The prompt is sent verbatim to a third-party embedding endpoint. Bound it: a -- request body may be up to max_req_body_size (64MB by default), and an oversized -- input would blow the embedding model's token limit, 400, and silently push every --- large prompt to the fallback. Routing intent lives in the opening sentences. -local MAX_EMBED_PROMPT_BYTES = 8192 +-- large prompt to the fallback. OpenAI-compatible embedding models (which LiteLLM +-- and others target) cap input at ~8192 tokens; we bound by bytes since counting +-- tokens needs a tokenizer, using ~2 bytes/token so the cap stays under the token +-- limit even for multi-byte scripts while keeping the whole routing prompt. +local MAX_EMBED_PROMPT_BYTES = 16384 local plugin_name = "ai-proxy-multi" local _M = { @@ -206,30 +209,29 @@ function _M.check_schema(conf) "/openai/deployments/{deployment}/embeddings?api-version=..." end end - -- The `fallback` instance, if named, is the only one exempt from the - -- examples requirement: it is reached by fallback, not by ranking. It must - -- name an instance that actually exists. + -- Every instance must declare non-empty `examples`. The `fallback` + -- instance is a normal ranked instance too -- it competes on similarity + -- like the rest and only additionally serves as the target when nothing + -- clears its threshold -- so it is not exempt. A named fallback must point + -- at an instance that actually exists. local fallback = semantic_opts.fallback local fallback_found = false for _, instance in ipairs(conf.instances) do - local is_fallback = fallback and instance.name == fallback - if is_fallback then + if fallback and instance.name == fallback then fallback_found = true - else - local has_example = false - if instance.examples then - for _, ex in ipairs(instance.examples) do - if type(ex) == "string" and ex ~= "" then - has_example = true - break - end + end + local has_example = false + if instance.examples then + for _, ex in ipairs(instance.examples) do + if type(ex) == "string" and ex ~= "" then + has_example = true + break end end - if not has_example then - return false, "instance '" .. (instance.name or "?") .. - "': must configure non-empty `examples` for the semantic " .. - "algorithm unless it is named by `semantic_opts.fallback`" - end + end + if not has_example then + return false, "instance '" .. (instance.name or "?") .. + "': must configure non-empty `examples` for the semantic algorithm" end end if fallback and not fallback_found then @@ -842,7 +844,7 @@ local function pick_semantic_instance(ctx, conf) local thr = inst.threshold or conf.semantic_opts.threshold or 0 if cand.score >= thr then if debugging then - core.response.set_header("X-AI-Semantic-Route", cand.name) + core.response.set_header("X-AI-Semantic-Picked-Instance", cand.name) end core.log.info("semantic routing picked instance: ", cand.name, ", score: ", cand.score) @@ -851,7 +853,7 @@ local function pick_semantic_instance(ctx, conf) end if debugging then - core.response.set_header("X-AI-Semantic-Route", "fallback") + core.response.set_header("X-AI-Semantic-Picked-Instance", "fallback") end -- Only on the fallback path: surface why nothing matched, without requiring -- debugging. Cheap, because this runs once per unmatched request. diff --git a/apisix/plugins/ai-proxy/schema.lua b/apisix/plugins/ai-proxy/schema.lua index 42a1af9417b9..d5f13751d000 100644 --- a/apisix/plugins/ai-proxy/schema.lua +++ b/apisix/plugins/ai-proxy/schema.lua @@ -211,7 +211,8 @@ local ai_instance_schema = { description = "Example utterances representing this instance's " .. "intent; each is embedded into its own reference vector " .. "for the semantic algorithm. Required for every instance " - .. "except the one named by semantic_opts.fallback.", + .. "when the balancer algorithm is semantic, including the one " + .. "named by semantic_opts.fallback.", }, threshold = { type = "number", @@ -293,9 +294,9 @@ local semantic_opts_schema = { type = "string", minLength = 1, description = "Name of the instance to route to when no instance " - .. "clears its threshold or the embedding request fails. Unlike a " - .. "ranked instance it needs no `examples`. Defaults to the first " - .. "instance when unset.", + .. "clears its threshold or the embedding request fails. It is " + .. "otherwise a normal ranked instance and needs `examples` like " + .. "the rest. Defaults to the first instance when unset.", }, debugging = { type = "boolean", diff --git a/docs/en/latest/plugins/ai-proxy-multi.md b/docs/en/latest/plugins/ai-proxy-multi.md index 87b56c980acd..dc58ca565fbf 100644 --- a/docs/en/latest/plugins/ai-proxy-multi.md +++ b/docs/en/latest/plugins/ai-proxy-multi.md @@ -95,7 +95,7 @@ When an instance's `provider` is set to `bedrock`, the Plugin expects requests i | instances.auth.aws.session_token | string | False | | minLength = 1 | AWS session token for temporary credentials (e.g., from STS assume-role). Encrypted at rest. | | instances.options | object | False | | | Model configurations. In addition to `model`, you can configure additional parameters and they will be forwarded to the upstream LLM service in the request body. For instance, if you are working with OpenAI, DeepSeek, or AIMLAPI, you can configure additional parameters such as `max_tokens`, `temperature`, `top_p`, and `stream`. See your LLM provider's API documentation for more available options. | | instances.options.model | string | False | | | Name of the LLM model, such as `gpt-4` or `gpt-3.5`. See your LLM provider's API documentation for more available models. For Bedrock, this can be a foundation model ID (e.g., `anthropic.claude-3-5-sonnet-20240620-v1:0`), a cross-region inference profile ID (e.g., `us.anthropic.claude-3-5-sonnet-20240620-v1:0`), or an application inference profile ARN (e.g., `arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/abc123`). | -| instances.examples | array[string] | False | | 1 to 64 items | Used when `algorithm` is `semantic`. Example utterances representing this instance's intent; each is embedded into its own reference vector. Required for every instance except the one named by `semantic_opts.fallback`. | +| instances.examples | array[string] | False | | 1 to 64 items | Used when `algorithm` is `semantic`. Example utterances representing this instance's intent; each is embedded into its own reference vector. Required for every instance when `algorithm` is `semantic`, including the one named by `semantic_opts.fallback`. | | instances.threshold | number | False | | -1 to 1 | Used when `algorithm` is `semantic`. Per-instance minimum cosine similarity; overrides `semantic_opts.threshold`. | | logging | object | False | | | Logging configurations. | | logging.summaries | boolean | False | false | | If true, log request LLM model, duration, request, and response tokens. | @@ -126,8 +126,8 @@ When an instance's `provider` is set to `bedrock`, the Plugin expects requests i | instances.checks.active.unhealthy.timeout | integer | False | 3 | between 1 and 254 inclusive | Number of probe timeouts to define an unhealthy node. | | semantic_opts | object | False | | | Options for the `semantic` balancer, grouped in one place. Required when `balancer.algorithm` is `semantic`. | | semantic_opts.threshold | number | False | 0 | -1 to 1 | Global minimum cosine similarity an instance must reach to be selected. **The default of 0 admits essentially any prompt** (real embeddings are rarely dissimilar enough to score below 0), so the fallback only ever runs if you raise this above 0. When no instance clears its threshold, the request falls back to the `fallback` instance, or the first instance if none is named. | -| semantic_opts.fallback | string | False | | | Name of the instance to route to when no instance clears its threshold or the embedding request fails. Unlike a ranked instance it needs no `examples`. Defaults to the first instance when unset. | -| semantic_opts.debugging | boolean | False | false | | If true, expose per-instance scores and the routing decision via `X-AI-Semantic-Scores` and `X-AI-Semantic-Route` response headers, for debugging. | +| semantic_opts.fallback | string | False | | | Name of the instance to route to when no instance clears its threshold or the embedding request fails. It is otherwise a normal ranked instance and needs `examples` like the rest. Defaults to the first instance when unset. | +| semantic_opts.debugging | boolean | False | false | | If true, expose per-instance scores and the routing decision via `X-AI-Semantic-Scores` and `X-AI-Semantic-Picked-Instance` response headers, for debugging. | | semantic_opts.embeddings | object | True | | | Embedding service configuration used to score prompts against each instance's `examples`. | | semantic_opts.embeddings.provider | string | True | | [openai, azure-openai] | Embedding provider used by the semantic algorithm. | | semantic_opts.embeddings.model | string | True | | | Embedding model name, e.g. `text-embedding-3-small`. | @@ -2994,7 +2994,7 @@ In the Kafka topic, you should also see a log entry corresponding to the request The following example demonstrates how you can use the `semantic` algorithm to pick an instance by the meaning of the request prompt, rather than by weight or hash. Each ranked instance declares `examples` that describe the kind of prompt it should handle; the Plugin embeds those examples and the incoming prompt with the configured embedding model, and forwards the request to the instance whose examples are most similar. -Create a Route as such and update the LLM providers, embedding model, and API keys accordingly. The `code` instance handles programming prompts, the `translate` instance handles translation prompts, and the `default` instance is named as the `semantic_opts.fallback`, used when no instance clears the similarity threshold: +Create a Route as such and update the LLM providers, embedding model, and API keys accordingly. The `code` instance handles programming prompts, the `translate` instance handles translation prompts, and the `default` instance handles general prompts and is named as the `semantic_opts.fallback`, used when no instance clears the similarity threshold: ```shell curl "http://127.0.0.1:9180/apisix/admin/routes" -X PUT \ @@ -3066,7 +3066,11 @@ curl "http://127.0.0.1:9180/apisix/admin/routes" -X PUT \ }, "options": { "model": "gpt-4o-mini" - } + }, + "examples": [ + "what is the weather today", + "tell me a joke" + ] } ] } @@ -3087,7 +3091,7 @@ curl "http://127.0.0.1:9080/anything" -X POST \ }' ``` -The request should be routed to the `code` instance and served by `gpt-4o`. A prompt unrelated to any instance's `examples` (for example, `what is the weather today`) clears no threshold and is routed to the `default` fallback instance instead. +The request should be routed to the `code` instance and served by `gpt-4o`. A general prompt such as `what is the weather today` instead matches the `default` instance's own `examples`. The `default` instance is also named as `semantic_opts.fallback`, so it additionally receives any prompt that is ambiguous enough that no instance clears the threshold. #### Debugging the routing decision @@ -3101,17 +3105,17 @@ curl -i "http://127.0.0.1:9080/anything" -X POST \ ```text HTTP/1.1 200 OK -X-AI-Semantic-Route: code -X-AI-Semantic-Scores: code:0.8213,translate:0.1904 +X-AI-Semantic-Picked-Instance: code +X-AI-Semantic-Scores: code:0.8213,default:0.2451,translate:0.1904 ``` -`X-AI-Semantic-Scores` lists every instance that has `examples`, highest score first, so you can see not only the winner but how close the runners-up came. Instances without `examples` — such as a pure fallback — carry no similarity score and therefore do not appear. +`X-AI-Semantic-Scores` lists every instance, highest score first, so you can see not only the winner but how close the runners-up came. The `default` fallback instance is ranked alongside the others — it needs `examples` too — and additionally receives the request whenever no instance clears its threshold. When no instance clears its threshold, the pick becomes `fallback` while the scores still show how close each one got: ```text -X-AI-Semantic-Route: fallback -X-AI-Semantic-Scores: code:0.3057,translate:0.2884 +X-AI-Semantic-Picked-Instance: fallback +X-AI-Semantic-Scores: code:0.3057,translate:0.2884,default:0.2013 ``` Use this to calibrate `threshold`: pick a value between the scores of the prompts you want matched and the scores of the prompts you want to reach the fallback. diff --git a/docs/zh/latest/plugins/ai-proxy-multi.md b/docs/zh/latest/plugins/ai-proxy-multi.md index 47b46b350f74..686ad7459eda 100644 --- a/docs/zh/latest/plugins/ai-proxy-multi.md +++ b/docs/zh/latest/plugins/ai-proxy-multi.md @@ -101,7 +101,7 @@ import TabItem from '@theme/TabItem'; | instances.override.llm_options.max_tokens | integer | 否 | | ≥ 1 | 最大输出 token 数。APISIX 会自动将该值映射为各上游服务商对应的字段名。始终强制覆盖客户端值。 | | instances.override.request_body | object | 否 | | | 按目标协议的请求体覆盖配置。请参阅 `ai-proxy` 文档中的[按协议的请求体覆盖](./ai-proxy.md#per-protocol-request-body-override)。 | | instances.override.request_body_force_override | boolean | 否 | false | | 为 `false`（默认）时，客户端请求体中的字段优先，`instances.override.request_body` 仅补充缺失字段。为 `true` 时，`instances.override.request_body` 的值强制覆盖客户端请求体中的同名字段。不影响 `instances.override.llm_options`。 | -| instances.examples | array[string] | 否 | | 1 到 64 项 | 当 `algorithm` 为 `semantic` 时使用。代表该实例意图的示例语句；每一条都会被嵌入为独立的参考向量。除被 `semantic_opts.fallback` 指定的实例外均为必填。 | +| instances.examples | array[string] | 否 | | 1 到 64 项 | 当 `algorithm` 为 `semantic` 时使用。代表该实例意图的示例语句；每一条都会被嵌入为独立的参考向量。当 `algorithm` 为 `semantic` 时每个实例都必填，包括被 `semantic_opts.fallback` 指定的实例。 | | instances.threshold | number | 否 | | -1 到 1 | 当 `algorithm` 为 `semantic` 时使用。该实例的最小余弦相似度，覆盖 `semantic_opts.threshold`。 | | logging | object | 否 | | | 日志配置。不影响 `error.log`。 | | logging.summaries | boolean | 否 | false | | 如果为 true，记录请求 LLM 模型、持续时间、请求和响应令牌。 | @@ -128,8 +128,8 @@ import TabItem from '@theme/TabItem'; | instances.checks.active.unhealthy.timeout | integer | 否 | 3 | 1 到 254（包含） | 定义不健康节点的探测超时次数。 | | semantic_opts | object | 否 | | | `semantic` 均衡器的选项，集中配置在一处。当 `balancer.algorithm` 为 `semantic` 时必填。 | | semantic_opts.threshold | number | 否 | 0 | -1 到 1 | 实例被选中所需达到的全局最小余弦相似度。**默认值 0 几乎会接纳任何提示词**（真实嵌入向量很少会不相似到得分低于 0），因此只有把它调到 0 以上，回退实例才会真正生效。当没有实例达到其阈值时，请求回退到 `fallback` 实例；若未指定，则回退到第一个实例。 | -| semantic_opts.fallback | string | 否 | | | 当没有实例达到其阈值或嵌入请求失败时，路由到的实例名称。与参与排名的实例不同，它无需配置 `examples`。未设置时默认回退到第一个实例。 | -| semantic_opts.debugging | boolean | 否 | false | | 若为 true，通过 `X-AI-Semantic-Scores` 和 `X-AI-Semantic-Route` 响应头暴露各实例的分数和路由决策，用于调试。 | +| semantic_opts.fallback | string | 否 | | | 当没有实例达到其阈值或嵌入请求失败时，路由到的实例名称。它本身也是一个参与排名的普通实例，同样需要配置 `examples`。未设置时默认回退到第一个实例。 | +| semantic_opts.debugging | boolean | 否 | false | | 若为 true，通过 `X-AI-Semantic-Scores` 和 `X-AI-Semantic-Picked-Instance` 响应头暴露各实例的分数和路由决策，用于调试。 | | semantic_opts.embeddings | object | 是 | | | 嵌入服务配置，用于将提示词与各实例的 `examples` 进行相似度打分。 | | semantic_opts.embeddings.provider | string | 是 | | [openai, azure-openai] | 语义算法使用的嵌入服务提供商。 | | semantic_opts.embeddings.model | string | 是 | | | 嵌入模型名称，例如 `text-embedding-3-small`。 | @@ -2799,7 +2799,7 @@ nginx_config: 以下示例演示了如何使用 `semantic` 算法，根据请求提示词的语义而非权重或哈希来选择实例。每个参与排名的实例通过 `examples` 声明它应当处理的提示词类型；插件使用配置的嵌入模型将这些示例和传入的提示词嵌入为向量，并将请求转发给示例与之最相似的实例。 -创建如下路由，并相应地更新 LLM 提供商、嵌入模型和 API 密钥。`code` 实例处理编程类提示词，`translate` 实例处理翻译类提示词，`default` 实例被指定为 `semantic_opts.fallback`，当没有实例达到相似度阈值时使用： +创建如下路由，并相应地更新 LLM 提供商、嵌入模型和 API 密钥。`code` 实例处理编程类提示词，`translate` 实例处理翻译类提示词，`default` 实例处理通用提示词并被指定为 `semantic_opts.fallback`，当没有实例达到相似度阈值时兜底： ```shell curl "http://127.0.0.1:9180/apisix/admin/routes" -X PUT \ @@ -2871,7 +2871,11 @@ curl "http://127.0.0.1:9180/apisix/admin/routes" -X PUT \ }, "options": { "model": "gpt-4o-mini" - } + }, + "examples": [ + "what is the weather today", + "tell me a joke" + ] } ] } @@ -2892,7 +2896,7 @@ curl "http://127.0.0.1:9080/anything" -X POST \ }' ``` -该请求应被路由到 `code` 实例，由 `gpt-4o` 处理。如果提示词与任何实例的 `examples` 都不相关（例如 `what is the weather today`），则不会达到任何阈值，请求会被路由到 `default` 这个回退实例。 +该请求应被路由到 `code` 实例，由 `gpt-4o` 处理。像 `what is the weather today` 这样的通用提示词则会匹配到 `default` 实例自己的 `examples`。`default` 同时被指定为 `semantic_opts.fallback`，因此当提示词过于模糊、没有任何实例达到阈值时，也会由它兜底接收。 #### 调试路由决策 @@ -2906,17 +2910,17 @@ curl -i "http://127.0.0.1:9080/anything" -X POST \ ```text HTTP/1.1 200 OK -X-AI-Semantic-Route: code -X-AI-Semantic-Scores: code:0.8213,translate:0.1904 +X-AI-Semantic-Picked-Instance: code +X-AI-Semantic-Scores: code:0.8213,default:0.2451,translate:0.1904 ``` -`X-AI-Semantic-Scores` 按得分从高到低列出**所有配置了 `examples` 的实例**，因此你不仅能看到胜出者，也能看到其他实例差了多少。没有配置 `examples` 的实例（例如纯回退实例）没有相似度得分，因此不会出现在其中。 +`X-AI-Semantic-Scores` 按得分从高到低列出**所有实例**，因此你不仅能看到胜出者，也能看到其他实例差了多少。`default` 回退实例与其他实例一起参与排名（它同样需要配置 `examples`），并在没有任何实例达到阈值时额外承担兜底。 当没有任何实例达到阈值时，选中结果变为 `fallback`，而得分仍会显示各实例的接近程度： ```text -X-AI-Semantic-Route: fallback -X-AI-Semantic-Scores: code:0.3057,translate:0.2884 +X-AI-Semantic-Picked-Instance: fallback +X-AI-Semantic-Scores: code:0.3057,translate:0.2884,default:0.2013 ``` 据此可以校准 `threshold`：取一个介于「希望匹配的提示词得分」与「希望落到回退实例的提示词得分」之间的值。 diff --git a/t/plugin/ai-proxy-semantic-routing.t b/t/plugin/ai-proxy-semantic-routing.t index 5c29bec17e6e..7a7b3a58ba60 100644 --- a/t/plugin/ai-proxy-semantic-routing.t +++ b/t/plugin/ai-proxy-semantic-routing.t @@ -75,11 +75,19 @@ add_block_preprocessor(sub { local function vec(text) text = string.lower(text or "") if text:find("code") or text:find("python") or text:find("debug") then - return {1, 0} + return {1, 0, 0} elseif text:find("translate") or text:find("summar") then - return {0, 1} + return {0, 1, 0} + elseif text:find("weather") or text:find("joke") then + -- the `default` instance's own domain (general chat), + -- so it competes on similarity like any other instance + return {0, 0, 1} end - return {0.6, 0.6} + -- A prompt matching no instance's domain: its components + -- are distinct and none aligns with an axis, so it clears + -- no high threshold and the request falls back. Distinct + -- values also keep the reported score order deterministic. + return {0.7, 0.5, 0.3} end local data = {} for i, t in ipairs(body.input or {}) do @@ -88,8 +96,8 @@ add_block_preprocessor(sub { -- to exercise fail-open on malformed data data[i] = { index = i - 1, embedding = json.null } elseif string.lower(t):find("dimmismatch") then - -- 3-D vector vs the 2-D reference vectors - data[i] = { index = i - 1, embedding = {1, 0, 0} } + -- 2-D vector vs the 3-D reference vectors + data[i] = { index = i - 1, embedding = {1, 0} } else data[i] = { index = i - 1, embedding = vec(t) } end @@ -199,7 +207,8 @@ __DATA__ instances = { inst("code", "model-code", {"write python code"}), inst("cheap", "model-cheap", {"translate this text"}), - inst("default", "model-fallback"), + inst("default", "model-fallback", + {"what is the weather today", "tell me a joke"}), }, ssl_verify = false, }, @@ -251,13 +260,13 @@ model-cheap === TEST 4: unrelated prompt clears no threshold, falls back to the fallback instance --- request POST /anything -{"model":"auto","messages":[{"role":"user","content":"what is the weather today"}]} +{"model":"auto","messages":[{"role":"user","content":"recommend a good hotel in paris"}]} --- more_headers Content-Type: application/json --- response_body chomp model-fallback --- error_log eval -qr/no instance cleared threshold \(scores: code:0\.\d+,cheap:0\.\d+\)/ +qr/no instance cleared threshold \(scores: code:0\.\d+,cheap:0\.\d+,default:0\.\d+\)/ --- no_error_log [error] @@ -317,7 +326,8 @@ semantic routing: query embedding failed instances = { inst("code", "model-code", {"write python code"}), inst("cheap", "model-cheap", {"translate this text"}), - inst("default", "model-fallback"), + inst("default", "model-fallback", + {"what is the weather today", "tell me a joke"}), }, ssl_verify = false, }, @@ -349,8 +359,8 @@ Content-Type: application/json --- response_body chomp model-code --- response_headers_like -X-AI-Semantic-Route: code -X-AI-Semantic-Scores: code:1\.\d+,cheap:0\.\d+ +X-AI-Semantic-Picked-Instance: code +X-AI-Semantic-Scores: ^code:1\.\d+ --- no_error_log [error] @@ -490,7 +500,8 @@ invalid embedding entry at index 0 -- embedding this instance's examples makes the mock -- fail the whole reference batch inst("code", "model-code", {"servererror example"}), - inst("default", "model-fallback"), + inst("default", "model-fallback", + {"what is the weather today", "tell me a joke"}), }, ssl_verify = false, }, @@ -566,7 +577,8 @@ failed to fetch reference embeddings }, instances = { inst("code", "model-code", {"write python code"}), - inst("default", "model-fallback"), + inst("default", "model-fallback", + {"what is the weather today", "tell me a joke"}), }, ssl_verify = false, }, @@ -687,8 +699,9 @@ no instance cleared threshold plugins = { ["ai-proxy-multi"] = { -- the global threshold is permissive (0.1); the code - -- instance raises its own bar to 0.9, so a prompt scoring - -- ~0.707 clears the global one but not the instance's + -- instance raises its own bar to 0.9, so a prompt that + -- partially matches it (~0.77) clears the global threshold + -- but not the instance's, and the request goes to default balancer = { algorithm = "semantic" }, semantic_opts = { embeddings = { @@ -719,6 +732,7 @@ no instance cleared threshold override = { endpoint = "http://127.0.0.1:6798/v1/chat/completions", }, + examples = {"what is the weather today", "tell me a joke"}, }, }, ssl_verify = false, @@ -745,7 +759,7 @@ passed === TEST 21: a prompt below the per-instance threshold reaches the fallback --- request POST /anything -{"model":"auto","messages":[{"role":"user","content":"what is the weather today"}]} +{"model":"auto","messages":[{"role":"user","content":"recommend a good hotel in paris"}]} --- more_headers Content-Type: application/json --- response_body chomp @@ -793,7 +807,8 @@ model-fallback instances = { inst("code", "model-code", {"write python code"}), inst("cheap", "model-cheap", {"translate this text"}), - inst("default", "model-fallback"), + inst("default", "model-fallback", + {"what is the weather today", "tell me a joke"}), }, ssl_verify = false, }, @@ -823,8 +838,8 @@ POST /llm/v1/responses --- more_headers Content-Type: application/json --- response_headers_like -X-AI-Semantic-Route: code -X-AI-Semantic-Scores: code:1\.\d+,cheap:0\.\d+ +X-AI-Semantic-Picked-Instance: code +X-AI-Semantic-Scores: ^code:1\.\d+ --- no_error_log [error] @@ -868,7 +883,8 @@ X-AI-Semantic-Scores: code:1\.\d+,cheap:0\.\d+ instances = { inst("code", "model-code", {"write python code"}), inst("cheap", "model-cheap", {"translate this text"}), - inst("default", "model-fallback"), + inst("default", "model-fallback", + {"what is the weather today", "tell me a joke"}), }, ssl_verify = false, }, @@ -898,8 +914,8 @@ POST /llm/v1/messages --- more_headers Content-Type: application/json --- response_headers_like -X-AI-Semantic-Route: code -X-AI-Semantic-Scores: code:1\.\d+,cheap:0\.\d+ +X-AI-Semantic-Picked-Instance: code +X-AI-Semantic-Scores: ^code:1\.\d+ @@ -942,6 +958,7 @@ X-AI-Semantic-Scores: code:1\.\d+,cheap:0\.\d+ override = { endpoint = "http://127.0.0.1:6798/v1/chat/completions", }, + examples = {"what is the weather today", "tell me a joke"}, }, }, ssl_verify = false, diff --git a/t/plugin/ai-proxy-semantic-schema.t b/t/plugin/ai-proxy-semantic-schema.t index 3be5041817ec..da756fe3b937 100644 --- a/t/plugin/ai-proxy-semantic-schema.t +++ b/t/plugin/ai-proxy-semantic-schema.t @@ -63,7 +63,8 @@ __DATA__ { "name": "default", "provider": "openai", "weight": 1, "auth": { "header": { "Authorization": "Bearer token" } }, - "options": { "model": "gpt-4o-mini" } + "options": { "model": "gpt-4o-mini" }, + "examples": ["what is the weather today", "tell me a joke"] } ], "ssl_verify": false @@ -264,7 +265,7 @@ passed -=== TEST 7: only the named fallback is exempt from the examples requirement +=== TEST 7: the named fallback is not exempt from the examples requirement --- config location /t { content_by_lua_block { @@ -285,7 +286,8 @@ passed { "name": "a", "provider": "openai", "weight": 1, "auth": { "header": { "Authorization": "Bearer token" } }, - "options": { "model": "gpt-4o" } + "options": { "model": "gpt-4o" }, + "examples": ["write code"] }, { "name": "default", "provider": "openai", "weight": 1, From 8c4f10f3859fcdbfc839aa365c301cbf682a22c8 Mon Sep 17 00:00:00 2001 From: AlinsRan Date: Fri, 24 Jul 2026 11:09:32 +0800 Subject: [PATCH 8/8] test(ai-proxy-multi): fix semantic score header assertion to full-match all instances response_headers_like anchors the pattern with ^...$, so the value must match in full. With the fallback now carrying examples, three instances appear in X-AI-Semantic-Scores; match the winner plus the two zero-scored instances in any order. --- t/plugin/ai-proxy-semantic-routing.t | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/t/plugin/ai-proxy-semantic-routing.t b/t/plugin/ai-proxy-semantic-routing.t index 7a7b3a58ba60..47afb6d19cdc 100644 --- a/t/plugin/ai-proxy-semantic-routing.t +++ b/t/plugin/ai-proxy-semantic-routing.t @@ -360,7 +360,7 @@ Content-Type: application/json model-code --- response_headers_like X-AI-Semantic-Picked-Instance: code -X-AI-Semantic-Scores: ^code:1\.\d+ +X-AI-Semantic-Scores: code:1\.\d+(,\w+:0\.\d+){2} --- no_error_log [error] @@ -839,7 +839,7 @@ POST /llm/v1/responses Content-Type: application/json --- response_headers_like X-AI-Semantic-Picked-Instance: code -X-AI-Semantic-Scores: ^code:1\.\d+ +X-AI-Semantic-Scores: code:1\.\d+(,\w+:0\.\d+){2} --- no_error_log [error] @@ -915,7 +915,7 @@ POST /llm/v1/messages Content-Type: application/json --- response_headers_like X-AI-Semantic-Picked-Instance: code -X-AI-Semantic-Scores: ^code:1\.\d+ +X-AI-Semantic-Scores: code:1\.\d+(,\w+:0\.\d+){2}