From fae44bec1133a25d45d94a7431f512c2163b1bc7 Mon Sep 17 00:00:00 2001 From: Povilas Kanapickas Date: Sat, 13 Jun 2026 10:54:09 +0000 Subject: [PATCH 1/4] chore: upgrade @anthropic-ai/sdk to 0.104.1 and @anthropic-ai/vertex-sdk to 0.17.1 --- pnpm-lock.yaml | 143 +++++++++-------------- src/api/providers/native-ollama.ts | 5 +- src/api/providers/openai-codex.ts | 6 +- src/api/providers/openai-native.ts | 6 +- src/api/transform/mistral-format.ts | 18 ++- src/api/transform/openai-format.ts | 19 ++- src/api/transform/r1-format.ts | 10 +- src/api/transform/responses-api-input.ts | 12 +- src/api/transform/vscode-lm-format.ts | 24 +++- src/api/transform/zai-format.ts | 10 +- src/integrations/misc/export-markdown.ts | 12 +- src/package.json | 4 +- 12 files changed, 144 insertions(+), 125 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c890389339..d19e879b22 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -455,11 +455,11 @@ importers: specifier: ^3.0.48 version: 3.0.48(zod@3.25.76) '@anthropic-ai/sdk': - specifier: ^0.37.0 - version: 0.37.0 + specifier: ^0.104.1 + version: 0.104.1(zod@3.25.76) '@anthropic-ai/vertex-sdk': - specifier: ^0.7.0 - version: 0.7.0 + specifier: ^0.17.1 + version: 0.17.1(zod@3.25.76) '@aws-sdk/client-bedrock-runtime': specifier: ^3.922.0 version: 3.922.0 @@ -1142,11 +1142,17 @@ packages: '@antfu/install-pkg@1.1.0': resolution: {integrity: sha512-MGQsmw10ZyI+EJo45CdSER4zEb+p31LpDAFp2Z3gkSd1yqVZGi0Ebx++YTEMonJy4oChEMLsxZ64j8FH6sSqtQ==} - '@anthropic-ai/sdk@0.37.0': - resolution: {integrity: sha512-tHjX2YbkUBwEgg0JZU3EFSSAQPoK4qQR/NFYa8Vtzd5UAyXzZksCw2In69Rml4R/TyHPBfRYaLK35XiOe33pjw==} + '@anthropic-ai/sdk@0.104.1': + resolution: {integrity: sha512-gGACa/+IaiXzRRmF96aOhamoBgapKRBiFWbmmTFP8aMkpaEcuStF+Q61bjo4vPxBM7gqWJNZqsngslRdnLHv0Q==} + hasBin: true + peerDependencies: + zod: 3.25.76 + peerDependenciesMeta: + zod: + optional: true - '@anthropic-ai/vertex-sdk@0.7.0': - resolution: {integrity: sha512-zNm3hUXgYmYDTyveIxOyxbcnh5VXFkrLo4bSnG6LAfGzW7k3k2iCNDSVKtR9qZrK2BCid7JtVu7jsEKaZ/9dSw==} + '@anthropic-ai/vertex-sdk@0.17.1': + resolution: {integrity: sha512-+am1LjqEpb7Qq/fKDdu0PTE2U9+XSPXSSr9Fa5dvJ8FtJ5L9XOroNBnr7iecT+6iGlbiV6BYgMIGkAwPuly/jA==} '@asamuzakjp/css-color@3.2.0': resolution: {integrity: sha512-K1A6z8tS3XsmCMM86xoWdn7Fkdn9m6RSVtocUrJYIwZnFVkng/PvkEoWtOWmP+Scc6saYWHWZYbndEEXxl24jw==} @@ -3251,6 +3257,9 @@ packages: resolution: {integrity: sha512-4aUIteuyxtBUhVdiQqcDhKFitwfd9hqoSDYY2KRXiWtgoWJ9Bmise+KfEPDiVHWeJepvF8xJO9/9+WDIciMFFw==} engines: {node: '>=18.0.0'} + '@stablelib/base64@1.0.1': + resolution: {integrity: sha512-1bnPQqSxSuc3Ii6MhBysoWCg58j97aUjuCSZrGSmDxNqtytIi0k8utUenAwTZN4V5mXXYGsVUI9zeBqy+jBOSQ==} + '@standard-schema/spec@1.1.0': resolution: {integrity: sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==} @@ -3621,9 +3630,6 @@ packages: resolution: {integrity: sha512-faK2Owokboz53g8ooq2dw3iDJ6/HMTCIa2RvMte5WMTiABy+wA558K+iuyRtlR67Un5q9gEKysSDtqZYbSa0Pg==} deprecated: This is a stub types definition. node-cache provides its own type definitions, so you do not need this installed. - '@types/node-fetch@2.6.12': - resolution: {integrity: sha512-8nneRWKCg3rMtF69nLQJnOYUcbafYeFSjqkw3jCRLsqkWFlHaoQrr5mXmofFGOx3DKn7UfmBMyov8ySvLRVldA==} - '@types/node-ipc@9.2.3': resolution: {integrity: sha512-/MvSiF71fYf3+zwqkh/zkVkZj1hl1Uobre9EMFy08mqfJNAmpR0vmPgOUdEIDVgifxHj6G1vYMPLSBLLxoDACQ==} @@ -3633,9 +3639,6 @@ packages: '@types/node@14.18.63': resolution: {integrity: sha512-fAtCfv4jJg+ExtXhvCkCqUKZ+4ok/JQk01qDKhL5BDDoS3AxKXhV5/MAVUZyQnSEd2GT92fkgZl0pz0Q0AzcIQ==} - '@types/node@18.19.100': - resolution: {integrity: sha512-ojmMP8SZBKprc3qGrGk8Ujpo80AXkrP7G2tOT4VWr5jlr5DHjsJF+emXJz+Wm0glmy4Js62oKMdZZ6B9Y+tEcA==} - '@types/node@20.19.43': resolution: {integrity: sha512-6oYBAi5ikg4Pl+kGsoYtawUMBT2zZMCvPNF7pVLnHZfd1zf38DRiWn/gT01RYCdUqkv7Fhr+C9ot4/tb+2sVvA==} @@ -3889,10 +3892,6 @@ packages: '@xobotyi/scrollbar-width@1.9.5': resolution: {integrity: sha512-N8tkAACJx2ww8vFMneJmaAgmjAG1tnVBZJRLRcx061tmsLRZHSEZSLuGWnwPtunsSLvSqXQ2wfp7Mgqg1I+2dQ==} - abort-controller@3.0.0: - resolution: {integrity: sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==} - engines: {node: '>=6.5'} - accepts@2.0.0: resolution: {integrity: sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==} engines: {node: '>= 0.6'} @@ -3915,10 +3914,6 @@ packages: resolution: {integrity: sha512-jRR5wdylq8CkOe6hei19GGZnxM6rBGwFl3Bg0YItGDimvjGtAvdZk4Pu6Cl4u4Igsws4a1fd1Vq3ezrhn4KmFw==} engines: {node: '>= 14'} - agentkeepalive@4.6.0: - resolution: {integrity: sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ==} - engines: {node: '>= 8.0.0'} - ai-sdk-provider-poe@2.0.18: resolution: {integrity: sha512-uzFR5Zq+PD6LfrqpYAr4dt2KmfOfS9a0Wh8QJsOaGI/vOByhW02P/0mj0NtOnOWF1RIUfBkQJo647XjrgnFjjg==} peerDependencies: @@ -5133,10 +5128,6 @@ packages: event-stream@3.3.4: resolution: {integrity: sha512-QHpkERcGsR0T7Qm3HNJSyXKEEj8AHNxkY3PK8TS2KJvQ7NiSHe3DDpwVKKtoYprL/AreyzFBeIkBIWChAqn60g==} - event-target-shim@5.0.1: - resolution: {integrity: sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==} - engines: {node: '>=6'} - eventemitter3@5.0.4: resolution: {integrity: sha512-mlsTRyGaPBjPedk6Bvw+aqbsXDtoAyAzm5MO7JgU+yVRyMQ5O8bD4Kcci7BS85f93veegeCPkL8R4GLClnjLFw==} @@ -5224,6 +5215,9 @@ packages: fast-levenshtein@2.0.6: resolution: {integrity: sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==} + fast-sha256@1.3.0: + resolution: {integrity: sha512-n11RGP/lrWEFI/bWdygLxhI+pVeo1ZYIVwvvPkW7azl/rOy+F3HYRZ2K5zeE9mmkhQppyv9sQFx0JM9UabnpPQ==} + fast-shallow-equal@1.0.0: resolution: {integrity: sha512-HPtaa38cPgWvaCFmRNhlc6NG7pv6NUHqjPgVAkWGoB9mQMwYB27/K0CvOM5Czy+qpT3e8XJ6Q4aPAnzpNpzNaw==} @@ -5324,9 +5318,6 @@ packages: resolution: {integrity: sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==} engines: {node: '>=14'} - form-data-encoder@1.7.2: - resolution: {integrity: sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==} - form-data@4.0.6: resolution: {integrity: sha512-vKatAh4SlVfgbv+YtmhiRjhEMJsYpsG1Y2rMQtR+SVSbytsSD1YGzDIcrAJmdFec88u/+VoGmxnl+80gL1tRCQ==} engines: {node: '>= 6'} @@ -5336,10 +5327,6 @@ packages: engines: {node: '>=18.3.0'} hasBin: true - formdata-node@4.4.1: - resolution: {integrity: sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==} - engines: {node: '>= 12.20'} - formdata-polyfill@4.0.10: resolution: {integrity: sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==} engines: {node: '>=12.20.0'} @@ -5683,9 +5670,6 @@ packages: resolution: {integrity: sha512-eKCa6bwnJhvxj14kZk5NCPc6Hb6BdsU9DZcOnmQKSnO1VKrfV0zCvtttPZUsBvjmNDn8rpcJfpwSYnHBjc95MQ==} engines: {node: '>=18.18.0'} - humanize-ms@1.2.1: - resolution: {integrity: sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==} - husky@9.1.7: resolution: {integrity: sha512-5gs5ytaNjBrh5Ow3zrvdUUY+0VxIuWVL4i9irt6friV+BqdCfmV11CQTWMiBYWHbXhco+J1kHfTOUkePhCDvMA==} engines: {node: '>=18'} @@ -6155,6 +6139,10 @@ packages: json-buffer@3.0.1: resolution: {integrity: sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==} + json-schema-to-ts@3.1.1: + resolution: {integrity: sha512-+DWg8jCJG2TEnpy7kOm/7/AxaYoaRbjVB4LFZLySZlWn8exGs3A4OLJR966cVvU26N7X9TWxl+Jsw7dzAqKT6g==} + engines: {node: '>=16'} + json-schema-traverse@0.4.1: resolution: {integrity: sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==} @@ -8063,6 +8051,9 @@ packages: standard-as-callback@2.1.0: resolution: {integrity: sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==} + standardwebhooks@1.0.0: + resolution: {integrity: sha512-BbHGOQK9olHPMvQNHWul6MYlrRTAOKn03rOe4A8O3CLWhNf4YHBqq2HJKKC+sfqpxiBY52pNeesD6jIiLDz8jg==} + statuses@2.0.2: resolution: {integrity: sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==} engines: {node: '>= 0.8'} @@ -8398,6 +8389,9 @@ packages: truncate-utf8-bytes@1.0.2: resolution: {integrity: sha512-95Pu1QXQvruGEhv62XCMO3Mm90GscOCClvrIUwCM0PYOXK3kaF3l3sIHxx71ThJfcbM2O5Au6SO3AWCSEfW4mQ==} + ts-algebra@2.0.0: + resolution: {integrity: sha512-FPAhNPFMrkwz76P7cdjdmiShwMynZYN6SgOujD1urY4oNm80Ou9oMdmbR45LotcKOXoy7wSmHkRFE6Mxbrhefw==} + ts-api-utils@2.1.0: resolution: {integrity: sha512-CUgTZL1irw8u29bzrOD/nH85jqyc74D6SshFgujOIA7osm2Rz7dYH77agkx7H4FBNxDq7Cjf+IjaX/8zwFW+ZQ==} engines: {node: '>=18.12'} @@ -8518,9 +8512,6 @@ packages: underscore@1.13.8: resolution: {integrity: sha512-DXtD3ZtEQzc7M8m4cXotyHR+FAS18C64asBYY5vqZexfYryNNnDc02W4hKg3rdQuqOYas1jkseX0+nZXjTXnvQ==} - undici-types@5.26.5: - resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==} - undici-types@6.21.0: resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==} @@ -8831,10 +8822,6 @@ packages: resolution: {integrity: sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==} engines: {node: '>= 8'} - web-streams-polyfill@4.0.0-beta.3: - resolution: {integrity: sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==} - engines: {node: '>= 14'} - web-tree-sitter@0.25.6: resolution: {integrity: sha512-WG+/YGbxw8r+rLlzzhV+OvgiOJCWdIpOucG3qBf3RCBFMkGDb1CanUi2BxCxjnkpzU3/hLWPT8VO5EKsMk9Fxg==} @@ -9250,25 +9237,21 @@ snapshots: package-manager-detector: 1.6.0 tinyexec: 1.2.4 - '@anthropic-ai/sdk@0.37.0': + '@anthropic-ai/sdk@0.104.1(zod@3.25.76)': dependencies: - '@types/node': 18.19.100 - '@types/node-fetch': 2.6.12 - abort-controller: 3.0.0 - agentkeepalive: 4.6.0 - form-data-encoder: 1.7.2 - formdata-node: 4.4.1 - node-fetch: 2.7.0 - transitivePeerDependencies: - - encoding + json-schema-to-ts: 3.1.1 + standardwebhooks: 1.0.0 + optionalDependencies: + zod: 3.25.76 - '@anthropic-ai/vertex-sdk@0.7.0': + '@anthropic-ai/vertex-sdk@0.17.1(zod@3.25.76)': dependencies: - '@anthropic-ai/sdk': 0.37.0 + '@anthropic-ai/sdk': 0.104.1(zod@3.25.76) google-auth-library: 9.15.1 transitivePeerDependencies: - encoding - supports-color + - zod '@asamuzakjp/css-color@3.2.0': dependencies: @@ -11836,6 +11819,8 @@ snapshots: dependencies: tslib: 2.8.1 + '@stablelib/base64@1.0.1': {} + '@standard-schema/spec@1.1.0': {} '@tailwindcss/node@4.1.6': @@ -12227,11 +12212,6 @@ snapshots: dependencies: node-cache: 5.1.2 - '@types/node-fetch@2.6.12': - dependencies: - '@types/node': 20.19.43 - form-data: 4.0.6 - '@types/node-ipc@9.2.3': dependencies: '@types/node': 20.19.43 @@ -12240,10 +12220,6 @@ snapshots: '@types/node@14.18.63': {} - '@types/node@18.19.100': - dependencies: - undici-types: 5.26.5 - '@types/node@20.19.43': dependencies: undici-types: 6.21.0 @@ -12580,10 +12556,6 @@ snapshots: '@xobotyi/scrollbar-width@1.9.5': {} - abort-controller@3.0.0: - dependencies: - event-target-shim: 5.0.1 - accepts@2.0.0: dependencies: mime-types: 3.0.1 @@ -12603,10 +12575,6 @@ snapshots: agent-base@7.1.3: {} - agentkeepalive@4.6.0: - dependencies: - humanize-ms: 1.2.1 - ai-sdk-provider-poe@2.0.18(ai@6.0.77(zod@3.25.76))(zod@3.25.76): dependencies: '@ai-sdk/anthropic': 3.0.42(zod@3.25.76) @@ -13992,8 +13960,6 @@ snapshots: stream-combiner: 0.0.4 through: 2.3.8 - event-target-shim@5.0.1: {} - eventemitter3@5.0.4: {} eventsource-parser@3.0.8: {} @@ -14144,6 +14110,8 @@ snapshots: fast-levenshtein@2.0.6: {} + fast-sha256@1.3.0: {} + fast-shallow-equal@1.0.0: {} fast-uri@3.1.0: {} @@ -14240,8 +14208,6 @@ snapshots: cross-spawn: 7.0.6 signal-exit: 4.1.0 - form-data-encoder@1.7.2: {} - form-data@4.0.6: dependencies: asynckit: 0.4.0 @@ -14254,11 +14220,6 @@ snapshots: dependencies: fd-package-json: 2.0.0 - formdata-node@4.4.1: - dependencies: - node-domexception: 1.0.0 - web-streams-polyfill: 4.0.0-beta.3 - formdata-polyfill@4.0.10: dependencies: fetch-blob: 3.2.0 @@ -14716,10 +14677,6 @@ snapshots: human-signals@8.0.1: {} - humanize-ms@1.2.1: - dependencies: - ms: 2.1.3 - husky@9.1.7: {} hyphenate-style-name@1.1.0: {} @@ -15177,6 +15134,11 @@ snapshots: json-buffer@3.0.1: {} + json-schema-to-ts@3.1.1: + dependencies: + '@babel/runtime': 7.29.7 + ts-algebra: 2.0.0 + json-schema-traverse@0.4.1: {} json-schema-traverse@1.0.0: {} @@ -17511,6 +17473,11 @@ snapshots: standard-as-callback@2.1.0: {} + standardwebhooks@1.0.0: + dependencies: + '@stablelib/base64': 1.0.1 + fast-sha256: 1.3.0 + statuses@2.0.2: {} std-env@4.1.0: {} @@ -17866,6 +17833,8 @@ snapshots: dependencies: utf8-byte-length: 1.0.5 + ts-algebra@2.0.0: {} + ts-api-utils@2.1.0(typescript@5.8.3): dependencies: typescript: 5.8.3 @@ -18011,8 +17980,6 @@ snapshots: underscore@1.13.8: {} - undici-types@5.26.5: {} - undici-types@6.21.0: {} undici@6.27.0: {} @@ -18291,8 +18258,6 @@ snapshots: web-streams-polyfill@3.3.3: {} - web-streams-polyfill@4.0.0-beta.3: {} - web-tree-sitter@0.25.6: {} web-vitals@4.2.4: {} diff --git a/src/api/providers/native-ollama.ts b/src/api/providers/native-ollama.ts index 7ee91282a4..4574c184f2 100644 --- a/src/api/providers/native-ollama.ts +++ b/src/api/providers/native-ollama.ts @@ -60,7 +60,10 @@ function convertToOllamaMessages(anthropicMessages: Anthropic.Messages.MessagePa } return "(see following user message for image)" } - return part.text + if (part.type === "text") { + return part.text + } + return "" }) .join("\n") ?? "" } diff --git a/src/api/providers/openai-codex.ts b/src/api/providers/openai-codex.ts index bc9d4cd26a..2b2a599c0f 100644 --- a/src/api/providers/openai-codex.ts +++ b/src/api/providers/openai-codex.ts @@ -429,8 +429,10 @@ export class OpenAiCodexHandler extends BaseProvider implements SingleCompletion content.push({ type: "input_text", text: block.text }) } else if (block.type === "image") { const image = block as Anthropic.Messages.ImageBlockParam - const imageUrl = `data:${image.source.media_type};base64,${image.source.data}` - content.push({ type: "input_image", image_url: imageUrl }) + if (image.source.type === "base64") { + const imageUrl = `data:${image.source.media_type};base64,${image.source.data}` + content.push({ type: "input_image", image_url: imageUrl }) + } } else if (block.type === "tool_result") { const result = typeof block.content === "string" diff --git a/src/api/providers/openai-native.ts b/src/api/providers/openai-native.ts index ea7a0667f3..8f86f46751 100644 --- a/src/api/providers/openai-native.ts +++ b/src/api/providers/openai-native.ts @@ -483,8 +483,10 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio content.push({ type: "input_text", text: block.text }) } else if (block.type === "image") { const image = block as Anthropic.Messages.ImageBlockParam - const imageUrl = `data:${image.source.media_type};base64,${image.source.data}` - content.push({ type: "input_image", image_url: imageUrl }) + if (image.source.type === "base64") { + const imageUrl = `data:${image.source.media_type};base64,${image.source.data}` + content.push({ type: "input_image", image_url: imageUrl }) + } } else if (block.type === "tool_result") { // Map Anthropic tool_result to Responses API function_call_output item const result = diff --git a/src/api/transform/mistral-format.ts b/src/api/transform/mistral-format.ts index d32f84d6e0..3f89a51e0e 100644 --- a/src/api/transform/mistral-format.ts +++ b/src/api/transform/mistral-format.ts @@ -104,14 +104,20 @@ export function convertToMistralMessages(anthropicMessages: Anthropic.Messages.M role: "user", content: nonToolMessages.map((part) => { if (part.type === "image") { - return { - type: "image_url", - imageUrl: { - url: `data:${part.source.media_type};base64,${part.source.data}`, - }, + if (part.source.type === "base64") { + return { + type: "image_url", + imageUrl: { + url: `data:${part.source.media_type};base64,${part.source.data}`, + }, + } } + return { type: "text", text: "[Image]" } } - return { type: "text", text: part.text } + if (part.type === "text") { + return { type: "text", text: part.text } + } + return { type: "text", text: "" } }), }) } diff --git a/src/api/transform/openai-format.ts b/src/api/transform/openai-format.ts index 38719feba1..f03778b425 100644 --- a/src/api/transform/openai-format.ts +++ b/src/api/transform/openai-format.ts @@ -361,7 +361,10 @@ export function convertToOpenAiMessages( toolResultImages.push(part) return "(see following user message for image)" } - return part.text + if (part.type === "text") { + return part.text + } + return "" }) .join("\n") ?? "" } @@ -422,12 +425,18 @@ export function convertToOpenAiMessages( role: "user", content: filteredNonToolMessages.map((part) => { if (part.type === "image") { - return { - type: "image_url", - image_url: { url: `data:${part.source.media_type};base64,${part.source.data}` }, + if (part.source.type === "base64") { + return { + type: "image_url", + image_url: { url: `data:${part.source.media_type};base64,${part.source.data}` }, + } } + return { type: "text", text: "[Image]" } + } + if (part.type === "text") { + return { type: "text", text: part.text } } - return { type: "text", text: part.text } + return { type: "text", text: "" } }), }) } diff --git a/src/api/transform/r1-format.ts b/src/api/transform/r1-format.ts index 6e8b782047..e59b282be2 100644 --- a/src/api/transform/r1-format.ts +++ b/src/api/transform/r1-format.ts @@ -61,10 +61,12 @@ export function convertToR1Format( if (part.type === "text") { textParts.push(part.text) } else if (part.type === "image") { - imageParts.push({ - type: "image_url", - image_url: { url: `data:${part.source.media_type};base64,${part.source.data}` }, - }) + if (part.source.type === "base64") { + imageParts.push({ + type: "image_url", + image_url: { url: `data:${part.source.media_type};base64,${part.source.data}` }, + }) + } } else if (part.type === "tool_result") { // Convert tool_result to OpenAI tool message format let content: string diff --git a/src/api/transform/responses-api-input.ts b/src/api/transform/responses-api-input.ts index a766dfef6e..9ef31be280 100644 --- a/src/api/transform/responses-api-input.ts +++ b/src/api/transform/responses-api-input.ts @@ -73,11 +73,13 @@ export function convertToResponsesApiInput(messages: Anthropic.Messages.MessageP contentParts.push({ type: "input_text", text: part.text }) break case "image": - contentParts.push({ - type: "input_image", - detail: "auto", - image_url: `data:${part.source.media_type};base64,${part.source.data}`, - }) + if (part.source.type === "base64") { + contentParts.push({ + type: "input_image", + detail: "auto", + image_url: `data:${part.source.media_type};base64,${part.source.data}`, + }) + } break case "tool_result": { // Flush any pending user content before the tool result diff --git a/src/api/transform/vscode-lm-format.ts b/src/api/transform/vscode-lm-format.ts index 388197c2c2..a9273b9a40 100644 --- a/src/api/transform/vscode-lm-format.ts +++ b/src/api/transform/vscode-lm-format.ts @@ -72,11 +72,19 @@ export function convertToVsCodeLmMessages( ? [new vscode.LanguageModelTextPart(toolMessage.content)] : (toolMessage.content?.map((part) => { if (part.type === "image") { + if (part.source.type === "base64") { + return new vscode.LanguageModelTextPart( + `[Image (${part.source?.type || "Unknown source-type"}): ${part.source?.media_type || "unknown media-type"} not supported by VSCode LM API]`, + ) + } return new vscode.LanguageModelTextPart( - `[Image (${part.source?.type || "Unknown source-type"}): ${part.source?.media_type || "unknown media-type"} not supported by VSCode LM API]`, + `[Image (${part.source?.type || "Unknown source-type"}): not supported by VSCode LM API]`, ) } - return new vscode.LanguageModelTextPart(part.text) + if (part.type === "text") { + return new vscode.LanguageModelTextPart(part.text) + } + return new vscode.LanguageModelTextPart("") }) ?? [new vscode.LanguageModelTextPart("")]) return new vscode.LanguageModelToolResultPart(toolMessage.tool_use_id, toolContentParts) @@ -85,11 +93,19 @@ export function convertToVsCodeLmMessages( // Convert non-tool messages to TextParts after tool messages ...nonToolMessages.map((part) => { if (part.type === "image") { + if (part.source.type === "base64") { + return new vscode.LanguageModelTextPart( + `[Image (${part.source?.type || "Unknown source-type"}): ${part.source?.media_type || "unknown media-type"} not supported by VSCode LM API]`, + ) + } return new vscode.LanguageModelTextPart( - `[Image (${part.source?.type || "Unknown source-type"}): ${part.source?.media_type || "unknown media-type"} not supported by VSCode LM API]`, + `[Image (${part.source?.type || "Unknown source-type"}): not supported by VSCode LM API]`, ) } - return new vscode.LanguageModelTextPart(part.text) + if (part.type === "text") { + return new vscode.LanguageModelTextPart(part.text) + } + return new vscode.LanguageModelTextPart("") }), ] diff --git a/src/api/transform/zai-format.ts b/src/api/transform/zai-format.ts index 79b2e88aeb..0bc8353d8e 100644 --- a/src/api/transform/zai-format.ts +++ b/src/api/transform/zai-format.ts @@ -56,10 +56,12 @@ export function convertToZAiFormat( if (part.type === "text") { textParts.push(part.text) } else if (part.type === "image") { - imageParts.push({ - type: "image_url", - image_url: { url: `data:${part.source.media_type};base64,${part.source.data}` }, - }) + if (part.source.type === "base64") { + imageParts.push({ + type: "image_url", + image_url: { url: `data:${part.source.media_type};base64,${part.source.data}` }, + }) + } } else if (part.type === "tool_result") { // Convert tool_result to OpenAI tool message format let content: string diff --git a/src/integrations/misc/export-markdown.ts b/src/integrations/misc/export-markdown.ts index d65bb3200e..0636a9e7b8 100644 --- a/src/integrations/misc/export-markdown.ts +++ b/src/integrations/misc/export-markdown.ts @@ -13,7 +13,11 @@ interface ThoughtSignatureBlock { type: "thoughtSignature" } -export type ExtendedContentBlock = Anthropic.Messages.ContentBlockParam | ReasoningBlock | ThoughtSignatureBlock +export type ExtendedContentBlock = + | Anthropic.Messages.ContentBlockParam + | Anthropic.Messages.ToolReferenceBlockParam + | ReasoningBlock + | ThoughtSignatureBlock export function getTaskFileName(dateTs: number): string { const date = new Date(dateTs) @@ -69,6 +73,12 @@ export function formatContentBlockToMarkdown(block: ExtendedContentBlock): strin return block.text case "image": return `[Image]` + case "document": + return `[Document]` + case "search_result": + return `[Search Result]` + case "tool_reference": + return `[Tool Reference]` case "tool_use": { let input: string if (typeof block.input === "object" && block.input !== null) { diff --git a/src/package.json b/src/package.json index 3b7295722f..0b62cacf5c 100644 --- a/src/package.json +++ b/src/package.json @@ -465,8 +465,8 @@ "@ai-sdk/google-vertex": "^4.0.45", "@ai-sdk/mistral": "^3.0.19", "@ai-sdk/xai": "^3.0.48", - "@anthropic-ai/sdk": "^0.37.0", - "@anthropic-ai/vertex-sdk": "^0.7.0", + "@anthropic-ai/sdk": "^0.104.1", + "@anthropic-ai/vertex-sdk": "^0.17.1", "@aws-sdk/client-bedrock-runtime": "^3.922.0", "@aws-sdk/credential-providers": "^3.922.0", "@google/genai": "^1.29.1", From 843d235844c4a71d0848db15bcca40a1a5c682d0 Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Sat, 27 Jun 2026 17:13:06 +0000 Subject: [PATCH 2/4] fix(transform): handle URL-sourced images and non-base64 blocks in format converters --- .../__tests__/mistral-format.spec.ts | 39 ++++++++ .../transform/__tests__/openai-format.spec.ts | 91 +++++++++++++++++++ .../__tests__/vscode-lm-format.spec.ts | 42 +++++++++ .../transform/__tests__/zai-format.spec.ts | 79 ++++++++++++++++ src/api/transform/mistral-format.ts | 6 ++ src/api/transform/openai-format.ts | 17 +++- src/api/transform/vscode-lm-format.ts | 8 +- .../misc/__tests__/export-markdown.spec.ts | 27 ++++++ 8 files changed, 302 insertions(+), 7 deletions(-) create mode 100644 src/api/transform/__tests__/zai-format.spec.ts diff --git a/src/api/transform/__tests__/mistral-format.spec.ts b/src/api/transform/__tests__/mistral-format.spec.ts index 290bea1ec5..6f3984bcc7 100644 --- a/src/api/transform/__tests__/mistral-format.spec.ts +++ b/src/api/transform/__tests__/mistral-format.spec.ts @@ -106,6 +106,45 @@ describe("convertToMistralMessages", () => { }) }) + it("should handle user messages with URL image content", () => { + const anthropicMessages: Anthropic.Messages.MessageParam[] = [ + { + role: "user", + content: [ + { type: "text", text: "Describe this:" }, + { + type: "image", + source: { type: "url", url: "https://example.com/photo.jpg" } as any, + }, + ], + }, + ] + + const mistralMessages = convertToMistralMessages(anthropicMessages) + const content = mistralMessages[0].content as Array<{ type: string; imageUrl?: { url: string }; text?: string }> + + expect(content[1]).toEqual({ type: "image_url", imageUrl: { url: "https://example.com/photo.jpg" } }) + }) + + it("should fall back to [Image] placeholder for unsupported image source types", () => { + const anthropicMessages: Anthropic.Messages.MessageParam[] = [ + { + role: "user", + content: [ + { + type: "image", + source: { type: "file" } as any, + }, + ], + }, + ] + + const mistralMessages = convertToMistralMessages(anthropicMessages) + const content = mistralMessages[0].content as Array<{ type: string; text?: string }> + + expect(content[0]).toEqual({ type: "text", text: "[Image]" }) + }) + it("should handle user messages with only tool results", () => { const anthropicMessages: Anthropic.Messages.MessageParam[] = [ { diff --git a/src/api/transform/__tests__/openai-format.spec.ts b/src/api/transform/__tests__/openai-format.spec.ts index 1a4c7f6518..b02ac83c63 100644 --- a/src/api/transform/__tests__/openai-format.spec.ts +++ b/src/api/transform/__tests__/openai-format.spec.ts @@ -76,6 +76,97 @@ describe("convertToOpenAiMessages", () => { }) }) + it("should handle messages with URL image content", () => { + const anthropicMessages: Anthropic.Messages.MessageParam[] = [ + { + role: "user", + content: [ + { type: "text", text: "What is in this image?" }, + { + type: "image", + source: { type: "url", url: "https://example.com/image.png" } as any, + }, + ], + }, + ] + + const openAiMessages = convertToOpenAiMessages(anthropicMessages) + const content = openAiMessages[0].content as Array<{ type: string; text?: string; image_url?: { url: string } }> + + expect(content[1]).toEqual({ + type: "image_url", + image_url: { url: "https://example.com/image.png" }, + }) + }) + + it("should fall back to [Image] placeholder for unsupported image source types", () => { + const anthropicMessages: Anthropic.Messages.MessageParam[] = [ + { + role: "user", + content: [ + { + type: "image", + source: { type: "file" } as any, + }, + ], + }, + ] + + const openAiMessages = convertToOpenAiMessages(anthropicMessages) + const content = openAiMessages[0].content as Array<{ type: string; text?: string }> + + expect(content[0]).toEqual({ type: "text", text: "[Image]" }) + }) + + it("should handle non-text/non-image blocks in tool results as empty string", () => { + const anthropicMessages: Anthropic.Messages.MessageParam[] = [ + { + role: "user", + content: [ + { + type: "tool_result", + tool_use_id: "tool-1", + content: [ + { type: "text", text: "some text" }, + { + type: "document", + source: { type: "base64", media_type: "application/pdf", data: "abc" }, + } as any, + ], + }, + ], + }, + ] + + const openAiMessages = convertToOpenAiMessages(anthropicMessages) + expect(openAiMessages[0].role).toBe("tool") + expect((openAiMessages[0] as any).content).toBe("some text\n") + }) + + it("should handle base64 image in tool result with placeholder and skip URL images", () => { + const anthropicMessages: Anthropic.Messages.MessageParam[] = [ + { + role: "user", + content: [ + { + type: "tool_result", + tool_use_id: "tool-1", + content: [ + { + type: "image", + source: { type: "url", url: "https://example.com/img.png" } as any, + }, + ], + }, + ], + }, + ] + + const openAiMessages = convertToOpenAiMessages(anthropicMessages) + expect(openAiMessages[0].role).toBe("tool") + expect((openAiMessages[0] as any).content).toBe("[Image]") + }) + it("should handle assistant messages with tool use (no normalization without normalizeToolCallId)", () => { const anthropicMessages: Anthropic.Messages.MessageParam[] = [ { diff --git a/src/api/transform/__tests__/vscode-lm-format.spec.ts b/src/api/transform/__tests__/vscode-lm-format.spec.ts index 67f835f6ab..1ec4b99f7f 100644 --- a/src/api/transform/__tests__/vscode-lm-format.spec.ts +++ b/src/api/transform/__tests__/vscode-lm-format.spec.ts @@ -178,6 +178,48 @@ describe("convertToVsCodeLmMessages", () => { const imagePlaceholder = result[0].content[1] as MockLanguageModelTextPart expect(imagePlaceholder.value).toContain("[Image (base64): image/png not supported by VSCode LM API]") }) + + it("should produce correct placeholder for URL image in non-tool messages", () => { + const messages: Anthropic.Messages.MessageParam[] = [ + { + role: "user", + content: [ + { + type: "image", + source: { type: "url", url: "https://example.com/img.png" } as any, + }, + ], + }, + ] + + const result = convertToVsCodeLmMessages(messages) + const imagePlaceholder = result[0].content[0] as MockLanguageModelTextPart + expect(imagePlaceholder.value).toContain("[Image (url): not supported by VSCode LM API]") + }) + + it("should produce correct placeholder for URL image inside tool result", () => { + const messages: Anthropic.Messages.MessageParam[] = [ + { + role: "user", + content: [ + { + type: "tool_result", + tool_use_id: "tool-1", + content: [ + { + type: "image", + source: { type: "url", url: "https://example.com/img.png" } as any, + }, + ], + }, + ], + }, + ] + + const result = convertToVsCodeLmMessages(messages) + const toolResult = result[0].content[0] as any + expect(toolResult.content[0].value).toContain("[Image (url): not supported by VSCode LM API]") + }) }) describe("convertToAnthropicRole", () => { diff --git a/src/api/transform/__tests__/zai-format.spec.ts b/src/api/transform/__tests__/zai-format.spec.ts new file mode 100644 index 0000000000..037b79c31d --- /dev/null +++ b/src/api/transform/__tests__/zai-format.spec.ts @@ -0,0 +1,79 @@ +// npx vitest run api/transform/__tests__/zai-format.spec.ts + +import { Anthropic } from "@anthropic-ai/sdk" + +import { convertToZAiFormat } from "../zai-format" + +describe("convertToZAiFormat", () => { + it("should convert simple user text messages", () => { + const messages: Anthropic.Messages.MessageParam[] = [{ role: "user", content: "Hello" }] + + const result = convertToZAiFormat(messages) + expect(result).toHaveLength(1) + expect(result[0]).toEqual({ role: "user", content: "Hello" }) + }) + + it("should handle base64 image in user message", () => { + const messages: Anthropic.Messages.MessageParam[] = [ + { + role: "user", + content: [ + { type: "text", text: "Describe this:" }, + { + type: "image", + source: { type: "base64", media_type: "image/png", data: "abc123" }, + }, + ], + }, + ] + + const result = convertToZAiFormat(messages) + expect(result).toHaveLength(1) + const content = (result[0] as any).content as Array<{ + type: string + image_url?: { url: string } + text?: string + }> + expect(content[0]).toEqual({ type: "text", text: "Describe this:" }) + expect(content[1]).toEqual({ type: "image_url", image_url: { url: "data:image/png;base64,abc123" } }) + }) + + it("should skip non-base64 images in user messages", () => { + const messages: Anthropic.Messages.MessageParam[] = [ + { + role: "user", + content: [ + { type: "text", text: "Hello" }, + { + type: "image", + source: { type: "url", url: "https://example.com/img.png" } as any, + }, + ], + }, + ] + + const result = convertToZAiFormat(messages) + expect(result).toHaveLength(1) + // URL image is skipped — only text remains, so content is a plain string + expect((result[0] as any).content).toBe("Hello") + }) + + it("should convert tool_result content", () => { + const messages: Anthropic.Messages.MessageParam[] = [ + { + role: "user", + content: [{ type: "tool_result", tool_use_id: "tool-1", content: "Result text" }], + }, + ] + + const result = convertToZAiFormat(messages) + expect(result[0]).toEqual({ role: "tool", tool_call_id: "tool-1", content: "Result text" }) + }) + + it("should convert assistant messages with text", () => { + const messages: Anthropic.Messages.MessageParam[] = [{ role: "assistant", content: "I can help with that." }] + + const result = convertToZAiFormat(messages) + expect(result[0]).toEqual({ role: "assistant", content: "I can help with that." }) + }) +}) diff --git a/src/api/transform/mistral-format.ts b/src/api/transform/mistral-format.ts index 3f89a51e0e..9a2b19098b 100644 --- a/src/api/transform/mistral-format.ts +++ b/src/api/transform/mistral-format.ts @@ -112,6 +112,12 @@ export function convertToMistralMessages(anthropicMessages: Anthropic.Messages.M }, } } + if (part.source.type === "url") { + return { + type: "image_url", + imageUrl: { url: part.source.url }, + } + } return { type: "text", text: "[Image]" } } if (part.type === "text") { diff --git a/src/api/transform/openai-format.ts b/src/api/transform/openai-format.ts index f03778b425..1a03b85e85 100644 --- a/src/api/transform/openai-format.ts +++ b/src/api/transform/openai-format.ts @@ -358,8 +358,11 @@ export function convertToOpenAiMessages( toolMessage.content ?.map((part) => { if (part.type === "image") { - toolResultImages.push(part) - return "(see following user message for image)" + if (part.source.type === "base64") { + toolResultImages.push(part) + return "(see following user message for image)" + } + return "[Image]" } if (part.type === "text") { return part.text @@ -428,7 +431,15 @@ export function convertToOpenAiMessages( if (part.source.type === "base64") { return { type: "image_url", - image_url: { url: `data:${part.source.media_type};base64,${part.source.data}` }, + image_url: { + url: `data:${part.source.media_type};base64,${part.source.data}`, + }, + } + } + if (part.source.type === "url") { + return { + type: "image_url", + image_url: { url: part.source.url }, } } return { type: "text", text: "[Image]" } diff --git a/src/api/transform/vscode-lm-format.ts b/src/api/transform/vscode-lm-format.ts index a9273b9a40..935f6a5474 100644 --- a/src/api/transform/vscode-lm-format.ts +++ b/src/api/transform/vscode-lm-format.ts @@ -74,11 +74,11 @@ export function convertToVsCodeLmMessages( if (part.type === "image") { if (part.source.type === "base64") { return new vscode.LanguageModelTextPart( - `[Image (${part.source?.type || "Unknown source-type"}): ${part.source?.media_type || "unknown media-type"} not supported by VSCode LM API]`, + `[Image (base64): ${part.source.media_type} not supported by VSCode LM API]`, ) } return new vscode.LanguageModelTextPart( - `[Image (${part.source?.type || "Unknown source-type"}): not supported by VSCode LM API]`, + `[Image (${part.source.type}): not supported by VSCode LM API]`, ) } if (part.type === "text") { @@ -95,11 +95,11 @@ export function convertToVsCodeLmMessages( if (part.type === "image") { if (part.source.type === "base64") { return new vscode.LanguageModelTextPart( - `[Image (${part.source?.type || "Unknown source-type"}): ${part.source?.media_type || "unknown media-type"} not supported by VSCode LM API]`, + `[Image (base64): ${part.source.media_type} not supported by VSCode LM API]`, ) } return new vscode.LanguageModelTextPart( - `[Image (${part.source?.type || "Unknown source-type"}): not supported by VSCode LM API]`, + `[Image (${part.source.type}): not supported by VSCode LM API]`, ) } if (part.type === "text") { diff --git a/src/integrations/misc/__tests__/export-markdown.spec.ts b/src/integrations/misc/__tests__/export-markdown.spec.ts index fd4c30c3d2..e599d504aa 100644 --- a/src/integrations/misc/__tests__/export-markdown.spec.ts +++ b/src/integrations/misc/__tests__/export-markdown.spec.ts @@ -72,5 +72,32 @@ describe("export-markdown", () => { const block = { type: "unknown_type" as const } as any expect(formatContentBlockToMarkdown(block)).toBe("[Unexpected content type: unknown_type]") }) + + it("should format document blocks", () => { + const block = { + type: "document", + source: { type: "base64", media_type: "application/pdf", data: "abc" }, + } as any as ExtendedContentBlock + expect(formatContentBlockToMarkdown(block)).toBe("[Document]") + }) + + it("should format search_result blocks", () => { + const block = { + type: "search_result", + source: "https://example.com", + title: "Example", + content: [], + } as any as ExtendedContentBlock + expect(formatContentBlockToMarkdown(block)).toBe("[Search Result]") + }) + + it("should format tool_reference blocks", () => { + const block = { + type: "tool_reference", + id: "tool-1", + name: "read_file", + } as any as ExtendedContentBlock + expect(formatContentBlockToMarkdown(block)).toBe("[Tool Reference]") + }) }) }) From 866b56f3359eb9304dad6edd1c0f767c1f739e65 Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Sat, 27 Jun 2026 17:42:38 +0000 Subject: [PATCH 3/4] test: bumping coverage --- .../providers/__tests__/native-ollama.spec.ts | 53 ++++++++++++++++++- .../providers/__tests__/openai-codex.spec.ts | 53 +++++++++++++++++++ .../providers/__tests__/openai-native.spec.ts | 50 +++++++++++++++++ .../transform/__tests__/openai-format.spec.ts | 30 ++++++++++- src/api/transform/__tests__/r1-format.spec.ts | 20 +++++++ .../__tests__/responses-api-input.spec.ts | 19 +++++++ .../__tests__/vscode-lm-format.spec.ts | 45 +++++++++++++++- .../transform/__tests__/zai-format.spec.ts | 2 +- .../misc/__tests__/export-markdown.spec.ts | 13 +++-- 9 files changed, 273 insertions(+), 12 deletions(-) diff --git a/src/api/providers/__tests__/native-ollama.spec.ts b/src/api/providers/__tests__/native-ollama.spec.ts index 200868022b..bd341ce86b 100644 --- a/src/api/providers/__tests__/native-ollama.spec.ts +++ b/src/api/providers/__tests__/native-ollama.spec.ts @@ -1,4 +1,6 @@ -// npx vitest run api/providers/__tests__/native-ollama.spec.ts +// pnpm exec vitest run api/providers/__tests__/native-ollama.spec.ts + +import { Anthropic } from "@anthropic-ai/sdk" import { NativeOllamaHandler } from "../native-ollama" import { ApiHandlerOptions } from "../../../shared/api" @@ -81,6 +83,55 @@ describe("NativeOllamaHandler", () => { expect(results[2]).toEqual({ type: "usage", inputTokens: 10, outputTokens: 2 }) }) + it("should map tool_result array content to a concatenated string, flushing base64 images", async () => { + mockChat.mockImplementation(async function* () { + yield { message: { content: "ok" } } + }) + + const messages: Anthropic.Messages.MessageParam[] = [ + { + role: "user", + content: [ + { + type: "tool_result", + tool_use_id: "tool-1", + content: [ + { type: "text", text: "line one" }, + { + type: "image", + source: { + type: "base64", + media_type: "image/png", + data: "imgdata", + }, + }, + { type: "text", text: "line two" }, + ], + }, + ], + }, + ] + + const stream = handler.createMessage("System", messages) + for await (const _ of stream) { + // consume stream + } + + // Text blocks are joined with "\n"; the image emits a placeholder and is + // flushed separately via the `images` field rather than inlined. + expect(mockChat).toHaveBeenCalledWith( + expect.objectContaining({ + messages: expect.arrayContaining([ + expect.objectContaining({ + role: "user", + content: "line one\n(see following user message for image)\nline two", + images: ["imgdata"], + }), + ]), + }), + ) + }) + it("should not include num_ctx by default", async () => { // Mock the chat response mockChat.mockImplementation(async function* () { diff --git a/src/api/providers/__tests__/openai-codex.spec.ts b/src/api/providers/__tests__/openai-codex.spec.ts index dcc0c4d035..2dc92a683d 100644 --- a/src/api/providers/__tests__/openai-codex.spec.ts +++ b/src/api/providers/__tests__/openai-codex.spec.ts @@ -1,6 +1,8 @@ // npx vitest run api/providers/__tests__/openai-codex.spec.ts +import { Anthropic } from "@anthropic-ai/sdk" import { OpenAiCodexHandler } from "../openai-codex" +import { openAiCodexOAuthManager } from "../../../integrations/openai-codex/oauth" describe("OpenAiCodexHandler.getModel", () => { it.each(["gpt-5.1", "gpt-5", "gpt-5.1-codex", "gpt-5-codex", "gpt-5-codex-mini", "gpt-5.3-codex-spark"])( @@ -42,3 +44,54 @@ describe("OpenAiCodexHandler.getModel", () => { expect(model.info).toBeDefined() }) }) + +describe("OpenAiCodexHandler.createMessage", () => { + it("should skip URL-sourced images in formatFullConversation (only base64 emits input_image)", async () => { + const handler = new OpenAiCodexHandler({ apiModelId: "gpt-5.1-codex" }) + + vitest.spyOn(openAiCodexOAuthManager, "getAccessToken").mockResolvedValue("test-token") + vitest.spyOn(openAiCodexOAuthManager, "getAccountId").mockResolvedValue("acct_test") + + const capturedInput: any[] = [] + ;(handler as any).client = { + responses: { + create: vitest.fn().mockImplementation(async (body: any) => { + capturedInput.push(...(body.input ?? [])) + return { + async *[Symbol.asyncIterator]() { + yield { + type: "response.completed", + response: { + id: "r1", + status: "completed", + output: [], + usage: { input_tokens: 1, output_tokens: 1 }, + }, + } + }, + } + }), + }, + } + + const messages: Anthropic.Messages.MessageParam[] = [ + { + role: "user", + content: [ + { type: "text", text: "Look at this:" }, + { type: "image", source: { type: "url", url: "https://example.com/img.png" } as any }, + ], + }, + ] + + const stream = handler.createMessage("system", messages) + for await (const _ of stream) { + // consume + } + + // URL image is skipped; only the text input_text block should be present + const userMsg = capturedInput.find((item: any) => item.role === "user") + expect(userMsg?.content).toEqual([{ type: "input_text", text: "Look at this:" }]) + expect(JSON.stringify(capturedInput)).not.toContain("input_image") + }) +}) diff --git a/src/api/providers/__tests__/openai-native.spec.ts b/src/api/providers/__tests__/openai-native.spec.ts index 4d35387992..6a922966f4 100644 --- a/src/api/providers/__tests__/openai-native.spec.ts +++ b/src/api/providers/__tests__/openai-native.spec.ts @@ -1845,4 +1845,54 @@ describe("GPT-5 streaming event coverage (additional)", () => { }) }) }) + + describe("URL image handling", () => { + it("should skip URL-sourced images in formatFullConversation (only base64 emits input_image)", async () => { + const mockFetch = vitest.fn().mockResolvedValue({ + ok: true, + body: new ReadableStream({ + start(controller) { + controller.enqueue( + new TextEncoder().encode('data: {"type":"response.output_text.delta","delta":"ok"}\n\n'), + ) + controller.enqueue(new TextEncoder().encode("data: [DONE]\n\n")) + controller.close() + }, + }), + }) + global.fetch = mockFetch as any + + mockResponsesCreate.mockRejectedValue(new Error("SDK not available")) + + const localHandler = new OpenAiNativeHandler({ + apiModelId: "gpt-4.1", + openAiNativeApiKey: "test-api-key", + }) + + const urlImageMessages: Anthropic.Messages.MessageParam[] = [ + { + role: "user", + content: [ + { type: "text", text: "Look at this:" }, + { + type: "image", + source: { type: "url", url: "https://example.com/img.png" } as any, + }, + ], + }, + ] + + const stream = localHandler.createMessage("You are a helpful assistant.", urlImageMessages) + for await (const _ of stream) { + // consume + } + + const bodyStr = (mockFetch.mock.calls[0][1] as any).body as string + const parsedBody = JSON.parse(bodyStr) + // URL image is skipped; only the text part is in the input + const userMsg = parsedBody.input[0] + expect(userMsg.content).toEqual([{ type: "input_text", text: "Look at this:" }]) + expect(bodyStr).not.toContain("input_image") + }) + }) }) diff --git a/src/api/transform/__tests__/openai-format.spec.ts b/src/api/transform/__tests__/openai-format.spec.ts index b02ac83c63..b58deea649 100644 --- a/src/api/transform/__tests__/openai-format.spec.ts +++ b/src/api/transform/__tests__/openai-format.spec.ts @@ -1,4 +1,4 @@ -// npx vitest run api/transform/__tests__/openai-format.spec.ts +// pnpm exec vitest run api/transform/__tests__/openai-format.spec.ts import { Anthropic } from "@anthropic-ai/sdk" import OpenAI from "openai" @@ -143,7 +143,33 @@ describe("convertToOpenAiMessages", () => { expect((openAiMessages[0] as any).content).toBe("some text\n") }) - it("should handle base64 image in tool result with placeholder and skip URL images", () => { + it("should handle base64 image in tool result with placeholder", () => { + const anthropicMessages: Anthropic.Messages.MessageParam[] = [ + { + role: "user", + content: [ + { + type: "tool_result", + tool_use_id: "tool-1", + content: [ + { + type: "image", + source: { type: "base64", media_type: "image/png", data: "base64data" }, + }, + ], + }, + ], + }, + ] + + const openAiMessages = convertToOpenAiMessages(anthropicMessages) + expect(openAiMessages[0].role).toBe("tool") + // base64 images in tool results emit a placeholder; the image itself is + // flushed in a separate user message (see comment block in openai-format.ts) + expect((openAiMessages[0] as any).content).toBe("(see following user message for image)") + }) + + it("should render [Image] placeholder for URL image in tool result", () => { const anthropicMessages: Anthropic.Messages.MessageParam[] = [ { role: "user", diff --git a/src/api/transform/__tests__/r1-format.spec.ts b/src/api/transform/__tests__/r1-format.spec.ts index 3d875e9392..7def13f1fe 100644 --- a/src/api/transform/__tests__/r1-format.spec.ts +++ b/src/api/transform/__tests__/r1-format.spec.ts @@ -69,6 +69,26 @@ describe("convertToR1Format", () => { expect(convertToR1Format(input)).toEqual(expected) }) + it("should skip non-base64 images (e.g. URL-sourced)", () => { + const input: Anthropic.Messages.MessageParam[] = [ + { + role: "user", + content: [ + { type: "text", text: "Look at this:" }, + { + type: "image", + source: { type: "url", url: "https://example.com/img.png" } as any, + }, + ], + }, + ] + + const result = convertToR1Format(input) + // URL image is skipped; only the text part survives + expect(result).toHaveLength(1) + expect(result[0]).toEqual({ role: "user", content: "Look at this:" }) + }) + it("should handle mixed text and image content", () => { const input: Anthropic.Messages.MessageParam[] = [ { diff --git a/src/api/transform/__tests__/responses-api-input.spec.ts b/src/api/transform/__tests__/responses-api-input.spec.ts index c57b61d895..dc13ebfce0 100644 --- a/src/api/transform/__tests__/responses-api-input.spec.ts +++ b/src/api/transform/__tests__/responses-api-input.spec.ts @@ -73,6 +73,25 @@ describe("convertToResponsesApiInput", () => { ]) }) + it("should skip non-base64 images (e.g. URL-sourced)", () => { + const messages: Anthropic.Messages.MessageParam[] = [ + { + role: "user", + content: [ + { type: "text", text: "Look at this:" }, + { + type: "image", + source: { type: "url", url: "https://example.com/img.png" } as any, + }, + ], + }, + ] + + const result = convertToResponsesApiInput(messages) + // URL image is not emitted; only the text part survives + expect(result).toEqual([{ role: "user", content: [{ type: "input_text", text: "Look at this:" }] }]) + }) + it("should convert tool_result to function_call_output", () => { const messages: Anthropic.Messages.MessageParam[] = [ { diff --git a/src/api/transform/__tests__/vscode-lm-format.spec.ts b/src/api/transform/__tests__/vscode-lm-format.spec.ts index 1ec4b99f7f..36325e9c93 100644 --- a/src/api/transform/__tests__/vscode-lm-format.spec.ts +++ b/src/api/transform/__tests__/vscode-lm-format.spec.ts @@ -1,4 +1,4 @@ -// npx vitest run src/api/transform/__tests__/vscode-lm-format.spec.ts +// pnpm exec vitest run api/transform/__tests__/vscode-lm-format.spec.ts import { Anthropic } from "@anthropic-ai/sdk" import * as vscode from "vscode" @@ -220,6 +220,49 @@ describe("convertToVsCodeLmMessages", () => { const toolResult = result[0].content[0] as any expect(toolResult.content[0].value).toContain("[Image (url): not supported by VSCode LM API]") }) + + it("should produce base64 image placeholder inside tool result", () => { + const messages: Anthropic.Messages.MessageParam[] = [ + { + role: "user", + content: [ + { + type: "tool_result", + tool_use_id: "tool-1", + content: [ + { + type: "image", + source: { type: "base64", media_type: "image/jpeg", data: "abc" }, + }, + ], + }, + ], + }, + ] + + const result = convertToVsCodeLmMessages(messages) + const toolResult = result[0].content[0] as any + expect(toolResult.content[0].value).toBe("[Image (base64): image/jpeg not supported by VSCode LM API]") + }) + + it("should return empty string for unknown block types inside tool result", () => { + const messages: Anthropic.Messages.MessageParam[] = [ + { + role: "user", + content: [ + { + type: "tool_result", + tool_use_id: "tool-1", + content: [{ type: "document" } as any], + }, + ], + }, + ] + + const result = convertToVsCodeLmMessages(messages) + const toolResult = result[0].content[0] as any + expect(toolResult.content[0].value).toBe("") + }) }) describe("convertToAnthropicRole", () => { diff --git a/src/api/transform/__tests__/zai-format.spec.ts b/src/api/transform/__tests__/zai-format.spec.ts index 037b79c31d..27ad4a2c90 100644 --- a/src/api/transform/__tests__/zai-format.spec.ts +++ b/src/api/transform/__tests__/zai-format.spec.ts @@ -1,4 +1,4 @@ -// npx vitest run api/transform/__tests__/zai-format.spec.ts +// pnpm exec vitest run api/transform/__tests__/zai-format.spec.ts import { Anthropic } from "@anthropic-ai/sdk" diff --git a/src/integrations/misc/__tests__/export-markdown.spec.ts b/src/integrations/misc/__tests__/export-markdown.spec.ts index e599d504aa..081fb26158 100644 --- a/src/integrations/misc/__tests__/export-markdown.spec.ts +++ b/src/integrations/misc/__tests__/export-markdown.spec.ts @@ -76,8 +76,8 @@ describe("export-markdown", () => { it("should format document blocks", () => { const block = { type: "document", - source: { type: "base64", media_type: "application/pdf", data: "abc" }, - } as any as ExtendedContentBlock + source: { type: "base64", media_type: "application/pdf", data: "abc" } as const, + } satisfies ExtendedContentBlock expect(formatContentBlockToMarkdown(block)).toBe("[Document]") }) @@ -86,17 +86,16 @@ describe("export-markdown", () => { type: "search_result", source: "https://example.com", title: "Example", - content: [], - } as any as ExtendedContentBlock + content: [{ type: "text", text: "result text" }], + } satisfies ExtendedContentBlock expect(formatContentBlockToMarkdown(block)).toBe("[Search Result]") }) it("should format tool_reference blocks", () => { const block = { type: "tool_reference", - id: "tool-1", - name: "read_file", - } as any as ExtendedContentBlock + tool_name: "read_file", + } satisfies ExtendedContentBlock expect(formatContentBlockToMarkdown(block)).toBe("[Tool Reference]") }) }) From 11feb817c453245fcc02cab1b2425dfe357582e6 Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Sun, 28 Jun 2026 02:43:38 +0000 Subject: [PATCH 4/4] test(e2e): adding image test --- apps/vscode-e2e/fixtures/openrouter.json | 14 ++++++ .../src/suite/providers/openrouter.test.ts | 50 ++++++++++++++++++- .../providers/__tests__/native-ollama.spec.ts | 40 +++++++++++++++ .../providers/__tests__/openai-codex.spec.ts | 50 +++++++++++++++++++ .../providers/__tests__/openai-native.spec.ts | 46 +++++++++++++++++ src/api/transform/mistral-format.ts | 5 +- src/api/transform/openai-format.ts | 5 +- src/api/transform/vscode-lm-format.ts | 5 +- 8 files changed, 202 insertions(+), 13 deletions(-) diff --git a/apps/vscode-e2e/fixtures/openrouter.json b/apps/vscode-e2e/fixtures/openrouter.json index 1eef046277..eb1e63cfe3 100644 --- a/apps/vscode-e2e/fixtures/openrouter.json +++ b/apps/vscode-e2e/fixtures/openrouter.json @@ -1,5 +1,19 @@ { "fixtures": [ + { + "match": { + "userMessage": "openrouter-image-e2e" + }, + "response": { + "toolCalls": [ + { + "name": "attempt_completion", + "arguments": "{\"result\":\"Red\"}", + "id": "call_openrouter_image_001" + } + ] + } + }, { "match": { "userMessage": "openrouter-identity-smoke" diff --git a/apps/vscode-e2e/src/suite/providers/openrouter.test.ts b/apps/vscode-e2e/src/suite/providers/openrouter.test.ts index 140175168c..8b7e3d788f 100644 --- a/apps/vscode-e2e/src/suite/providers/openrouter.test.ts +++ b/apps/vscode-e2e/src/suite/providers/openrouter.test.ts @@ -9,6 +9,7 @@ type CapturedOpenRouterRequest = { xTitle: string | undefined httpReferer: string | undefined userAgent: string | undefined + userMessageContent?: unknown } function getRequestUrl(input: RequestInfo | URL): string { @@ -40,10 +41,22 @@ function installOpenRouterRequestCapture(capture: CapturedOpenRouterRequest[], b if (new URL(url).origin === targetOrigin) { const xTitle = getHeaderValue(init, "X-Title") ?? getHeaderValue(init, "x-title") if (xTitle !== undefined) { + let userMessageContent: unknown + if (init?.body && typeof init.body === "string") { + try { + const body = JSON.parse(init.body) + const messages: Array<{ role?: string; content?: unknown }> = body.messages ?? [] + const lastUser = [...messages].reverse().find((m) => m.role === "user") + userMessageContent = lastUser?.content + } catch { + // ignore parse errors + } + } capture.push({ xTitle, httpReferer: getHeaderValue(init, "HTTP-Referer") ?? getHeaderValue(init, "http-referer"), userAgent: getHeaderValue(init, "User-Agent") ?? getHeaderValue(init, "user-agent"), + userMessageContent, }) } } @@ -82,7 +95,7 @@ suite("OpenRouter provider", function () { await globalThis.api.setConfiguration({ apiProvider: "openrouter" as const, openRouterApiKey: aimockUrl && !isRecord ? "mock-key" : OPENROUTER_API_KEY!, - openRouterModelId: "openai/gpt-4.1", + openRouterModelId: "anthropic/claude-haiku-4-5", ...(aimockUrl && { openRouterBaseUrl: `${aimockUrl}/v1` }), }) }) @@ -92,6 +105,41 @@ suite("OpenRouter provider", function () { restoreFetch = undefined }) + test("Should forward base64 images as image_url content parts in outbound request", async () => { + requests.length = 0 + + // 8x8 red PNG (1x1 is too small and rejected by Anthropic's vision API) + const base64Png = + "iVBORw0KGgoAAAANSUhEUgAAAAgAAAAICAIAAABLbSncAAAAEklEQVR4nGP4z8CAFWEXHbQSACj/P8Fu7N9hAAAAAElFTkSuQmCC" + const dataUri = `data:image/png;base64,${base64Png}` + + const api = globalThis.api + const taskId = await api.startNewTask({ + configuration: { mode: "ask", autoApprovalEnabled: true }, + text: "openrouter-image-e2e: describe the image in one word.", + images: [dataUri], + }) + + await waitUntilCompleted({ api, taskId }) + + // Find the first request that contains the probe tag + const probeRequest = requests.find((r) => { + const content = r.userMessageContent + return Array.isArray(content) && JSON.stringify(content).includes("openrouter-image-e2e") + }) + + assert.ok(probeRequest, "Should have captured an outbound request containing the probe tag") + + const content = probeRequest.userMessageContent as Array<{ type: string; image_url?: { url: string } }> + const imagePart = content.find((p) => p.type === "image_url") + assert.ok(imagePart, "User message should contain an image_url content part") + assert.strictEqual( + imagePart?.image_url?.url, + dataUri, + "image_url.url should be the original data URI passed via startNewTask", + ) + }) + test("Should identify as Zoo Code in outbound DEFAULT_HEADERS", async () => { requests.length = 0 diff --git a/src/api/providers/__tests__/native-ollama.spec.ts b/src/api/providers/__tests__/native-ollama.spec.ts index bd341ce86b..8a6b025c8f 100644 --- a/src/api/providers/__tests__/native-ollama.spec.ts +++ b/src/api/providers/__tests__/native-ollama.spec.ts @@ -132,6 +132,46 @@ describe("NativeOllamaHandler", () => { ) }) + it("should drop unknown block types in tool_result content (empty string contribution)", async () => { + mockChat.mockImplementation(async function* () { + yield { message: { content: "ok" } } + }) + + const messages: Anthropic.Messages.MessageParam[] = [ + { + role: "user", + content: [ + { + type: "tool_result", + tool_use_id: "tool-1", + content: [ + { type: "text", text: "before" }, + { type: "document" } as any, + { type: "text", text: "after" }, + ], + }, + ], + }, + ] + + const stream = handler.createMessage("System", messages) + for await (const _ of stream) { + // consume + } + + // The unknown block contributes "" so the join produces "before\n\nafter" + expect(mockChat).toHaveBeenCalledWith( + expect.objectContaining({ + messages: expect.arrayContaining([ + expect.objectContaining({ + role: "user", + content: "before\n\nafter", + }), + ]), + }), + ) + }) + it("should not include num_ctx by default", async () => { // Mock the chat response mockChat.mockImplementation(async function* () { diff --git a/src/api/providers/__tests__/openai-codex.spec.ts b/src/api/providers/__tests__/openai-codex.spec.ts index 2dc92a683d..4ab9755252 100644 --- a/src/api/providers/__tests__/openai-codex.spec.ts +++ b/src/api/providers/__tests__/openai-codex.spec.ts @@ -94,4 +94,54 @@ describe("OpenAiCodexHandler.createMessage", () => { expect(userMsg?.content).toEqual([{ type: "input_text", text: "Look at this:" }]) expect(JSON.stringify(capturedInput)).not.toContain("input_image") }) + + it("should emit input_image for base64 images in formatFullConversation", async () => { + const handler = new OpenAiCodexHandler({ apiModelId: "gpt-5.1-codex" }) + + vitest.spyOn(openAiCodexOAuthManager, "getAccessToken").mockResolvedValue("test-token") + vitest.spyOn(openAiCodexOAuthManager, "getAccountId").mockResolvedValue("acct_test") + + const capturedInput: any[] = [] + ;(handler as any).client = { + responses: { + create: vitest.fn().mockImplementation(async (body: any) => { + capturedInput.push(...(body.input ?? [])) + return { + async *[Symbol.asyncIterator]() { + yield { + type: "response.completed", + response: { + id: "r1", + status: "completed", + output: [], + usage: { input_tokens: 1, output_tokens: 1 }, + }, + } + }, + } + }), + }, + } + + const messages: Anthropic.Messages.MessageParam[] = [ + { + role: "user", + content: [ + { type: "text", text: "Look at this:" }, + { type: "image", source: { type: "base64", media_type: "image/png", data: "abc123" } }, + ], + }, + ] + + const stream = handler.createMessage("system", messages) + for await (const _ of stream) { + // consume + } + + const userMsg = capturedInput.find((item: any) => item.role === "user") + expect(userMsg?.content).toContainEqual({ + type: "input_image", + image_url: "data:image/png;base64,abc123", + }) + }) }) diff --git a/src/api/providers/__tests__/openai-native.spec.ts b/src/api/providers/__tests__/openai-native.spec.ts index 6a922966f4..2a3ec0afb2 100644 --- a/src/api/providers/__tests__/openai-native.spec.ts +++ b/src/api/providers/__tests__/openai-native.spec.ts @@ -1894,5 +1894,51 @@ describe("GPT-5 streaming event coverage (additional)", () => { expect(userMsg.content).toEqual([{ type: "input_text", text: "Look at this:" }]) expect(bodyStr).not.toContain("input_image") }) + + it("should emit input_image for base64 images in formatFullConversation", async () => { + const mockFetch = vitest.fn().mockResolvedValue({ + ok: true, + body: new ReadableStream({ + start(controller) { + controller.enqueue( + new TextEncoder().encode('data: {"type":"response.output_text.delta","delta":"ok"}\n\n'), + ) + controller.enqueue(new TextEncoder().encode("data: [DONE]\n\n")) + controller.close() + }, + }), + }) + global.fetch = mockFetch as any + + mockResponsesCreate.mockRejectedValue(new Error("SDK not available")) + + const localHandler = new OpenAiNativeHandler({ + apiModelId: "gpt-4.1", + openAiNativeApiKey: "test-api-key", + }) + + const b64ImageMessages: Anthropic.Messages.MessageParam[] = [ + { + role: "user", + content: [ + { type: "text", text: "Look at this:" }, + { type: "image", source: { type: "base64", media_type: "image/png", data: "abc123" } }, + ], + }, + ] + + const stream = localHandler.createMessage("You are a helpful assistant.", b64ImageMessages) + for await (const _ of stream) { + // consume + } + + const bodyStr = (mockFetch.mock.calls[0][1] as any).body as string + const parsedBody = JSON.parse(bodyStr) + const userMsg = parsedBody.input[0] + expect(userMsg.content).toContainEqual({ + type: "input_image", + image_url: "data:image/png;base64,abc123", + }) + }) }) }) diff --git a/src/api/transform/mistral-format.ts b/src/api/transform/mistral-format.ts index 9a2b19098b..c29d248714 100644 --- a/src/api/transform/mistral-format.ts +++ b/src/api/transform/mistral-format.ts @@ -120,10 +120,7 @@ export function convertToMistralMessages(anthropicMessages: Anthropic.Messages.M } return { type: "text", text: "[Image]" } } - if (part.type === "text") { - return { type: "text", text: part.text } - } - return { type: "text", text: "" } + return { type: "text", text: part.text } }), }) } diff --git a/src/api/transform/openai-format.ts b/src/api/transform/openai-format.ts index 1a03b85e85..175872c635 100644 --- a/src/api/transform/openai-format.ts +++ b/src/api/transform/openai-format.ts @@ -444,10 +444,7 @@ export function convertToOpenAiMessages( } return { type: "text", text: "[Image]" } } - if (part.type === "text") { - return { type: "text", text: part.text } - } - return { type: "text", text: "" } + return { type: "text", text: part.text } }), }) } diff --git a/src/api/transform/vscode-lm-format.ts b/src/api/transform/vscode-lm-format.ts index 935f6a5474..94716b209d 100644 --- a/src/api/transform/vscode-lm-format.ts +++ b/src/api/transform/vscode-lm-format.ts @@ -102,10 +102,7 @@ export function convertToVsCodeLmMessages( `[Image (${part.source.type}): not supported by VSCode LM API]`, ) } - if (part.type === "text") { - return new vscode.LanguageModelTextPart(part.text) - } - return new vscode.LanguageModelTextPart("") + return new vscode.LanguageModelTextPart(part.text) }), ]