diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c890389339..d19e879b22 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -455,11 +455,11 @@ importers: specifier: ^3.0.48 version: 3.0.48(zod@3.25.76) '@anthropic-ai/sdk': - specifier: ^0.37.0 - version: 0.37.0 + specifier: ^0.104.1 + version: 0.104.1(zod@3.25.76) '@anthropic-ai/vertex-sdk': - specifier: ^0.7.0 - version: 0.7.0 + specifier: ^0.17.1 + version: 0.17.1(zod@3.25.76) '@aws-sdk/client-bedrock-runtime': specifier: ^3.922.0 version: 3.922.0 @@ -1142,11 +1142,17 @@ packages: '@antfu/install-pkg@1.1.0': resolution: {integrity: sha512-MGQsmw10ZyI+EJo45CdSER4zEb+p31LpDAFp2Z3gkSd1yqVZGi0Ebx++YTEMonJy4oChEMLsxZ64j8FH6sSqtQ==} - '@anthropic-ai/sdk@0.37.0': - resolution: {integrity: sha512-tHjX2YbkUBwEgg0JZU3EFSSAQPoK4qQR/NFYa8Vtzd5UAyXzZksCw2In69Rml4R/TyHPBfRYaLK35XiOe33pjw==} + '@anthropic-ai/sdk@0.104.1': + resolution: {integrity: sha512-gGACa/+IaiXzRRmF96aOhamoBgapKRBiFWbmmTFP8aMkpaEcuStF+Q61bjo4vPxBM7gqWJNZqsngslRdnLHv0Q==} + hasBin: true + peerDependencies: + zod: 3.25.76 + peerDependenciesMeta: + zod: + optional: true - '@anthropic-ai/vertex-sdk@0.7.0': - resolution: {integrity: sha512-zNm3hUXgYmYDTyveIxOyxbcnh5VXFkrLo4bSnG6LAfGzW7k3k2iCNDSVKtR9qZrK2BCid7JtVu7jsEKaZ/9dSw==} + '@anthropic-ai/vertex-sdk@0.17.1': + resolution: {integrity: sha512-+am1LjqEpb7Qq/fKDdu0PTE2U9+XSPXSSr9Fa5dvJ8FtJ5L9XOroNBnr7iecT+6iGlbiV6BYgMIGkAwPuly/jA==} '@asamuzakjp/css-color@3.2.0': resolution: {integrity: sha512-K1A6z8tS3XsmCMM86xoWdn7Fkdn9m6RSVtocUrJYIwZnFVkng/PvkEoWtOWmP+Scc6saYWHWZYbndEEXxl24jw==} @@ -3251,6 +3257,9 @@ packages: resolution: {integrity: sha512-4aUIteuyxtBUhVdiQqcDhKFitwfd9hqoSDYY2KRXiWtgoWJ9Bmise+KfEPDiVHWeJepvF8xJO9/9+WDIciMFFw==} engines: {node: '>=18.0.0'} + '@stablelib/base64@1.0.1': + resolution: {integrity: sha512-1bnPQqSxSuc3Ii6MhBysoWCg58j97aUjuCSZrGSmDxNqtytIi0k8utUenAwTZN4V5mXXYGsVUI9zeBqy+jBOSQ==} + '@standard-schema/spec@1.1.0': resolution: {integrity: sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==} @@ -3621,9 +3630,6 @@ packages: resolution: {integrity: sha512-faK2Owokboz53g8ooq2dw3iDJ6/HMTCIa2RvMte5WMTiABy+wA558K+iuyRtlR67Un5q9gEKysSDtqZYbSa0Pg==} deprecated: This is a stub types definition. node-cache provides its own type definitions, so you do not need this installed. - '@types/node-fetch@2.6.12': - resolution: {integrity: sha512-8nneRWKCg3rMtF69nLQJnOYUcbafYeFSjqkw3jCRLsqkWFlHaoQrr5mXmofFGOx3DKn7UfmBMyov8ySvLRVldA==} - '@types/node-ipc@9.2.3': resolution: {integrity: sha512-/MvSiF71fYf3+zwqkh/zkVkZj1hl1Uobre9EMFy08mqfJNAmpR0vmPgOUdEIDVgifxHj6G1vYMPLSBLLxoDACQ==} @@ -3633,9 +3639,6 @@ packages: '@types/node@14.18.63': resolution: {integrity: sha512-fAtCfv4jJg+ExtXhvCkCqUKZ+4ok/JQk01qDKhL5BDDoS3AxKXhV5/MAVUZyQnSEd2GT92fkgZl0pz0Q0AzcIQ==} - '@types/node@18.19.100': - resolution: {integrity: sha512-ojmMP8SZBKprc3qGrGk8Ujpo80AXkrP7G2tOT4VWr5jlr5DHjsJF+emXJz+Wm0glmy4Js62oKMdZZ6B9Y+tEcA==} - '@types/node@20.19.43': resolution: {integrity: sha512-6oYBAi5ikg4Pl+kGsoYtawUMBT2zZMCvPNF7pVLnHZfd1zf38DRiWn/gT01RYCdUqkv7Fhr+C9ot4/tb+2sVvA==} @@ -3889,10 +3892,6 @@ packages: '@xobotyi/scrollbar-width@1.9.5': resolution: {integrity: sha512-N8tkAACJx2ww8vFMneJmaAgmjAG1tnVBZJRLRcx061tmsLRZHSEZSLuGWnwPtunsSLvSqXQ2wfp7Mgqg1I+2dQ==} - abort-controller@3.0.0: - resolution: {integrity: sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==} - engines: {node: '>=6.5'} - accepts@2.0.0: resolution: {integrity: sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==} engines: {node: '>= 0.6'} @@ -3915,10 +3914,6 @@ packages: resolution: {integrity: sha512-jRR5wdylq8CkOe6hei19GGZnxM6rBGwFl3Bg0YItGDimvjGtAvdZk4Pu6Cl4u4Igsws4a1fd1Vq3ezrhn4KmFw==} engines: {node: '>= 14'} - agentkeepalive@4.6.0: - resolution: {integrity: sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ==} - engines: {node: '>= 8.0.0'} - ai-sdk-provider-poe@2.0.18: resolution: {integrity: sha512-uzFR5Zq+PD6LfrqpYAr4dt2KmfOfS9a0Wh8QJsOaGI/vOByhW02P/0mj0NtOnOWF1RIUfBkQJo647XjrgnFjjg==} peerDependencies: @@ -5133,10 +5128,6 @@ packages: event-stream@3.3.4: resolution: {integrity: sha512-QHpkERcGsR0T7Qm3HNJSyXKEEj8AHNxkY3PK8TS2KJvQ7NiSHe3DDpwVKKtoYprL/AreyzFBeIkBIWChAqn60g==} - event-target-shim@5.0.1: - resolution: {integrity: sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==} - engines: {node: '>=6'} - eventemitter3@5.0.4: resolution: {integrity: sha512-mlsTRyGaPBjPedk6Bvw+aqbsXDtoAyAzm5MO7JgU+yVRyMQ5O8bD4Kcci7BS85f93veegeCPkL8R4GLClnjLFw==} @@ -5224,6 +5215,9 @@ packages: fast-levenshtein@2.0.6: resolution: {integrity: sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==} + fast-sha256@1.3.0: + resolution: {integrity: sha512-n11RGP/lrWEFI/bWdygLxhI+pVeo1ZYIVwvvPkW7azl/rOy+F3HYRZ2K5zeE9mmkhQppyv9sQFx0JM9UabnpPQ==} + fast-shallow-equal@1.0.0: resolution: {integrity: sha512-HPtaa38cPgWvaCFmRNhlc6NG7pv6NUHqjPgVAkWGoB9mQMwYB27/K0CvOM5Czy+qpT3e8XJ6Q4aPAnzpNpzNaw==} @@ -5324,9 +5318,6 @@ packages: resolution: {integrity: sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==} engines: {node: '>=14'} - form-data-encoder@1.7.2: - resolution: {integrity: sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==} - form-data@4.0.6: resolution: {integrity: sha512-vKatAh4SlVfgbv+YtmhiRjhEMJsYpsG1Y2rMQtR+SVSbytsSD1YGzDIcrAJmdFec88u/+VoGmxnl+80gL1tRCQ==} engines: {node: '>= 6'} @@ -5336,10 +5327,6 @@ packages: engines: {node: '>=18.3.0'} hasBin: true - formdata-node@4.4.1: - resolution: {integrity: sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==} - engines: {node: '>= 12.20'} - formdata-polyfill@4.0.10: resolution: {integrity: sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==} engines: {node: '>=12.20.0'} @@ -5683,9 +5670,6 @@ packages: resolution: {integrity: sha512-eKCa6bwnJhvxj14kZk5NCPc6Hb6BdsU9DZcOnmQKSnO1VKrfV0zCvtttPZUsBvjmNDn8rpcJfpwSYnHBjc95MQ==} engines: {node: '>=18.18.0'} - humanize-ms@1.2.1: - resolution: {integrity: sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==} - husky@9.1.7: resolution: {integrity: sha512-5gs5ytaNjBrh5Ow3zrvdUUY+0VxIuWVL4i9irt6friV+BqdCfmV11CQTWMiBYWHbXhco+J1kHfTOUkePhCDvMA==} engines: {node: '>=18'} @@ -6155,6 +6139,10 @@ packages: json-buffer@3.0.1: resolution: {integrity: sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==} + json-schema-to-ts@3.1.1: + resolution: {integrity: sha512-+DWg8jCJG2TEnpy7kOm/7/AxaYoaRbjVB4LFZLySZlWn8exGs3A4OLJR966cVvU26N7X9TWxl+Jsw7dzAqKT6g==} + engines: {node: '>=16'} + json-schema-traverse@0.4.1: resolution: {integrity: sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==} @@ -8063,6 +8051,9 @@ packages: standard-as-callback@2.1.0: resolution: {integrity: sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==} + standardwebhooks@1.0.0: + resolution: {integrity: sha512-BbHGOQK9olHPMvQNHWul6MYlrRTAOKn03rOe4A8O3CLWhNf4YHBqq2HJKKC+sfqpxiBY52pNeesD6jIiLDz8jg==} + statuses@2.0.2: resolution: {integrity: sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==} engines: {node: '>= 0.8'} @@ -8398,6 +8389,9 @@ packages: truncate-utf8-bytes@1.0.2: resolution: {integrity: sha512-95Pu1QXQvruGEhv62XCMO3Mm90GscOCClvrIUwCM0PYOXK3kaF3l3sIHxx71ThJfcbM2O5Au6SO3AWCSEfW4mQ==} + ts-algebra@2.0.0: + resolution: {integrity: sha512-FPAhNPFMrkwz76P7cdjdmiShwMynZYN6SgOujD1urY4oNm80Ou9oMdmbR45LotcKOXoy7wSmHkRFE6Mxbrhefw==} + ts-api-utils@2.1.0: resolution: {integrity: sha512-CUgTZL1irw8u29bzrOD/nH85jqyc74D6SshFgujOIA7osm2Rz7dYH77agkx7H4FBNxDq7Cjf+IjaX/8zwFW+ZQ==} engines: {node: '>=18.12'} @@ -8518,9 +8512,6 @@ packages: underscore@1.13.8: resolution: {integrity: sha512-DXtD3ZtEQzc7M8m4cXotyHR+FAS18C64asBYY5vqZexfYryNNnDc02W4hKg3rdQuqOYas1jkseX0+nZXjTXnvQ==} - undici-types@5.26.5: - resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==} - undici-types@6.21.0: resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==} @@ -8831,10 +8822,6 @@ packages: resolution: {integrity: sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==} engines: {node: '>= 8'} - web-streams-polyfill@4.0.0-beta.3: - resolution: {integrity: sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==} - engines: {node: '>= 14'} - web-tree-sitter@0.25.6: resolution: {integrity: sha512-WG+/YGbxw8r+rLlzzhV+OvgiOJCWdIpOucG3qBf3RCBFMkGDb1CanUi2BxCxjnkpzU3/hLWPT8VO5EKsMk9Fxg==} @@ -9250,25 +9237,21 @@ snapshots: package-manager-detector: 1.6.0 tinyexec: 1.2.4 - '@anthropic-ai/sdk@0.37.0': + '@anthropic-ai/sdk@0.104.1(zod@3.25.76)': dependencies: - '@types/node': 18.19.100 - '@types/node-fetch': 2.6.12 - abort-controller: 3.0.0 - agentkeepalive: 4.6.0 - form-data-encoder: 1.7.2 - formdata-node: 4.4.1 - node-fetch: 2.7.0 - transitivePeerDependencies: - - encoding + json-schema-to-ts: 3.1.1 + standardwebhooks: 1.0.0 + optionalDependencies: + zod: 3.25.76 - '@anthropic-ai/vertex-sdk@0.7.0': + '@anthropic-ai/vertex-sdk@0.17.1(zod@3.25.76)': dependencies: - '@anthropic-ai/sdk': 0.37.0 + '@anthropic-ai/sdk': 0.104.1(zod@3.25.76) google-auth-library: 9.15.1 transitivePeerDependencies: - encoding - supports-color + - zod '@asamuzakjp/css-color@3.2.0': dependencies: @@ -11836,6 +11819,8 @@ snapshots: dependencies: tslib: 2.8.1 + '@stablelib/base64@1.0.1': {} + '@standard-schema/spec@1.1.0': {} '@tailwindcss/node@4.1.6': @@ -12227,11 +12212,6 @@ snapshots: dependencies: node-cache: 5.1.2 - '@types/node-fetch@2.6.12': - dependencies: - '@types/node': 20.19.43 - form-data: 4.0.6 - '@types/node-ipc@9.2.3': dependencies: '@types/node': 20.19.43 @@ -12240,10 +12220,6 @@ snapshots: '@types/node@14.18.63': {} - '@types/node@18.19.100': - dependencies: - undici-types: 5.26.5 - '@types/node@20.19.43': dependencies: undici-types: 6.21.0 @@ -12580,10 +12556,6 @@ snapshots: '@xobotyi/scrollbar-width@1.9.5': {} - abort-controller@3.0.0: - dependencies: - event-target-shim: 5.0.1 - accepts@2.0.0: dependencies: mime-types: 3.0.1 @@ -12603,10 +12575,6 @@ snapshots: agent-base@7.1.3: {} - agentkeepalive@4.6.0: - dependencies: - humanize-ms: 1.2.1 - ai-sdk-provider-poe@2.0.18(ai@6.0.77(zod@3.25.76))(zod@3.25.76): dependencies: '@ai-sdk/anthropic': 3.0.42(zod@3.25.76) @@ -13992,8 +13960,6 @@ snapshots: stream-combiner: 0.0.4 through: 2.3.8 - event-target-shim@5.0.1: {} - eventemitter3@5.0.4: {} eventsource-parser@3.0.8: {} @@ -14144,6 +14110,8 @@ snapshots: fast-levenshtein@2.0.6: {} + fast-sha256@1.3.0: {} + fast-shallow-equal@1.0.0: {} fast-uri@3.1.0: {} @@ -14240,8 +14208,6 @@ snapshots: cross-spawn: 7.0.6 signal-exit: 4.1.0 - form-data-encoder@1.7.2: {} - form-data@4.0.6: dependencies: asynckit: 0.4.0 @@ -14254,11 +14220,6 @@ snapshots: dependencies: fd-package-json: 2.0.0 - formdata-node@4.4.1: - dependencies: - node-domexception: 1.0.0 - web-streams-polyfill: 4.0.0-beta.3 - formdata-polyfill@4.0.10: dependencies: fetch-blob: 3.2.0 @@ -14716,10 +14677,6 @@ snapshots: human-signals@8.0.1: {} - humanize-ms@1.2.1: - dependencies: - ms: 2.1.3 - husky@9.1.7: {} hyphenate-style-name@1.1.0: {} @@ -15177,6 +15134,11 @@ snapshots: json-buffer@3.0.1: {} + json-schema-to-ts@3.1.1: + dependencies: + '@babel/runtime': 7.29.7 + ts-algebra: 2.0.0 + json-schema-traverse@0.4.1: {} json-schema-traverse@1.0.0: {} @@ -17511,6 +17473,11 @@ snapshots: standard-as-callback@2.1.0: {} + standardwebhooks@1.0.0: + dependencies: + '@stablelib/base64': 1.0.1 + fast-sha256: 1.3.0 + statuses@2.0.2: {} std-env@4.1.0: {} @@ -17866,6 +17833,8 @@ snapshots: dependencies: utf8-byte-length: 1.0.5 + ts-algebra@2.0.0: {} + ts-api-utils@2.1.0(typescript@5.8.3): dependencies: typescript: 5.8.3 @@ -18011,8 +17980,6 @@ snapshots: underscore@1.13.8: {} - undici-types@5.26.5: {} - undici-types@6.21.0: {} undici@6.27.0: {} @@ -18291,8 +18258,6 @@ snapshots: web-streams-polyfill@3.3.3: {} - web-streams-polyfill@4.0.0-beta.3: {} - web-tree-sitter@0.25.6: {} web-vitals@4.2.4: {} diff --git a/src/api/providers/__tests__/native-ollama.spec.ts b/src/api/providers/__tests__/native-ollama.spec.ts index 200868022b..c46eeeeb00 100644 --- a/src/api/providers/__tests__/native-ollama.spec.ts +++ b/src/api/providers/__tests__/native-ollama.spec.ts @@ -81,6 +81,54 @@ describe("NativeOllamaHandler", () => { expect(results[2]).toEqual({ type: "usage", inputTokens: 10, outputTokens: 2 }) }) + it("should map tool_result array content to a concatenated string (text kept, others dropped)", async () => { + mockChat.mockImplementation(async function* () { + yield { message: { content: "ok" } } + }) + + const messages = [ + { + role: "user" as const, + content: [ + { + type: "tool_result" as const, + tool_use_id: "tool-1", + content: [ + { type: "text" as const, text: "line one" }, + // Non-text block (document) should be dropped to "" + { + type: "document" as const, + source: { + type: "base64" as const, + media_type: "application/pdf", + data: "abc", + } as const, + }, + { type: "text" as const, text: "line two" }, + ], + }, + ], + }, + ] + + const stream = handler.createMessage("System", messages) + for await (const _ of stream) { + // consume stream + } + + // Text blocks are joined with "\n"; the non-text block contributes "" + expect(mockChat).toHaveBeenCalledWith( + expect.objectContaining({ + messages: expect.arrayContaining([ + expect.objectContaining({ + role: "user", + content: "line one\n\nline two", + }), + ]), + }), + ) + }) + it("should not include num_ctx by default", async () => { // Mock the chat response mockChat.mockImplementation(async function* () { diff --git a/src/api/providers/native-ollama.ts b/src/api/providers/native-ollama.ts index 7ee91282a4..4574c184f2 100644 --- a/src/api/providers/native-ollama.ts +++ b/src/api/providers/native-ollama.ts @@ -60,7 +60,10 @@ function convertToOllamaMessages(anthropicMessages: Anthropic.Messages.MessagePa } return "(see following user message for image)" } - return part.text + if (part.type === "text") { + return part.text + } + return "" }) .join("\n") ?? "" } diff --git a/src/api/providers/openai-codex.ts b/src/api/providers/openai-codex.ts index bc9d4cd26a..2b2a599c0f 100644 --- a/src/api/providers/openai-codex.ts +++ b/src/api/providers/openai-codex.ts @@ -429,8 +429,10 @@ export class OpenAiCodexHandler extends BaseProvider implements SingleCompletion content.push({ type: "input_text", text: block.text }) } else if (block.type === "image") { const image = block as Anthropic.Messages.ImageBlockParam - const imageUrl = `data:${image.source.media_type};base64,${image.source.data}` - content.push({ type: "input_image", image_url: imageUrl }) + if (image.source.type === "base64") { + const imageUrl = `data:${image.source.media_type};base64,${image.source.data}` + content.push({ type: "input_image", image_url: imageUrl }) + } } else if (block.type === "tool_result") { const result = typeof block.content === "string" diff --git a/src/api/providers/openai-native.ts b/src/api/providers/openai-native.ts index ea7a0667f3..8f86f46751 100644 --- a/src/api/providers/openai-native.ts +++ b/src/api/providers/openai-native.ts @@ -483,8 +483,10 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio content.push({ type: "input_text", text: block.text }) } else if (block.type === "image") { const image = block as Anthropic.Messages.ImageBlockParam - const imageUrl = `data:${image.source.media_type};base64,${image.source.data}` - content.push({ type: "input_image", image_url: imageUrl }) + if (image.source.type === "base64") { + const imageUrl = `data:${image.source.media_type};base64,${image.source.data}` + content.push({ type: "input_image", image_url: imageUrl }) + } } else if (block.type === "tool_result") { // Map Anthropic tool_result to Responses API function_call_output item const result = diff --git a/src/api/transform/__tests__/mistral-format.spec.ts b/src/api/transform/__tests__/mistral-format.spec.ts index 290bea1ec5..6f3984bcc7 100644 --- a/src/api/transform/__tests__/mistral-format.spec.ts +++ b/src/api/transform/__tests__/mistral-format.spec.ts @@ -106,6 +106,45 @@ describe("convertToMistralMessages", () => { }) }) + it("should handle user messages with URL image content", () => { + const anthropicMessages: Anthropic.Messages.MessageParam[] = [ + { + role: "user", + content: [ + { type: "text", text: "Describe this:" }, + { + type: "image", + source: { type: "url", url: "https://example.com/photo.jpg" } as any, + }, + ], + }, + ] + + const mistralMessages = convertToMistralMessages(anthropicMessages) + const content = mistralMessages[0].content as Array<{ type: string; imageUrl?: { url: string }; text?: string }> + + expect(content[1]).toEqual({ type: "image_url", imageUrl: { url: "https://example.com/photo.jpg" } }) + }) + + it("should fall back to [Image] placeholder for unsupported image source types", () => { + const anthropicMessages: Anthropic.Messages.MessageParam[] = [ + { + role: "user", + content: [ + { + type: "image", + source: { type: "file" } as any, + }, + ], + }, + ] + + const mistralMessages = convertToMistralMessages(anthropicMessages) + const content = mistralMessages[0].content as Array<{ type: string; text?: string }> + + expect(content[0]).toEqual({ type: "text", text: "[Image]" }) + }) + it("should handle user messages with only tool results", () => { const anthropicMessages: Anthropic.Messages.MessageParam[] = [ { diff --git a/src/api/transform/__tests__/openai-format.spec.ts b/src/api/transform/__tests__/openai-format.spec.ts index 1a4c7f6518..b58deea649 100644 --- a/src/api/transform/__tests__/openai-format.spec.ts +++ b/src/api/transform/__tests__/openai-format.spec.ts @@ -1,4 +1,4 @@ -// npx vitest run api/transform/__tests__/openai-format.spec.ts +// pnpm exec vitest run api/transform/__tests__/openai-format.spec.ts import { Anthropic } from "@anthropic-ai/sdk" import OpenAI from "openai" @@ -76,6 +76,123 @@ describe("convertToOpenAiMessages", () => { }) }) + it("should handle messages with URL image content", () => { + const anthropicMessages: Anthropic.Messages.MessageParam[] = [ + { + role: "user", + content: [ + { type: "text", text: "What is in this image?" }, + { + type: "image", + source: { type: "url", url: "https://example.com/image.png" } as any, + }, + ], + }, + ] + + const openAiMessages = convertToOpenAiMessages(anthropicMessages) + const content = openAiMessages[0].content as Array<{ type: string; text?: string; image_url?: { url: string } }> + + expect(content[1]).toEqual({ + type: "image_url", + image_url: { url: "https://example.com/image.png" }, + }) + }) + + it("should fall back to [Image] placeholder for unsupported image source types", () => { + const anthropicMessages: Anthropic.Messages.MessageParam[] = [ + { + role: "user", + content: [ + { + type: "image", + source: { type: "file" } as any, + }, + ], + }, + ] + + const openAiMessages = convertToOpenAiMessages(anthropicMessages) + const content = openAiMessages[0].content as Array<{ type: string; text?: string }> + + expect(content[0]).toEqual({ type: "text", text: "[Image]" }) + }) + + it("should handle non-text/non-image blocks in tool results as empty string", () => { + const anthropicMessages: Anthropic.Messages.MessageParam[] = [ + { + role: "user", + content: [ + { + type: "tool_result", + tool_use_id: "tool-1", + content: [ + { type: "text", text: "some text" }, + { + type: "document", + source: { type: "base64", media_type: "application/pdf", data: "abc" }, + } as any, + ], + }, + ], + }, + ] + + const openAiMessages = convertToOpenAiMessages(anthropicMessages) + expect(openAiMessages[0].role).toBe("tool") + expect((openAiMessages[0] as any).content).toBe("some text\n") + }) + + it("should handle base64 image in tool result with placeholder", () => { + const anthropicMessages: Anthropic.Messages.MessageParam[] = [ + { + role: "user", + content: [ + { + type: "tool_result", + tool_use_id: "tool-1", + content: [ + { + type: "image", + source: { type: "base64", media_type: "image/png", data: "base64data" }, + }, + ], + }, + ], + }, + ] + + const openAiMessages = convertToOpenAiMessages(anthropicMessages) + expect(openAiMessages[0].role).toBe("tool") + // base64 images in tool results emit a placeholder; the image itself is + // flushed in a separate user message (see comment block in openai-format.ts) + expect((openAiMessages[0] as any).content).toBe("(see following user message for image)") + }) + + it("should render [Image] placeholder for URL image in tool result", () => { + const anthropicMessages: Anthropic.Messages.MessageParam[] = [ + { + role: "user", + content: [ + { + type: "tool_result", + tool_use_id: "tool-1", + content: [ + { + type: "image", + source: { type: "url", url: "https://example.com/img.png" } as any, + }, + ], + }, + ], + }, + ] + + const openAiMessages = convertToOpenAiMessages(anthropicMessages) + expect(openAiMessages[0].role).toBe("tool") + expect((openAiMessages[0] as any).content).toBe("[Image]") + }) + it("should handle assistant messages with tool use (no normalization without normalizeToolCallId)", () => { const anthropicMessages: Anthropic.Messages.MessageParam[] = [ { diff --git a/src/api/transform/__tests__/vscode-lm-format.spec.ts b/src/api/transform/__tests__/vscode-lm-format.spec.ts index 67f835f6ab..6e58c75ad4 100644 --- a/src/api/transform/__tests__/vscode-lm-format.spec.ts +++ b/src/api/transform/__tests__/vscode-lm-format.spec.ts @@ -1,4 +1,4 @@ -// npx vitest run src/api/transform/__tests__/vscode-lm-format.spec.ts +// pnpm exec vitest run api/transform/__tests__/vscode-lm-format.spec.ts import { Anthropic } from "@anthropic-ai/sdk" import * as vscode from "vscode" @@ -178,6 +178,48 @@ describe("convertToVsCodeLmMessages", () => { const imagePlaceholder = result[0].content[1] as MockLanguageModelTextPart expect(imagePlaceholder.value).toContain("[Image (base64): image/png not supported by VSCode LM API]") }) + + it("should produce correct placeholder for URL image in non-tool messages", () => { + const messages: Anthropic.Messages.MessageParam[] = [ + { + role: "user", + content: [ + { + type: "image", + source: { type: "url", url: "https://example.com/img.png" } as any, + }, + ], + }, + ] + + const result = convertToVsCodeLmMessages(messages) + const imagePlaceholder = result[0].content[0] as MockLanguageModelTextPart + expect(imagePlaceholder.value).toContain("[Image (url): not supported by VSCode LM API]") + }) + + it("should produce correct placeholder for URL image inside tool result", () => { + const messages: Anthropic.Messages.MessageParam[] = [ + { + role: "user", + content: [ + { + type: "tool_result", + tool_use_id: "tool-1", + content: [ + { + type: "image", + source: { type: "url", url: "https://example.com/img.png" } as any, + }, + ], + }, + ], + }, + ] + + const result = convertToVsCodeLmMessages(messages) + const toolResult = result[0].content[0] as any + expect(toolResult.content[0].value).toContain("[Image (url): not supported by VSCode LM API]") + }) }) describe("convertToAnthropicRole", () => { diff --git a/src/api/transform/__tests__/zai-format.spec.ts b/src/api/transform/__tests__/zai-format.spec.ts new file mode 100644 index 0000000000..27ad4a2c90 --- /dev/null +++ b/src/api/transform/__tests__/zai-format.spec.ts @@ -0,0 +1,79 @@ +// pnpm exec vitest run api/transform/__tests__/zai-format.spec.ts + +import { Anthropic } from "@anthropic-ai/sdk" + +import { convertToZAiFormat } from "../zai-format" + +describe("convertToZAiFormat", () => { + it("should convert simple user text messages", () => { + const messages: Anthropic.Messages.MessageParam[] = [{ role: "user", content: "Hello" }] + + const result = convertToZAiFormat(messages) + expect(result).toHaveLength(1) + expect(result[0]).toEqual({ role: "user", content: "Hello" }) + }) + + it("should handle base64 image in user message", () => { + const messages: Anthropic.Messages.MessageParam[] = [ + { + role: "user", + content: [ + { type: "text", text: "Describe this:" }, + { + type: "image", + source: { type: "base64", media_type: "image/png", data: "abc123" }, + }, + ], + }, + ] + + const result = convertToZAiFormat(messages) + expect(result).toHaveLength(1) + const content = (result[0] as any).content as Array<{ + type: string + image_url?: { url: string } + text?: string + }> + expect(content[0]).toEqual({ type: "text", text: "Describe this:" }) + expect(content[1]).toEqual({ type: "image_url", image_url: { url: "data:image/png;base64,abc123" } }) + }) + + it("should skip non-base64 images in user messages", () => { + const messages: Anthropic.Messages.MessageParam[] = [ + { + role: "user", + content: [ + { type: "text", text: "Hello" }, + { + type: "image", + source: { type: "url", url: "https://example.com/img.png" } as any, + }, + ], + }, + ] + + const result = convertToZAiFormat(messages) + expect(result).toHaveLength(1) + // URL image is skipped — only text remains, so content is a plain string + expect((result[0] as any).content).toBe("Hello") + }) + + it("should convert tool_result content", () => { + const messages: Anthropic.Messages.MessageParam[] = [ + { + role: "user", + content: [{ type: "tool_result", tool_use_id: "tool-1", content: "Result text" }], + }, + ] + + const result = convertToZAiFormat(messages) + expect(result[0]).toEqual({ role: "tool", tool_call_id: "tool-1", content: "Result text" }) + }) + + it("should convert assistant messages with text", () => { + const messages: Anthropic.Messages.MessageParam[] = [{ role: "assistant", content: "I can help with that." }] + + const result = convertToZAiFormat(messages) + expect(result[0]).toEqual({ role: "assistant", content: "I can help with that." }) + }) +}) diff --git a/src/api/transform/mistral-format.ts b/src/api/transform/mistral-format.ts index d32f84d6e0..9a2b19098b 100644 --- a/src/api/transform/mistral-format.ts +++ b/src/api/transform/mistral-format.ts @@ -104,14 +104,26 @@ export function convertToMistralMessages(anthropicMessages: Anthropic.Messages.M role: "user", content: nonToolMessages.map((part) => { if (part.type === "image") { - return { - type: "image_url", - imageUrl: { - url: `data:${part.source.media_type};base64,${part.source.data}`, - }, + if (part.source.type === "base64") { + return { + type: "image_url", + imageUrl: { + url: `data:${part.source.media_type};base64,${part.source.data}`, + }, + } } + if (part.source.type === "url") { + return { + type: "image_url", + imageUrl: { url: part.source.url }, + } + } + return { type: "text", text: "[Image]" } + } + if (part.type === "text") { + return { type: "text", text: part.text } } - return { type: "text", text: part.text } + return { type: "text", text: "" } }), }) } diff --git a/src/api/transform/openai-format.ts b/src/api/transform/openai-format.ts index 38719feba1..1a03b85e85 100644 --- a/src/api/transform/openai-format.ts +++ b/src/api/transform/openai-format.ts @@ -358,10 +358,16 @@ export function convertToOpenAiMessages( toolMessage.content ?.map((part) => { if (part.type === "image") { - toolResultImages.push(part) - return "(see following user message for image)" + if (part.source.type === "base64") { + toolResultImages.push(part) + return "(see following user message for image)" + } + return "[Image]" } - return part.text + if (part.type === "text") { + return part.text + } + return "" }) .join("\n") ?? "" } @@ -422,12 +428,26 @@ export function convertToOpenAiMessages( role: "user", content: filteredNonToolMessages.map((part) => { if (part.type === "image") { - return { - type: "image_url", - image_url: { url: `data:${part.source.media_type};base64,${part.source.data}` }, + if (part.source.type === "base64") { + return { + type: "image_url", + image_url: { + url: `data:${part.source.media_type};base64,${part.source.data}`, + }, + } + } + if (part.source.type === "url") { + return { + type: "image_url", + image_url: { url: part.source.url }, + } } + return { type: "text", text: "[Image]" } + } + if (part.type === "text") { + return { type: "text", text: part.text } } - return { type: "text", text: part.text } + return { type: "text", text: "" } }), }) } diff --git a/src/api/transform/r1-format.ts b/src/api/transform/r1-format.ts index 6e8b782047..e59b282be2 100644 --- a/src/api/transform/r1-format.ts +++ b/src/api/transform/r1-format.ts @@ -61,10 +61,12 @@ export function convertToR1Format( if (part.type === "text") { textParts.push(part.text) } else if (part.type === "image") { - imageParts.push({ - type: "image_url", - image_url: { url: `data:${part.source.media_type};base64,${part.source.data}` }, - }) + if (part.source.type === "base64") { + imageParts.push({ + type: "image_url", + image_url: { url: `data:${part.source.media_type};base64,${part.source.data}` }, + }) + } } else if (part.type === "tool_result") { // Convert tool_result to OpenAI tool message format let content: string diff --git a/src/api/transform/responses-api-input.ts b/src/api/transform/responses-api-input.ts index a766dfef6e..9ef31be280 100644 --- a/src/api/transform/responses-api-input.ts +++ b/src/api/transform/responses-api-input.ts @@ -73,11 +73,13 @@ export function convertToResponsesApiInput(messages: Anthropic.Messages.MessageP contentParts.push({ type: "input_text", text: part.text }) break case "image": - contentParts.push({ - type: "input_image", - detail: "auto", - image_url: `data:${part.source.media_type};base64,${part.source.data}`, - }) + if (part.source.type === "base64") { + contentParts.push({ + type: "input_image", + detail: "auto", + image_url: `data:${part.source.media_type};base64,${part.source.data}`, + }) + } break case "tool_result": { // Flush any pending user content before the tool result diff --git a/src/api/transform/vscode-lm-format.ts b/src/api/transform/vscode-lm-format.ts index 388197c2c2..935f6a5474 100644 --- a/src/api/transform/vscode-lm-format.ts +++ b/src/api/transform/vscode-lm-format.ts @@ -72,11 +72,19 @@ export function convertToVsCodeLmMessages( ? [new vscode.LanguageModelTextPart(toolMessage.content)] : (toolMessage.content?.map((part) => { if (part.type === "image") { + if (part.source.type === "base64") { + return new vscode.LanguageModelTextPart( + `[Image (base64): ${part.source.media_type} not supported by VSCode LM API]`, + ) + } return new vscode.LanguageModelTextPart( - `[Image (${part.source?.type || "Unknown source-type"}): ${part.source?.media_type || "unknown media-type"} not supported by VSCode LM API]`, + `[Image (${part.source.type}): not supported by VSCode LM API]`, ) } - return new vscode.LanguageModelTextPart(part.text) + if (part.type === "text") { + return new vscode.LanguageModelTextPart(part.text) + } + return new vscode.LanguageModelTextPart("") }) ?? [new vscode.LanguageModelTextPart("")]) return new vscode.LanguageModelToolResultPart(toolMessage.tool_use_id, toolContentParts) @@ -85,11 +93,19 @@ export function convertToVsCodeLmMessages( // Convert non-tool messages to TextParts after tool messages ...nonToolMessages.map((part) => { if (part.type === "image") { + if (part.source.type === "base64") { + return new vscode.LanguageModelTextPart( + `[Image (base64): ${part.source.media_type} not supported by VSCode LM API]`, + ) + } return new vscode.LanguageModelTextPart( - `[Image (${part.source?.type || "Unknown source-type"}): ${part.source?.media_type || "unknown media-type"} not supported by VSCode LM API]`, + `[Image (${part.source.type}): not supported by VSCode LM API]`, ) } - return new vscode.LanguageModelTextPart(part.text) + if (part.type === "text") { + return new vscode.LanguageModelTextPart(part.text) + } + return new vscode.LanguageModelTextPart("") }), ] diff --git a/src/api/transform/zai-format.ts b/src/api/transform/zai-format.ts index 79b2e88aeb..0bc8353d8e 100644 --- a/src/api/transform/zai-format.ts +++ b/src/api/transform/zai-format.ts @@ -56,10 +56,12 @@ export function convertToZAiFormat( if (part.type === "text") { textParts.push(part.text) } else if (part.type === "image") { - imageParts.push({ - type: "image_url", - image_url: { url: `data:${part.source.media_type};base64,${part.source.data}` }, - }) + if (part.source.type === "base64") { + imageParts.push({ + type: "image_url", + image_url: { url: `data:${part.source.media_type};base64,${part.source.data}` }, + }) + } } else if (part.type === "tool_result") { // Convert tool_result to OpenAI tool message format let content: string diff --git a/src/integrations/misc/__tests__/export-markdown.spec.ts b/src/integrations/misc/__tests__/export-markdown.spec.ts index fd4c30c3d2..081fb26158 100644 --- a/src/integrations/misc/__tests__/export-markdown.spec.ts +++ b/src/integrations/misc/__tests__/export-markdown.spec.ts @@ -72,5 +72,31 @@ describe("export-markdown", () => { const block = { type: "unknown_type" as const } as any expect(formatContentBlockToMarkdown(block)).toBe("[Unexpected content type: unknown_type]") }) + + it("should format document blocks", () => { + const block = { + type: "document", + source: { type: "base64", media_type: "application/pdf", data: "abc" } as const, + } satisfies ExtendedContentBlock + expect(formatContentBlockToMarkdown(block)).toBe("[Document]") + }) + + it("should format search_result blocks", () => { + const block = { + type: "search_result", + source: "https://example.com", + title: "Example", + content: [{ type: "text", text: "result text" }], + } satisfies ExtendedContentBlock + expect(formatContentBlockToMarkdown(block)).toBe("[Search Result]") + }) + + it("should format tool_reference blocks", () => { + const block = { + type: "tool_reference", + tool_name: "read_file", + } satisfies ExtendedContentBlock + expect(formatContentBlockToMarkdown(block)).toBe("[Tool Reference]") + }) }) }) diff --git a/src/integrations/misc/export-markdown.ts b/src/integrations/misc/export-markdown.ts index d65bb3200e..0636a9e7b8 100644 --- a/src/integrations/misc/export-markdown.ts +++ b/src/integrations/misc/export-markdown.ts @@ -13,7 +13,11 @@ interface ThoughtSignatureBlock { type: "thoughtSignature" } -export type ExtendedContentBlock = Anthropic.Messages.ContentBlockParam | ReasoningBlock | ThoughtSignatureBlock +export type ExtendedContentBlock = + | Anthropic.Messages.ContentBlockParam + | Anthropic.Messages.ToolReferenceBlockParam + | ReasoningBlock + | ThoughtSignatureBlock export function getTaskFileName(dateTs: number): string { const date = new Date(dateTs) @@ -69,6 +73,12 @@ export function formatContentBlockToMarkdown(block: ExtendedContentBlock): strin return block.text case "image": return `[Image]` + case "document": + return `[Document]` + case "search_result": + return `[Search Result]` + case "tool_reference": + return `[Tool Reference]` case "tool_use": { let input: string if (typeof block.input === "object" && block.input !== null) { diff --git a/src/package.json b/src/package.json index 3b7295722f..0b62cacf5c 100644 --- a/src/package.json +++ b/src/package.json @@ -465,8 +465,8 @@ "@ai-sdk/google-vertex": "^4.0.45", "@ai-sdk/mistral": "^3.0.19", "@ai-sdk/xai": "^3.0.48", - "@anthropic-ai/sdk": "^0.37.0", - "@anthropic-ai/vertex-sdk": "^0.7.0", + "@anthropic-ai/sdk": "^0.104.1", + "@anthropic-ai/vertex-sdk": "^0.17.1", "@aws-sdk/client-bedrock-runtime": "^3.922.0", "@aws-sdk/credential-providers": "^3.922.0", "@google/genai": "^1.29.1",