diff --git a/bun.lock b/bun.lock index 7a40e43908..c33705bf46 100644 --- a/bun.lock +++ b/bun.lock @@ -5,9 +5,9 @@ "": { "name": "hapi", "devDependencies": { - "@playwright/test": "^1.60.0", + "@playwright/test": "^1.61.0", "concurrently": "^9.2.1", - "playwright": "1.60.0", + "playwright": "1.61.0", "react-devtools-core": "^7.0.1", "vite-plugin-pwa": "^1.2.0", }, @@ -720,7 +720,7 @@ "@pinojs/redact": ["@pinojs/redact@0.4.0", "", {}, "sha512-k2ENnmBugE/rzQfEcdWHcCY+/FM3VLzH9cYEsbdsoqrvzAKRhUZeRNhAZvB8OitQJ1TBed3yqWtdjzS6wJKBwg=="], - "@playwright/test": ["@playwright/test@1.60.0", "", { "dependencies": { "playwright": "1.60.0" }, "bin": { "playwright": "cli.js" } }, "sha512-O71yZIbAh/PxDMNGns37GHBIfrVkEVyn+AXyIa5dOTfb4/xNvRWV+Vv/NMbNCtODB/pO7vLlF2OTmMVLhmr7Ag=="], + "@playwright/test": ["@playwright/test@1.61.0", "", { "dependencies": { "playwright": "1.61.0" }, "bin": { "playwright": "cli.js" } }, "sha512-cKA5B6lpFEMyMGjxF54QihfYpB4FkEGH+qZhtArDEG+wezQAJY8Pq6C7T1SjWz+FFzt3TbyoXBQYk/0292TdJA=="], "@primer/octicons": ["@primer/octicons@19.23.1", "", { "dependencies": { "object-assign": "^4.1.1" } }, "sha512-CzjGmxkmNhyst6EekrS3SJPdtzgIkUMP/LSJch65y99/kmiFXbO1a+q7zoYe3hnI9NaOM0IN+ydDIbOmd8YqcA=="], @@ -2440,9 +2440,9 @@ "pkg-types": ["pkg-types@1.3.1", "", { "dependencies": { "confbox": "^0.1.8", "mlly": "^1.7.4", "pathe": "^2.0.1" } }, "sha512-/Jm5M4RvtBFVkKWRu2BLUTNP8/M2a+UwuAX+ae4770q1qVGtfjG+WTCupoZixokjmHiry8uI+dlY8KXYV5HVVQ=="], - "playwright": ["playwright@1.60.0", "", { "dependencies": { "playwright-core": "1.60.0" }, "optionalDependencies": { "fsevents": "2.3.2" }, "bin": { "playwright": "cli.js" } }, "sha512-hheHdokM8cdqCb0lcE3s+zT4t4W+vvjpGxsZlDnikarzx8tSzMebh3UiFtgqwFwnTnjYQcsyMF8ei2mCO/tpeA=="], + "playwright": ["playwright@1.61.0", "", { "dependencies": { "playwright-core": "1.61.0" }, "optionalDependencies": { "fsevents": "2.3.2" }, "bin": { "playwright": "cli.js" } }, "sha512-Z+7BeeqQPRRzklHsVFP4KTGIyMxKUmfeRA4WisM6G3/XW6nwGeX6fX9qYaDa+CiUqpOkb2f6X3nar05R3kSuJQ=="], - "playwright-core": ["playwright-core@1.60.0", "", { "bin": { "playwright-core": "cli.js" } }, "sha512-9bW6zvX/m0lEbgTKJ6YppOKx8H3VOPBMOCFh2irXFOT4BbHgrx5hPjwJYLT40Lu+4qtD36qKc/Hn56StUW57IA=="], + "playwright-core": ["playwright-core@1.61.0", "", { "bin": { "playwright-core": "cli.js" } }, "sha512-caX7TrY3Ml6egyDX0WUcTHDxodl/b51y5wJOdCEA36QviK/s2g081hvmGs8eaE3DWb6NYZQ6BjO/QkNRPenoPA=="], "pngjs": ["pngjs@5.0.0", "", {}, "sha512-40QW5YalBNfQo5yRYmiw7Yz6TKKVr3h6970B2YE+3fQpsWcrbj1PzJgxeJ19DRQjhMbKPIuMY8rFaXc8moolVw=="], diff --git a/cli/src/agent/backends/acp/AcpMessageHandler.test.ts b/cli/src/agent/backends/acp/AcpMessageHandler.test.ts index ed699f590a..1d9f464723 100644 --- a/cli/src/agent/backends/acp/AcpMessageHandler.test.ts +++ b/cli/src/agent/backends/acp/AcpMessageHandler.test.ts @@ -4,6 +4,7 @@ import { fileURLToPath } from 'node:url'; import type { AgentMessage } from '@/agent/types'; import { AcpMessageHandler } from './AcpMessageHandler'; import { ACP_SESSION_UPDATE_TYPES } from './constants'; +import { clearGeneratedImages } from '@/modules/common/generatedImages'; function getToolResult(messages: AgentMessage[], id: string): Extract { const result = messages.find((message): message is Extract => @@ -2093,4 +2094,93 @@ describe('AcpMessageHandler', () => { }); }); }); + + it('emits generated_image agent messages from ACP image content blocks', async () => { + const messages: AgentMessage[] = []; + const handler = new AcpMessageHandler((message) => messages.push(message)); + const pngHeader = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x00]); + + handler.handleUpdate({ + sessionUpdate: ACP_SESSION_UPDATE_TYPES.agentMessageChunk, + content: { + type: 'image', + mimeType: 'image/png', + data: pngHeader.toString('base64') + } + }); + + await vi.waitFor(() => { + expect(messages.some((message) => message.type === 'generated_image')).toBe(true); + }); + + const imageMessage = messages.find( + (message): message is Extract => + message.type === 'generated_image' + ); + expect(imageMessage?.mimeType).toBe('image/png'); + expect(imageMessage?.fileName).toBeTruthy(); + expect(imageMessage?.imageId).toBeTruthy(); + expect(imageMessage?.source).toEqual({ ingress: 'acp' }); + clearGeneratedImages(); + }); + + it('emits generated_image before later tool_call when image registration is async', async () => { + const messages: AgentMessage[] = []; + const handler = new AcpMessageHandler((message) => messages.push(message)); + const pngHeader = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x00]); + + await handler.handleUpdate({ + sessionUpdate: ACP_SESSION_UPDATE_TYPES.agentMessageChunk, + content: { + type: 'image', + mimeType: 'image/png', + data: pngHeader.toString('base64'), + }, + }); + await handler.handleUpdate({ + sessionUpdate: ACP_SESSION_UPDATE_TYPES.toolCall, + toolCallId: 'call-after-image', + title: 'Read', + kind: 'read', + status: 'in_progress', + }); + + const imageIndex = messages.findIndex((message) => message.type === 'generated_image'); + const toolIndex = messages.findIndex((message) => message.type === 'tool_call'); + expect(imageIndex).toBeGreaterThanOrEqual(0); + expect(toolIndex).toBeGreaterThan(imageIndex); + clearGeneratedImages(); + }); + + it('emits buffered text before generated_image when text precedes an ACP image block', async () => { + const messages: AgentMessage[] = []; + const handler = new AcpMessageHandler((message) => messages.push(message), { flavor: 'cursor' }); + const pngHeader = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x00]); + + await handler.handleUpdate({ + sessionUpdate: ACP_SESSION_UPDATE_TYPES.agentMessageChunk, + content: { type: 'text', text: 'Here is the screenshot:' } + }); + await handler.handleUpdate({ + sessionUpdate: ACP_SESSION_UPDATE_TYPES.agentMessageChunk, + content: { + type: 'image', + mimeType: 'image/png', + data: pngHeader.toString('base64') + } + }); + + await vi.waitFor(() => { + expect(messages.some((message) => message.type === 'generated_image')).toBe(true); + }); + + const textIndex = messages.findIndex((message) => message.type === 'text'); + const imageIndex = messages.findIndex((message) => message.type === 'generated_image'); + expect(textIndex).toBeGreaterThanOrEqual(0); + expect(imageIndex).toBeGreaterThan(textIndex); + if (messages[imageIndex]?.type === 'generated_image') { + expect(messages[imageIndex].source).toEqual({ ingress: 'acp', flavor: 'cursor' }); + } + clearGeneratedImages(); + }); }); diff --git a/cli/src/agent/backends/acp/AcpMessageHandler.ts b/cli/src/agent/backends/acp/AcpMessageHandler.ts index 68a9ac00b1..f72e6b50dd 100644 --- a/cli/src/agent/backends/acp/AcpMessageHandler.ts +++ b/cli/src/agent/backends/acp/AcpMessageHandler.ts @@ -1,5 +1,8 @@ -import type { AgentMessage, PlanItem } from '@/agent/types'; import { randomUUID } from 'node:crypto'; +import { logger } from '@/ui/logger'; +import type { AgentMessage, PlanItem } from '@/agent/types'; +import { registerGeneratedImageFromAcpBlock } from '@/modules/common/generatedImages'; +import type { InlineMediaSource } from '@/modules/common/inlineMediaSource'; import { asString, isObject } from '@hapi/protocol'; import { deriveToolNameWithSource, isPlaceholderToolName } from '@/agent/utils'; import { parseRateLimitText } from '@/agent/rateLimitParser'; @@ -381,7 +384,10 @@ export class AcpMessageHandler { private lastReasoningSnapshotText = ''; private reasoningSnapshotEmitted = false; - constructor(private readonly onMessage: (message: AgentMessage) => void) {} + constructor( + private readonly onMessage: (message: AgentMessage) => void, + private readonly options?: { flavor?: string } + ) {} /** * Emits any buffered assistant text as a single message and clears the @@ -528,7 +534,7 @@ export class AcpMessageHandler { this.reasoningSnapshotEmitted = false; } - handleUpdate(update: unknown): void { + async handleUpdate(update: unknown): Promise { if (!isObject(update)) return; const updateType = asString(update.sessionUpdate); if (!updateType) return; @@ -554,6 +560,12 @@ export class AcpMessageHandler { if (updateType === ACP_SESSION_UPDATE_TYPES.agentMessageChunk) { const content = update.content; + if (isObject(content) && content.type === 'image') { + this.flushReasoning(); + this.flushText(); + await this.emitGeneratedImageFromAcpContent(content); + return; + } const text = extractTextContent(content); if (text) { // Check once whether the buffered text is a prefix of this @@ -629,6 +641,35 @@ export class AcpMessageHandler { } } + private async emitGeneratedImageFromAcpContent(content: Record): Promise { + try { + const image = await registerGeneratedImageFromAcpBlock(content); + if (!image) { + return; + } + this.onMessage({ + type: 'generated_image', + imageId: image.id, + fileName: image.fileName, + mimeType: image.mimeType, + source: this.buildAcpInlineMediaSource(), + }); + } catch (error) { + logger.debug( + '[AcpMessageHandler] Failed to register ACP image block:', + error instanceof Error ? error.message : String(error) + ); + } + } + + private buildAcpInlineMediaSource(): InlineMediaSource { + const source: InlineMediaSource = { ingress: 'acp' }; + if (this.options?.flavor) { + source.flavor = this.options.flavor; + } + return source; + } + private handleToolCall(update: Record): void { const toolCallId = asString(update.toolCallId); if (!toolCallId) return; diff --git a/cli/src/agent/backends/acp/AcpSdkBackend.ts b/cli/src/agent/backends/acp/AcpSdkBackend.ts index b82421a94a..8c1e9b7235 100644 --- a/cli/src/agent/backends/acp/AcpSdkBackend.ts +++ b/cli/src/agent/backends/acp/AcpSdkBackend.ts @@ -70,6 +70,7 @@ export class AcpSdkBackend implements AgentBackend { private promptUsageCallback: ((msg: AgentMessage) => void) | null = null; private usageUpdateListener: ((msg: AgentMessage) => void) | null = null; private lastForwardedUsageUpdate: AcpUsageUpdate | null = null; + private sessionUpdateQueue: Promise = Promise.resolve(); /** Retry configuration for ACP initialization */ private static readonly INIT_RETRY_OPTIONS = { @@ -101,7 +102,12 @@ export class AcpSdkBackend implements AgentBackend { private static readonly LATE_FLUSH_QUIET_PERIOD_MS = 250; private static readonly LATE_FLUSH_WINDOW_MS = 6000; - constructor(private readonly options: { command: string; args?: string[]; env?: Record }) {} + constructor(private readonly options: { + command: string; + args?: string[]; + env?: Record; + flavor?: AgentFlavor; + }) {} async initialize(): Promise { if (this.transport) return; @@ -400,8 +406,9 @@ export class AcpSdkBackend implements AgentBackend { AcpSdkBackend.PRE_PROMPT_UPDATE_QUIET_PERIOD_MS, AcpSdkBackend.PRE_PROMPT_UPDATE_DRAIN_TIMEOUT_MS ); + await this.sessionUpdateQueue; this.messageHandler?.drainBuffers(); - this.messageHandler = new AcpMessageHandler(onUpdate); + this.messageHandler = new AcpMessageHandler(onUpdate, { flavor: this.options.flavor }); this.isProcessingMessage = true; this.lastSessionUpdateAt = Date.now(); this.latestUsageUpdate = null; @@ -425,12 +432,17 @@ export class AcpSdkBackend implements AgentBackend { AcpSdkBackend.UPDATE_QUIET_PERIOD_MS, AcpSdkBackend.UPDATE_DRAIN_TIMEOUT_MS ); + await this.sessionUpdateQueue; this.messageHandler?.drainBuffers(); // Block here until the model truly stops streaming straggler // chunks (or LATE_FLUSH_WINDOW_MS elapses), so turn_complete and // the launcher's ready signal only fire once every chunk has been // emitted to this turn's onUpdate. await this.drainLateBuffers(); + // Late window can enqueue async image registration; drain again + // before turn_complete so generated_image precedes turn boundary. + await this.sessionUpdateQueue; + this.messageHandler?.drainBuffers(); try { const latestUsageUpdate = this.readLatestUsageUpdate(); if (promptUsage) { @@ -543,6 +555,7 @@ export class AcpSdkBackend implements AgentBackend { async disconnect(): Promise { if (!this.transport) return; + await this.sessionUpdateQueue; this.messageHandler?.drainBuffers(); this.messageHandler = null; this.activeSessionId = null; @@ -561,8 +574,17 @@ export class AcpSdkBackend implements AgentBackend { } this.lastSessionUpdateAt = Date.now(); const update = params.update; - this.captureUsageUpdate(update); - this.messageHandler?.handleUpdate(update); + this.sessionUpdateQueue = this.sessionUpdateQueue + .then(async () => { + this.captureUsageUpdate(update); + await this.messageHandler?.handleUpdate(update); + }) + .catch((error) => { + logger.debug( + '[AcpSdkBackend] session update failed:', + error instanceof Error ? error.message : String(error) + ); + }); } private captureUsageUpdate(update: unknown): void { diff --git a/cli/src/agent/messageConverter.test.ts b/cli/src/agent/messageConverter.test.ts index 86ca29cc28..9acf60b9ad 100644 --- a/cli/src/agent/messageConverter.test.ts +++ b/cli/src/agent/messageConverter.test.ts @@ -89,4 +89,23 @@ describe('convertAgentMessage', () => { } }); }); + + it('converts generated_image messages into generated-image wire payloads', () => { + const converted = convertAgentMessage({ + type: 'generated_image', + imageId: 'img-1', + fileName: 'inline.png', + mimeType: 'image/png', + source: { ingress: 'mcp', toolName: 'display_image' }, + }); + + expect(converted).toMatchObject({ + type: 'generated-image', + imageId: 'img-1', + fileName: 'inline.png', + mimeType: 'image/png', + source: { ingress: 'mcp', toolName: 'display_image' }, + }); + expect(converted && 'id' in converted && typeof converted.id === 'string').toBe(true); + }); }); diff --git a/cli/src/agent/messageConverter.ts b/cli/src/agent/messageConverter.ts index 4e1dbed50d..126874a0c5 100644 --- a/cli/src/agent/messageConverter.ts +++ b/cli/src/agent/messageConverter.ts @@ -1,5 +1,6 @@ import { randomUUID } from 'node:crypto'; import type { AgentMessage, PlanItem } from './types'; +import type { InlineMediaSource } from '@/modules/common/inlineMediaSource'; export type CodexMessage = | { type: 'message'; message: string } @@ -32,7 +33,15 @@ export type CodexMessage = is_error?: boolean; } | { type: 'plan'; entries: PlanItem[] } - | { type: 'error'; message: string }; + | { type: 'error'; message: string } + | { + type: 'generated-image'; + imageId: string; + fileName: string; + mimeType: string; + id: string; + source?: InlineMediaSource; + }; export function convertAgentMessage(message: AgentMessage): CodexMessage | null { switch (message.type) { @@ -78,6 +87,15 @@ export function convertAgentMessage(message: AgentMessage): CodexMessage | null type: 'plan', entries: message.items }; + case 'generated_image': + return { + type: 'generated-image', + imageId: message.imageId, + fileName: message.fileName, + mimeType: message.mimeType, + id: randomUUID(), + source: message.source, + }; case 'error': return { type: 'error', message: message.message }; case 'turn_complete': diff --git a/cli/src/agent/types.ts b/cli/src/agent/types.ts index 03d8543187..e867f55667 100644 --- a/cli/src/agent/types.ts +++ b/cli/src/agent/types.ts @@ -1,4 +1,5 @@ import type { AgentFlavor } from '@hapi/protocol'; +import type { InlineMediaSource } from '@/modules/common/inlineMediaSource'; export type McpEnvVar = { name: string; @@ -44,6 +45,7 @@ export type AgentMessage = contextWindow?: number; } | { type: 'plan'; items: PlanItem[] } + | { type: 'generated_image'; imageId: string; fileName: string; mimeType: string; source?: InlineMediaSource } | { type: 'turn_complete'; stopReason: string } | { type: 'error'; message: string }; diff --git a/cli/src/claude/utils/startHappyServer.ts b/cli/src/claude/utils/startHappyServer.ts index b2001a5703..5fb8d9107e 100644 --- a/cli/src/claude/utils/startHappyServer.ts +++ b/cli/src/claude/utils/startHappyServer.ts @@ -12,7 +12,8 @@ import { z } from "zod"; import { logger } from "@/ui/logger"; import { ApiSessionClient } from "@/api/apiSession"; import { randomUUID } from "node:crypto"; -import { detectImageMimeType, registerGeneratedImage } from "@/modules/common/generatedImages"; +import { detectImageMimeType, detectVideoMimeType, registerGeneratedImage } from "@/modules/common/generatedImages"; +import type { InlineMediaSource } from "@/modules/common/inlineMediaSource"; type StartHappyServerOptions = { emitTitleSummary?: boolean; @@ -50,8 +51,62 @@ function createHapiMcpServer(client: ApiSessionClient, emitTitleSummary: boolean title: z.string().optional().describe('Optional display title or filename for the image'), }); + const displayVideoInputSchema: z.ZodTypeAny = z.object({ + path: z.string().describe('Local filesystem path of the video to display inline (mp4 or webm)'), + title: z.string().optional().describe('Optional display title or filename for the video'), + }); + + const maxInlineMediaBytes = 25 * 1024 * 1024; + + async function displayInlineMedia( + args: { path: string; title?: string }, + mediaKind: 'image' | 'video', + toolName: 'display_image' | 'display_video' + ) { + const info = await lstat(args.path); + if (!info.isFile()) { + throw new Error('Path is not a regular file'); + } + + if (info.size > maxInlineMediaBytes) { + throw new Error('File is too large to display inline'); + } + + const bytes = await readFile(args.path); + const mimeType = mediaKind === 'video' + ? detectVideoMimeType(bytes) + : detectImageMimeType(bytes); + if (!mimeType) { + throw new Error(mediaKind === 'video' ? 'Unsupported video content' : 'Unsupported image content'); + } + + const media = registerGeneratedImage({ + id: randomUUID(), + path: args.path, + fileName: args.title, + mimeType, + bytes + }); + + const source: InlineMediaSource = { + ingress: 'mcp', + toolName, + }; + + client.sendAgentMessage({ + type: 'generated-image', + imageId: media.id, + fileName: media.fileName, + mimeType: media.mimeType, + id: randomUUID(), + source, + }); + + return media; + } + mcp.registerTool('change_title', { - description: 'Change the title of the current chat session', + description: 'Change the title of the current HAPI chat session. Call once when the user\'s primary objective is clear; use a concise task title.', title: 'Change Chat Title', inputSchema: changeTitleInputSchema, }, async (args: { title: string }) => { @@ -82,44 +137,14 @@ function createHapiMcpServer(client: ApiSessionClient, emitTitleSummary: boolean }); mcp.registerTool('display_image', { - description: 'Display a local image file inline in the current HAPI chat session', + description: 'Display a local image file inline in the current HAPI chat session. Call with the absolute filesystem path when the user should see a screenshot, diagram, or generated image.', title: 'Display Image', inputSchema: displayImageInputSchema, }, async (args: { path: string; title?: string }) => { logger.debug('[hapiMCP] Display image:', args.path); try { - const info = await lstat(args.path); - if (!info.isFile()) { - throw new Error('Path is not a regular file'); - } - - const maxImageBytes = 25 * 1024 * 1024; - if (info.size > maxImageBytes) { - throw new Error('Image is too large to display inline'); - } - - const bytes = await readFile(args.path); - const mimeType = detectImageMimeType(bytes); - if (!mimeType) { - throw new Error('Unsupported image content'); - } - - const image = registerGeneratedImage({ - id: randomUUID(), - path: args.path, - fileName: args.title, - mimeType, - bytes - }); - - client.sendAgentMessage({ - type: 'generated-image', - imageId: image.id, - fileName: image.fileName, - mimeType: image.mimeType, - id: randomUUID() - }); + const image = await displayInlineMedia(args, 'image', 'display_image'); return { content: [ @@ -145,6 +170,40 @@ function createHapiMcpServer(client: ApiSessionClient, emitTitleSummary: boolean } }); + mcp.registerTool('display_video', { + description: 'Display a local mp4 or webm file inline in the current HAPI chat session. Call with the absolute filesystem path when the user should see a screen recording or video artifact.', + title: 'Display Video', + inputSchema: displayVideoInputSchema, + }, async (args: { path: string; title?: string }) => { + logger.debug('[hapiMCP] Display video:', args.path); + + try { + const video = await displayInlineMedia(args, 'video', 'display_video'); + + return { + content: [ + { + type: 'text' as const, + text: `Displayed video: ${video.fileName}`, + }, + ], + isError: false, + }; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + logger.debug('[hapiMCP] Failed to display video:', message); + return { + content: [ + { + type: 'text' as const, + text: `Failed to display video: ${message}`, + }, + ], + isError: true, + }; + } + }); + return mcp; } @@ -221,7 +280,7 @@ export async function startHappyServer(client: ApiSessionClient, options: StartH return { url: mcpUrl, - toolNames: ['change_title', 'display_image'], + toolNames: ['change_title', 'display_image', 'display_video'], stop: () => { logger.debug('[hapiMCP] Stopping server'); for (const mcp of mcps.values()) { diff --git a/cli/src/claude/utils/systemPrompt.ts b/cli/src/claude/utils/systemPrompt.ts index 9e90179d22..eb0e07aef7 100644 --- a/cli/src/claude/utils/systemPrompt.ts +++ b/cli/src/claude/utils/systemPrompt.ts @@ -1,12 +1,14 @@ import { trimIdent } from "@/utils/trimIdent"; import { shouldIncludeCoAuthoredBy } from "./claudeSettings"; +import { DISPLAY_IMAGE_PROMPT_CLAUDE, DISPLAY_VIDEO_PROMPT_CLAUDE } from "@/modules/common/displayImagePrompt"; /** * Base system prompt shared across all configurations */ const BASE_SYSTEM_PROMPT = (() => trimIdent(` Use the title tool sparingly. For a new chat, call the tool "mcp__hapi__change_title" once after the user's initial request is clear, and set a concise task title. Do not rename the chat for routine progress, substeps, implementation details, or a slightly better wording. Rename only when the user's primary objective changes substantially and the existing title would be misleading. - When you create or find a local image file that the user should see, call the tool "mcp__hapi__display_image" with the image path so HAPI can show it inline. + ${DISPLAY_IMAGE_PROMPT_CLAUDE} + ${DISPLAY_VIDEO_PROMPT_CLAUDE} `))(); /** diff --git a/cli/src/codex/codexRemoteLauncher.ts b/cli/src/codex/codexRemoteLauncher.ts index 2cb80fe17f..61a8d8846e 100644 --- a/cli/src/codex/codexRemoteLauncher.ts +++ b/cli/src/codex/codexRemoteLauncher.ts @@ -1,6 +1,5 @@ import React from 'react'; import { randomUUID } from 'node:crypto'; -import { lstat, readFile } from 'node:fs/promises'; import { CodexAppServerClient } from './codexAppServerClient'; import { CodexPermissionHandler } from './utils/permissionHandler'; @@ -14,7 +13,7 @@ import type { CodexSession } from './session'; import type { EnhancedMode } from './loop'; import { hasCodexCliOverrides } from './utils/codexCliOverrides'; import { AppServerEventConverter } from './utils/appServerEventConverter'; -import { detectImageMimeType, registerGeneratedImage } from '@/modules/common/generatedImages'; +import { registerGeneratedImageFromPath } from '@/modules/common/generatedImages'; import { registerAppServerPermissionHandlers } from './utils/appServerPermissionAdapter'; import { buildThreadStartParams, buildTurnStartParams } from './utils/appServerConfig'; import type { ThreadGoal, ThreadGoalStatus } from './appServerTypes'; @@ -27,32 +26,17 @@ import { } from '@/modules/common/remote/RemoteLauncherBase'; -async function registerGeneratedImageFromPath(args: { id: string; path: string; fileName?: string | null }): Promise | null> { - try { - const info = await lstat(args.path); - if (!info.isFile()) { - throw new Error('Path is not a regular file'); - } - const maxImageBytes = 25 * 1024 * 1024; - if (info.size > maxImageBytes) { - throw new Error('Image is too large to display inline'); - } - const bytes = await readFile(args.path); - const mimeType = detectImageMimeType(bytes); - if (!mimeType) { - throw new Error('Unsupported image content'); - } - return registerGeneratedImage({ - id: args.id, - path: args.path, - fileName: args.fileName, - mimeType, - bytes - }); - } catch (error) { - logger.debug('[CodexRemoteLauncher] Failed to register generated image:', error instanceof Error ? error.message : String(error)); - return null; + +async function registerGeneratedImageFromPathWrapper(args: { id: string; path: string; fileName?: string | null }): Promise> | null> { + const image = await registerGeneratedImageFromPath({ + id: args.id, + path: args.path, + fileName: args.fileName + }); + if (!image) { + logger.debug('[CodexRemoteLauncher] Failed to register generated image from path'); } + return image; } type HappyServer = Awaited>['server']; @@ -2215,7 +2199,7 @@ class CodexRemoteLauncher extends RemoteLauncherBase { const imageId = randomUUID(); const savedPath = asString(msg.saved_path ?? msg.savedPath); if (savedPath) { - const image = await registerGeneratedImageFromPath({ + const image = await registerGeneratedImageFromPathWrapper({ id: imageId, path: savedPath, fileName: asString(msg.file_name ?? msg.fileName) @@ -2229,7 +2213,12 @@ class CodexRemoteLauncher extends RemoteLauncherBase { sourceImageId, fileName: image.fileName, mimeType: image.mimeType, - id: randomUUID() + id: randomUUID(), + source: { + ingress: 'tool_result', + flavor: 'codex', + toolCallId: asString(msg.call_id ?? msg.callId), + }, }); } } diff --git a/cli/src/codex/happyMcpStdioBridge.ts b/cli/src/codex/happyMcpStdioBridge.ts index 7c30617c2b..518eeffb9b 100644 --- a/cli/src/codex/happyMcpStdioBridge.ts +++ b/cli/src/codex/happyMcpStdioBridge.ts @@ -123,6 +123,34 @@ export async function runHappyMcpStdioBridge(argv: string[]): Promise { } ); + const displayVideoInputSchema: z.ZodTypeAny = z.object({ + path: z.string().describe('Local filesystem path of the video to display inline (mp4 or webm)'), + title: z.string().optional().describe('Optional display title or filename for the video'), + }); + + server.registerTool( + 'display_video', + { + description: 'Display a local mp4 or webm file inline in the current HAPI chat session', + title: 'Display Video', + inputSchema: displayVideoInputSchema, + }, + async (args: Record) => { + try { + const client = await ensureHttpClient(); + const response = await client.callTool({ name: 'display_video', arguments: args }); + return response as any; + } catch (error) { + return { + content: [ + { type: 'text' as const, text: `Failed to display video: ${error instanceof Error ? error.message : String(error)}` }, + ], + isError: true, + }; + } + } + ); + // Start STDIO transport const stdio = new StdioServerTransport(); await server.connect(stdio); diff --git a/cli/src/codex/utils/buildHapiMcpBridge.test.ts b/cli/src/codex/utils/buildHapiMcpBridge.test.ts new file mode 100644 index 0000000000..164cb88d53 --- /dev/null +++ b/cli/src/codex/utils/buildHapiMcpBridge.test.ts @@ -0,0 +1,31 @@ +import { describe, expect, it, vi } from 'vitest'; +import { buildHapiMcpBridge } from './buildHapiMcpBridge'; + +vi.mock('@/claude/utils/startHappyServer', () => ({ + startHappyServer: vi.fn(async () => ({ + url: 'http://127.0.0.1:63995/', + stop: vi.fn(), + toolNames: ['change_title', 'display_image', 'display_video'] + })) +})); + +vi.mock('@/utils/spawnHappyCLI', () => ({ + getHappyCliCommand: vi.fn(() => ({ + command: 'hapi', + args: ['mcp', '--url', 'http://127.0.0.1:63995/'] + })) +})); + +describe('buildHapiMcpBridge', () => { + it('auto-approves change_title, display_image, and display_video MCP tools', async () => { + const client = {} as never; + const bridge = await buildHapiMcpBridge(client); + + expect(bridge.mcpServers.hapi.tools).toEqual({ + change_title: { approval_mode: 'approve' }, + display_image: { approval_mode: 'approve' }, + display_video: { approval_mode: 'approve' } + }); + expect(bridge.server.url).toBe('http://127.0.0.1:63995/'); + }); +}); diff --git a/cli/src/codex/utils/buildHapiMcpBridge.ts b/cli/src/codex/utils/buildHapiMcpBridge.ts index 72ef85eaa1..7fd7cf329c 100644 --- a/cli/src/codex/utils/buildHapiMcpBridge.ts +++ b/cli/src/codex/utils/buildHapiMcpBridge.ts @@ -74,6 +74,12 @@ export async function buildHapiMcpBridge( tools: { change_title: { approval_mode: 'approve' + }, + display_image: { + approval_mode: 'approve' + }, + display_video: { + approval_mode: 'approve' } } } diff --git a/cli/src/codex/utils/codexMcpConfig.test.ts b/cli/src/codex/utils/codexMcpConfig.test.ts index 298f2dbd0e..6ced105183 100644 --- a/cli/src/codex/utils/codexMcpConfig.test.ts +++ b/cli/src/codex/utils/codexMcpConfig.test.ts @@ -31,6 +31,12 @@ describe('codexMcpConfig', () => { tools: { change_title: { approval_mode: 'approve' as const + }, + display_image: { + approval_mode: 'approve' as const + }, + display_video: { + approval_mode: 'approve' as const } } } @@ -39,6 +45,8 @@ describe('codexMcpConfig', () => { const args = buildMcpServerConfigArgs(mcpServers); expect(args).toContain('mcp_servers.hapi.tools.change_title.approval_mode="approve"'); + expect(args).toContain('mcp_servers.hapi.tools.display_image.approval_mode="approve"'); + expect(args).toContain('mcp_servers.hapi.tools.display_video.approval_mode="approve"'); }); it('builds config args for multiple MCP servers', () => { diff --git a/cli/src/codex/utils/systemPrompt.ts b/cli/src/codex/utils/systemPrompt.ts index a6057be03b..82e3da28f3 100644 --- a/cli/src/codex/utils/systemPrompt.ts +++ b/cli/src/codex/utils/systemPrompt.ts @@ -6,6 +6,7 @@ */ import { trimIdent } from '@/utils/trimIdent'; +import { DISPLAY_IMAGE_PROMPT_CODEX, DISPLAY_VIDEO_PROMPT_CODEX } from '@/modules/common/displayImagePrompt'; /** * Title instruction for Codex to call the hapi MCP tool. @@ -18,7 +19,8 @@ export const TITLE_INSTRUCTION = trimIdent(` If that exact tool name is unavailable, call an equivalent alias such as hapi__change_title, mcp__hapi__change_title, or hapi_change_title. Do not rename the chat for routine progress, substeps, implementation details, or a slightly better wording. Rename only when the user's primary objective changes substantially and the existing title would be misleading. - When you create or find a local image file that the user should see, call functions.hapi__display_image with the image path. If that exact tool name is unavailable, use an equivalent alias such as hapi__display_image, mcp__hapi__display_image, or hapi_display_image. + ${DISPLAY_IMAGE_PROMPT_CODEX} + ${DISPLAY_VIDEO_PROMPT_CODEX} `); /** diff --git a/cli/src/commands/doctor.ts b/cli/src/commands/doctor.ts index 39067ae9c3..17fab1748e 100644 --- a/cli/src/commands/doctor.ts +++ b/cli/src/commands/doctor.ts @@ -1,5 +1,6 @@ import { killRunawayHappyProcesses } from '@/runner/doctor' import { runDoctorCommand } from '@/ui/doctor' +import { runDoctorInlineMedia } from '@/ui/doctorInlineMedia' import type { CommandDefinition } from './types' export const doctorCommand: CommandDefinition = { @@ -14,6 +15,10 @@ export const doctorCommand: CommandDefinition = { } process.exit(0) } + if (commandArgs[0] === 'inline-media') { + const code = await runDoctorInlineMedia() + process.exit(code) + } await runDoctorCommand() } } diff --git a/cli/src/cursor/cursorAcpRemoteLauncher.test.ts b/cli/src/cursor/cursorAcpRemoteLauncher.test.ts index 761e86c387..25ab03d2b2 100644 --- a/cli/src/cursor/cursorAcpRemoteLauncher.test.ts +++ b/cli/src/cursor/cursorAcpRemoteLauncher.test.ts @@ -119,8 +119,14 @@ vi.mock('@/agent/permissionAdapter', () => ({ vi.mock('@/codex/utils/buildHapiMcpBridge', () => ({ buildHapiMcpBridge: async () => ({ server: { stop: () => {} }, - mcpServers: {} - }) + mcpServers: { + hapi: { command: 'hapi', args: ['mcp', '--url', 'http://127.0.0.1:1/'] }, + }, + }), +})); + +vi.mock('./utils/cursorMcpOverlay', () => ({ + installCursorMcpOverlay: () => ({ cleanup: () => {} }), })); vi.mock('@/ui/ink/OpencodeDisplay', () => ({ diff --git a/cli/src/cursor/cursorAcpRemoteLauncher.ts b/cli/src/cursor/cursorAcpRemoteLauncher.ts index 5e3eaf75b5..b56df37ad8 100644 --- a/cli/src/cursor/cursorAcpRemoteLauncher.ts +++ b/cli/src/cursor/cursorAcpRemoteLauncher.ts @@ -20,6 +20,7 @@ import { applyCursorAcpMode, applyCursorAcpModel, wireIdForCursorSessionState } import { buildCursorModelsSeedPayload, seedCursorModelsCache } from '@/modules/common/cursorModels'; import { readSharedCursorModelsCache } from '@/modules/common/cursorModelsSharedCache'; import type { AcpSdkBackend } from '@/agent/backends/acp'; +import { installCursorMcpOverlay, type CursorMcpOverlayHandle } from './utils/cursorMcpOverlay'; class CursorAcpRemoteLauncher extends RemoteLauncherBase { private readonly session: CursorSession; @@ -33,6 +34,7 @@ class CursorAcpRemoteLauncher extends RemoteLauncherBase { private defaultBackendModel: string | null = null; private unregisterModelApplyHandler: (() => void) | null = null; private modelApplySeq = 0; + private cursorMcpOverlay: CursorMcpOverlayHandle | null = null; constructor(session: CursorSession) { super(process.env.DEBUG ? session.logPath : undefined); @@ -57,6 +59,14 @@ class CursorAcpRemoteLauncher extends RemoteLauncherBase { const { server: happyServer, mcpServers } = await buildHapiMcpBridge(session.client); this.happyServer = happyServer; + const hapiBridge = mcpServers.hapi; + if (hapiBridge) { + this.cursorMcpOverlay = installCursorMcpOverlay(session.path, { + command: hapiBridge.command, + args: hapiBridge.args, + }); + } + const backend = createCursorAcpBackend({ cwd: session.path, model: session.model }); this.backend = backend; @@ -101,7 +111,8 @@ class CursorAcpRemoteLauncher extends RemoteLauncherBase { ); const resumeSessionId = session.sessionId; - const mcpServerList = toAcpMcpServers(mcpServers); + // Cursor ACP ignores session/new|load mcpServers; native .cursor/mcp.json is wired above. + const mcpServerList: McpServerStdio[] = []; let acpSessionId: string; if (resumeSessionId && backend.supportsLoadSession()) { @@ -111,7 +122,7 @@ class CursorAcpRemoteLauncher extends RemoteLauncherBase { acpSessionId = await backend.loadSession({ sessionId: resumeSessionId, cwd: session.path, - mcpServers: mcpServerList + mcpServers: mcpServerList, }); } catch (error) { logger.warn('[cursor-acp] session/load failed', formatAcpLoadError(error)); @@ -213,7 +224,7 @@ class CursorAcpRemoteLauncher extends RemoteLauncherBase { const promptContent: PromptContent[] = [{ type: 'text', - text: batch.message + text: batch.message, }]; session.onThinkingChange(true); @@ -267,6 +278,11 @@ class CursorAcpRemoteLauncher extends RemoteLauncherBase { this.happyServer = null; } + if (this.cursorMcpOverlay) { + this.cursorMcpOverlay.cleanup(); + this.cursorMcpOverlay = null; + } + setCursorAcpModelsSnapshot(null); } @@ -296,6 +312,9 @@ class CursorAcpRemoteLauncher extends RemoteLauncherBase { case 'error': this.messageBuffer.addMessage(message.message, 'status'); break; + case 'generated_image': + this.messageBuffer.addMessage(`Generated image: ${message.fileName}`, 'assistant'); + break; case 'turn_complete': break; default: @@ -507,15 +526,6 @@ function syncCursorModelsFromAcp(backend: AcpSdkBackend, acpSessionId: string): seedCursorModelsCache(payload); } -function toAcpMcpServers(config: Record): McpServerStdio[] { - return Object.entries(config).map(([name, entry]) => ({ - name, - command: entry.command, - args: entry.args, - env: [] - })); -} - export async function cursorAcpRemoteLauncher(session: CursorSession): Promise<'switch' | 'exit'> { const launcher = new CursorAcpRemoteLauncher(session); return launcher.launch(); diff --git a/cli/src/cursor/utils/cursorAcpBackend.ts b/cli/src/cursor/utils/cursorAcpBackend.ts index 119ab57feb..6b9d0f6423 100644 --- a/cli/src/cursor/utils/cursorAcpBackend.ts +++ b/cli/src/cursor/utils/cursorAcpBackend.ts @@ -25,7 +25,8 @@ export function createCursorAcpBackend(opts: { cwd: string; model?: string | nul return new AcpSdkBackend({ command: 'agent', args, - env: filterEnv(process.env) + env: filterEnv(process.env), + flavor: 'cursor', }); } diff --git a/cli/src/cursor/utils/cursorMcpOverlay.test.ts b/cli/src/cursor/utils/cursorMcpOverlay.test.ts new file mode 100644 index 0000000000..a3c69766ae --- /dev/null +++ b/cli/src/cursor/utils/cursorMcpOverlay.test.ts @@ -0,0 +1,77 @@ +import { afterEach, describe, expect, it } from 'vitest'; +import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { randomUUID } from 'node:crypto'; +import { CURSOR_HAPI_MCP_SERVER_ID, installCursorMcpOverlay } from './cursorMcpOverlay'; + +describe('installCursorMcpOverlay', () => { + const roots: string[] = []; + + afterEach(() => { + for (const root of roots.splice(0)) { + rmSync(root, { recursive: true, force: true }); + } + }); + + function makeProjectDir(initialMcpJson?: string): string { + const root = join(tmpdir(), `hapi-cursor-mcp-${randomUUID()}`); + mkdirSync(root, { recursive: true }); + roots.push(root); + if (initialMcpJson !== undefined) { + mkdirSync(join(root, '.cursor'), { recursive: true }); + writeFileSync(join(root, '.cursor', 'mcp.json'), initialMcpJson, 'utf-8'); + } + return root; + } + + it('writes hapi bridge into .cursor/mcp.json and restores on cleanup', () => { + const cwd = makeProjectDir(JSON.stringify({ + mcpServers: { + other: { command: 'echo', args: ['x'] }, + }, + }, null, 2)); + + const mcpPath = join(cwd, '.cursor', 'mcp.json'); + const before = readFileSync(mcpPath, 'utf-8'); + + const handle = installCursorMcpOverlay(cwd, { + command: '/bin/hapi', + args: ['mcp', '--url', 'http://127.0.0.1:12345/'], + }); + + const merged = JSON.parse(readFileSync(mcpPath, 'utf-8')) as { + mcpServers: Record; + }; + expect(merged.mcpServers.other).toEqual({ command: 'echo', args: ['x'] }); + expect(merged.mcpServers[CURSOR_HAPI_MCP_SERVER_ID]).toEqual({ + command: '/bin/hapi', + args: ['mcp', '--url', 'http://127.0.0.1:12345/'], + }); + + handle.cleanup(); + expect(readFileSync(mcpPath, 'utf-8')).toBe(before); + }); + + it('creates .cursor/mcp.json when missing and removes hapi entry on cleanup', () => { + const cwd = makeProjectDir(); + expect(existsSync(join(cwd, '.cursor', 'mcp.json'))).toBe(false); + + const handle = installCursorMcpOverlay(cwd, { + command: 'hapi', + args: ['mcp', '--url', 'http://127.0.0.1:9999/'], + }); + + const mcpPath = join(cwd, '.cursor', 'mcp.json'); + expect(existsSync(mcpPath)).toBe(true); + + handle.cleanup(); + + if (existsSync(mcpPath)) { + const after = JSON.parse(readFileSync(mcpPath, 'utf-8')) as { + mcpServers?: Record; + }; + expect(after.mcpServers?.[CURSOR_HAPI_MCP_SERVER_ID]).toBeUndefined(); + } + }); +}); diff --git a/cli/src/cursor/utils/cursorMcpOverlay.ts b/cli/src/cursor/utils/cursorMcpOverlay.ts new file mode 100644 index 0000000000..81b57265ca --- /dev/null +++ b/cli/src/cursor/utils/cursorMcpOverlay.ts @@ -0,0 +1,115 @@ +/** + * Cursor ACP does not connect MCP servers passed on session/new (upstream limitation). + * The working path is project .cursor/mcp.json + `agent mcp enable `. + * See https://forum.cursor.com/t/acp-agent-silently-ignores-mcpservers-in-session-new/153623 + */ + +import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { logger } from '@/ui/logger'; + +export const CURSOR_HAPI_MCP_SERVER_ID = 'hapi'; + +type McpServerEntry = { + command: string; + args: string[]; + env?: Record; +}; + +type CursorMcpJson = { + mcpServers?: Record; +}; + +export type CursorMcpOverlayHandle = { + cleanup: () => void; +}; + +function parseMcpJson(raw: string): CursorMcpJson { + const parsed = JSON.parse(raw) as unknown; + if (parsed === null || typeof parsed !== 'object') { + return { mcpServers: {} }; + } + return parsed as CursorMcpJson; +} + +function readMcpJson(path: string): CursorMcpJson { + if (!existsSync(path)) { + return { mcpServers: {} }; + } + return parseMcpJson(readFileSync(path, 'utf-8')); +} + +function writeMcpJson(path: string, config: CursorMcpJson): void { + writeFileSync(path, `${JSON.stringify(config, null, 2)}\n`, 'utf-8'); +} + +/** + * Merge the per-session HAPI stdio bridge into `/.cursor/mcp.json` and approve it + * for Cursor's native MCP loader. Restores prior file contents on cleanup. + */ +export function installCursorMcpOverlay( + cwd: string, + bridge: { command: string; args: string[] } +): CursorMcpOverlayHandle { + const cursorDir = join(cwd, '.cursor'); + const mcpJsonPath = join(cursorDir, 'mcp.json'); + mkdirSync(cursorDir, { recursive: true }); + + const hadFile = existsSync(mcpJsonPath); + const previousContent = hadFile ? readFileSync(mcpJsonPath, 'utf-8') : null; + + const config = hadFile && previousContent + ? parseMcpJson(previousContent) + : { mcpServers: {} }; + config.mcpServers ??= {}; + + config.mcpServers[CURSOR_HAPI_MCP_SERVER_ID] = { + command: bridge.command, + args: [...bridge.args], + }; + + writeMcpJson(mcpJsonPath, config); + + const enable = spawnSync('agent', ['mcp', 'enable', CURSOR_HAPI_MCP_SERVER_ID], { + cwd, + encoding: 'utf-8', + timeout: 30_000, + }); + + if (enable.status !== 0) { + const detail = (enable.stderr || enable.stdout || '').trim(); + logger.warn( + `[cursor-acp] agent mcp enable ${CURSOR_HAPI_MCP_SERVER_ID} failed (status=${enable.status ?? 'null'}${detail ? `: ${detail}` : ''})` + ); + } else { + logger.debug(`[cursor-acp] enabled native MCP server ${CURSOR_HAPI_MCP_SERVER_ID} via .cursor/mcp.json`); + } + + return { + cleanup: () => { + try { + if (previousContent !== null) { + writeFileSync(mcpJsonPath, previousContent, 'utf-8'); + return; + } + + if (!existsSync(mcpJsonPath)) { + return; + } + + const current = readMcpJson(mcpJsonPath); + if (current.mcpServers) { + delete current.mcpServers[CURSOR_HAPI_MCP_SERVER_ID]; + } + if (!current.mcpServers || Object.keys(current.mcpServers).length === 0) { + rmSync(mcpJsonPath, { force: true }); + return; + } + writeMcpJson(mcpJsonPath, current); + } catch (error) { + logger.debug('[cursor-acp] cursor MCP overlay cleanup failed', error); + } + }, + }; +} diff --git a/cli/src/kimi/kimiRemoteLauncher.ts b/cli/src/kimi/kimiRemoteLauncher.ts index 21fb438f8a..cb9caf4cd8 100644 --- a/cli/src/kimi/kimiRemoteLauncher.ts +++ b/cli/src/kimi/kimiRemoteLauncher.ts @@ -158,7 +158,7 @@ class KimiRemoteLauncher extends RemoteLauncherBase { const promptContent: PromptContent[] = [{ type: 'text', - text: batch.message + text: batch.message, }]; session.onThinkingChange(true); @@ -236,6 +236,9 @@ class KimiRemoteLauncher extends RemoteLauncherBase { case 'error': this.messageBuffer.addMessage(message.message, 'status'); break; + case 'generated_image': + this.messageBuffer.addMessage(`Generated image: ${message.fileName}`, 'assistant'); + break; case 'turn_complete': this.messageBuffer.addMessage('Turn complete', 'status'); break; diff --git a/cli/src/kimi/utils/kimiBackend.ts b/cli/src/kimi/utils/kimiBackend.ts index e99563b031..6705f30a11 100644 --- a/cli/src/kimi/utils/kimiBackend.ts +++ b/cli/src/kimi/utils/kimiBackend.ts @@ -25,6 +25,7 @@ export function createKimiBackend(opts: { return new AcpSdkBackend({ command: 'kimi', args: ['acp'], - env + env, + flavor: 'kimi', }); } diff --git a/cli/src/modules/common/displayImagePrompt.ts b/cli/src/modules/common/displayImagePrompt.ts new file mode 100644 index 0000000000..a81ea4b65e --- /dev/null +++ b/cli/src/modules/common/displayImagePrompt.ts @@ -0,0 +1,37 @@ +import { trimIdent } from '@/utils/trimIdent'; + +/** + * Shared display_image MCP tool hints โ€” one export per tool naming convention. + * Inject into flavor system prompts and first-prompt bridge instructions. + */ +export const DISPLAY_IMAGE_PROMPT_CLAUDE = trimIdent(` + When you create or find a local image file that the user should see, call the tool "mcp__hapi__display_image" with the image path so HAPI can show it inline. +`); + +export const DISPLAY_IMAGE_PROMPT_CODEX = trimIdent(` + When you create or find a local image file that the user should see, call functions.hapi__display_image with the image path. If that exact tool name is unavailable, use an equivalent alias such as hapi__display_image, mcp__hapi__display_image, or hapi_display_image. +`); + +export const DISPLAY_IMAGE_PROMPT_HAPI_MCP = trimIdent(` + When you create or find a local image file that the user should see, call the tool "hapi_display_image" with the image path so HAPI can show it inline. If that exact tool name is unavailable, use an equivalent alias such as display_image or mcp__hapi__display_image. +`); + +export const DISPLAY_VIDEO_PROMPT_CLAUDE = trimIdent(` + When you create or find a local mp4 or webm recording the user should see, call the tool "mcp__hapi__display_video" with the file path so HAPI can show it inline. +`); + +export const DISPLAY_VIDEO_PROMPT_CODEX = trimIdent(` + When you create or find a local mp4 or webm file the user should see, call functions.hapi__display_video with the file path. If that exact tool name is unavailable, use an equivalent alias such as hapi__display_video, mcp__hapi__display_video, or hapi_display_video. +`); + +export const DISPLAY_VIDEO_PROMPT_HAPI_MCP = trimIdent(` + When you create or find a local mp4 or webm recording the user should see, call the tool "hapi_display_video" with the file path so HAPI can show it inline. If that exact tool name is unavailable, use an equivalent alias such as display_video or mcp__hapi__display_video. +`); + +export const DISPLAY_IMAGE_PROMPT_CURSOR = trimIdent(` + When you create or find a local image file that the user should see, call the tool "display_image" with the absolute filesystem path so HAPI can show it inline. +`); + +export const DISPLAY_VIDEO_PROMPT_CURSOR = trimIdent(` + When you create or find a local mp4 or webm recording the user should see, call the tool "display_video" with the absolute filesystem path so HAPI can show it inline. +`); diff --git a/cli/src/modules/common/generatedImages.test.ts b/cli/src/modules/common/generatedImages.test.ts index 471539e41e..427fb32af8 100644 --- a/cli/src/modules/common/generatedImages.test.ts +++ b/cli/src/modules/common/generatedImages.test.ts @@ -1,5 +1,8 @@ import { describe, expect, it } from 'vitest' -import { clearGeneratedImages, detectImageMimeType, getGeneratedImage, registerGeneratedImage } from './generatedImages' +import { mkdirSync, writeFileSync } from 'node:fs' +import { join } from 'node:path' +import { tmpdir } from 'node:os' +import { clearGeneratedImages, detectImageMimeType, detectVideoMimeType, getGeneratedImage, registerGeneratedImage, registerGeneratedImageFromAcpBlock, registerGeneratedImageFromPath } from './generatedImages' describe('generatedImages', () => { it('detects supported image MIME types from file bytes', () => { @@ -10,6 +13,12 @@ describe('generatedImages', () => { expect(detectImageMimeType(Buffer.from([0x00, 0x00, 0x00, 0x1c, 0x66, 0x74, 0x79, 0x70, 0x61, 0x76, 0x69, 0x66]))).toBe('image/avif') }) + it('detects supported video MIME types from file bytes', () => { + expect(detectVideoMimeType(Buffer.from([0x00, 0x00, 0x00, 0x18, 0x66, 0x74, 0x79, 0x70, 0x69, 0x73, 0x6f, 0x6d]))).toBe('video/mp4') + expect(detectVideoMimeType(Buffer.from([0x1a, 0x45, 0xdf, 0xa3]))).toBe('video/webm') + expect(detectVideoMimeType(Buffer.from([0x00, 0x00, 0x00, 0x1c, 0x66, 0x74, 0x79, 0x70, 0x61, 0x76, 0x69, 0x66]))).toBeNull() + }) + it('rejects non-image bytes even if the path has an image extension', () => { expect(detectImageMimeType(Buffer.from('not really a png'))).toBeNull() }) @@ -47,7 +56,7 @@ describe('generatedImages', () => { path: '/tmp/large.png', mimeType: 'image/png', bytes: new Uint8Array(25 * 1024 * 1024 + 1) - })).toThrow('Image is too large to display inline') + })).toThrow('File is too large to display inline') clearGeneratedImages() }) @@ -67,4 +76,41 @@ describe('generatedImages', () => { clearGeneratedImages() }) + it('registers images from ACP base64 image blocks after MIME sniffing', async () => { + const pngHeader = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x00]) + const image = await registerGeneratedImageFromAcpBlock({ + type: 'image', + mimeType: 'image/png', + data: pngHeader.toString('base64') + }) + + expect(image?.mimeType).toBe('image/png') + expect(getGeneratedImage(image!.id)?.content.subarray(0, 8)).toEqual(pngHeader.subarray(0, 8)) + clearGeneratedImages() + }) + + it('registers images from local file paths in ACP uri blocks', async () => { + const dir = join(tmpdir(), `hapi-acp-image-${Date.now()}`) + mkdirSync(dir, { recursive: true }) + const path = join(dir, 'inline.png') + const bytes = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00]) + writeFileSync(path, bytes) + + const image = await registerGeneratedImageFromPath({ path }) + expect(image?.mimeType).toBe('image/png') + clearGeneratedImages() + }) + + it('registers mp4 from local file paths after MIME sniffing', async () => { + const dir = join(tmpdir(), `hapi-inline-mp4-${Date.now()}`) + mkdirSync(dir, { recursive: true }) + const path = join(dir, 'inline.mp4') + const bytes = Buffer.from([0x00, 0x00, 0x00, 0x18, 0x66, 0x74, 0x79, 0x70, 0x69, 0x73, 0x6f, 0x6d]) + writeFileSync(path, bytes) + + const video = await registerGeneratedImageFromPath({ path }) + expect(video?.mimeType).toBe('video/mp4') + clearGeneratedImages() + }) + }) diff --git a/cli/src/modules/common/generatedImages.ts b/cli/src/modules/common/generatedImages.ts index fdf6cb9441..247f4608bd 100644 --- a/cli/src/modules/common/generatedImages.ts +++ b/cli/src/modules/common/generatedImages.ts @@ -1,4 +1,8 @@ -import { basename } from 'path' +import { basename } from 'node:path' +import { fileURLToPath } from 'node:url' +import { lstat, readFile } from 'node:fs/promises' +import { randomUUID } from 'node:crypto' +import { asString, isObject } from '@hapi/protocol' export type GeneratedImageMetadata = { id: string @@ -8,7 +12,7 @@ export type GeneratedImageMetadata = { createdAt: number } -const MAX_GENERATED_IMAGE_BYTES = 25 * 1024 * 1024 +export const MAX_GENERATED_IMAGE_BYTES = 25 * 1024 * 1024 const MAX_GENERATED_IMAGE_TOTAL_BYTES = 100 * 1024 * 1024 const MAX_GENERATED_IMAGE_COUNT = 100 @@ -55,6 +59,30 @@ export function detectImageMimeType(bytes: Uint8Array): string | null { return null } +export function detectVideoMimeType(bytes: Uint8Array): string | null { + if (bytes.length >= 12 && ascii(bytes, 4, 8) === 'ftyp') { + const brand = ascii(bytes, 8, 12) + if (brand === 'avif' || brand === 'avis') { + return null + } + return 'video/mp4' + } + + if (bytes.length >= 4 + && bytes[0] === 0x1a + && bytes[1] === 0x45 + && bytes[2] === 0xdf + && bytes[3] === 0xa3) { + return 'video/webm' + } + + return null +} + +export function isInlineMediaMimeType(mimeType: string): boolean { + return mimeType.startsWith('image/') || mimeType.startsWith('video/') +} + function ascii(bytes: Uint8Array, start: number, end: number): string { return String.fromCharCode(...bytes.subarray(start, end)) } @@ -62,7 +90,11 @@ function ascii(bytes: Uint8Array, start: number, end: number): string { export function registerGeneratedImage(args: { id: string; path: string; mimeType: string; bytes: Uint8Array; fileName?: string | null }): GeneratedImageMetadata { const content = Buffer.from(args.bytes) if (content.byteLength > MAX_GENERATED_IMAGE_BYTES) { - throw new Error('Image is too large to display inline') + throw new Error('File is too large to display inline') + } + + if (!isInlineMediaMimeType(args.mimeType)) { + throw new Error('Unsupported inline media MIME type') } const previous = generatedImages.get(args.id) @@ -105,3 +137,92 @@ export function clearGeneratedImages(): void { generatedImages.clear() generatedImageBytes = 0 } + +export async function registerGeneratedImageFromPath(args: { + id?: string + path: string + fileName?: string | null +}): Promise { + try { + const info = await lstat(args.path) + if (!info.isFile()) { + throw new Error('Path is not a regular file') + } + if (info.size > MAX_GENERATED_IMAGE_BYTES) { + throw new Error('Image is too large to display inline') + } + const bytes = await readFile(args.path) + const mimeType = detectImageMimeType(bytes) ?? detectVideoMimeType(bytes) + if (!mimeType) { + throw new Error('Unsupported inline media content') + } + return registerGeneratedImage({ + id: args.id ?? randomUUID(), + path: args.path, + fileName: args.fileName, + mimeType, + bytes + }) + } catch { + return null + } +} + +function parseAcpImageUri(uri: string): string | null { + if (uri.startsWith('file://')) { + try { + return fileURLToPath(uri) + } catch { + return null + } + } + if (/^https?:\/\//i.test(uri)) { + return null + } + return uri +} + +export async function registerGeneratedImageFromAcpBlock(block: unknown): Promise { + if (!isObject(block) || block.type !== 'image') { + return null + } + + const data = asString(block.data) + const declaredMimeType = asString(block.mimeType ?? block.mime_type) + const uri = asString(block.uri ?? block.url) + + if (data) { + const bytes = Buffer.from(data, 'base64') + if (bytes.byteLength > MAX_GENERATED_IMAGE_BYTES) { + return null + } + const sniffedMimeType = detectImageMimeType(bytes) + if (!sniffedMimeType) { + return null + } + if (declaredMimeType && declaredMimeType !== sniffedMimeType) { + return null + } + const path = uri ? parseAcpImageUri(uri) ?? uri : `${randomUUID()}.bin` + return registerGeneratedImage({ + id: randomUUID(), + path, + fileName: basename(path), + mimeType: sniffedMimeType, + bytes + }) + } + + if (uri) { + const path = parseAcpImageUri(uri) + if (!path) { + return null + } + return registerGeneratedImageFromPath({ + path, + fileName: basename(path) + }) + } + + return null +} diff --git a/cli/src/modules/common/hapiMcpBridgePrompt.ts b/cli/src/modules/common/hapiMcpBridgePrompt.ts new file mode 100644 index 0000000000..0c66637125 --- /dev/null +++ b/cli/src/modules/common/hapiMcpBridgePrompt.ts @@ -0,0 +1,36 @@ +import { trimIdent } from '@/utils/trimIdent'; +import { + DISPLAY_IMAGE_PROMPT_HAPI_MCP, + DISPLAY_VIDEO_PROMPT_HAPI_MCP, + DISPLAY_IMAGE_PROMPT_CURSOR, + DISPLAY_VIDEO_PROMPT_CURSOR, +} from './displayImagePrompt'; + +/** Shell fallback for `hapi doctor inline-media` only โ€” not injected into agent prompts. */ +export const INLINE_MEDIA_SHELL_FALLBACK = trimIdent(` + If display_image / display_video tools are not in your tool list, inline media via shell from the HAPI repo (needs bun + @modelcontextprotocol/sdk): + cd && bun scripts/tooling/hapi-display-image.mjs "title" + Use the HAPI session uuid prefix from the web URL /sessions/ (first 8 chars), not cursorSessionId or other agent-native ids. + Run hapi doctor inline-media to list active bridges and copy the exact command. +`); + +/** + * Title + display_image / display_video instructions for OpenCode local config + * (written to hapi-instructions.md โ€” not prepended to user turns). + */ +export const HAPI_MCP_TITLE_INSTRUCTION = trimIdent(` + Use the title tool sparingly. For a new chat, call the tool "hapi_change_title" once after the user's initial request is clear, and set a concise task title. Do not rename the chat for routine progress, substeps, implementation details, or a slightly better wording. Rename only when the user's primary objective changes substantially and the existing title would be misleading. +`); + +export const HAPI_MCP_BRIDGE_PROMPT = trimIdent(` + ${HAPI_MCP_TITLE_INSTRUCTION} + ${DISPLAY_IMAGE_PROMPT_HAPI_MCP} + ${DISPLAY_VIDEO_PROMPT_HAPI_MCP} +`); + +/** Cursor native MCP exposes bare tool names (display_image, not hapi_display_image). */ +export const HAPI_MCP_BRIDGE_PROMPT_CURSOR = trimIdent(` + Use change_title sparingly โ€” once when the user's primary objective is clear. + ${DISPLAY_IMAGE_PROMPT_CURSOR} + ${DISPLAY_VIDEO_PROMPT_CURSOR} +`); diff --git a/cli/src/modules/common/inlineMediaSource.test.ts b/cli/src/modules/common/inlineMediaSource.test.ts new file mode 100644 index 0000000000..abe470e5f5 --- /dev/null +++ b/cli/src/modules/common/inlineMediaSource.test.ts @@ -0,0 +1,22 @@ +import { describe, expect, it } from 'vitest'; +import { inlineMediaSourceFromWire } from './inlineMediaSource'; + +describe('inlineMediaSourceFromWire', () => { + it('parses ingress and snake_case tool fields', () => { + expect(inlineMediaSourceFromWire({ + ingress: 'mcp', + flavor: 'cursor', + tool_call_id: 'call-1', + tool_name: 'display_video', + })).toEqual({ + ingress: 'mcp', + flavor: 'cursor', + toolCallId: 'call-1', + toolName: 'display_video', + }); + }); + + it('accepts legacy path alias for ingress', () => { + expect(inlineMediaSourceFromWire({ path: 'acp' })).toEqual({ ingress: 'acp' }); + }); +}); diff --git a/cli/src/modules/common/inlineMediaSource.ts b/cli/src/modules/common/inlineMediaSource.ts new file mode 100644 index 0000000000..d3a96026f5 --- /dev/null +++ b/cli/src/modules/common/inlineMediaSource.ts @@ -0,0 +1,28 @@ +/** How inline image/video entered the session (v1 provenance seed for #956 / artifact follow-up). */ +export type InlineMediaIngress = 'mcp' | 'acp' | 'tool_result'; + +export type InlineMediaSource = { + ingress: InlineMediaIngress; + flavor?: string; + toolCallId?: string; + toolName?: string; +}; + +export function inlineMediaSourceFromWire(value: unknown): InlineMediaSource | undefined { + if (!value || typeof value !== 'object' || Array.isArray(value)) return undefined; + const record = value as Record; + const ingress = record.ingress ?? record.path; + if (ingress !== 'mcp' && ingress !== 'acp' && ingress !== 'tool_result') return undefined; + const flavor = typeof record.flavor === 'string' ? record.flavor : undefined; + const toolCallId = typeof record.toolCallId === 'string' + ? record.toolCallId + : typeof record.tool_call_id === 'string' + ? record.tool_call_id + : undefined; + const toolName = typeof record.toolName === 'string' + ? record.toolName + : typeof record.tool_name === 'string' + ? record.tool_name + : undefined; + return { ingress, flavor, toolCallId, toolName }; +} diff --git a/cli/src/modules/common/permission/BasePermissionHandler.ts b/cli/src/modules/common/permission/BasePermissionHandler.ts index f70908f5eb..e03908405e 100644 --- a/cli/src/modules/common/permission/BasePermissionHandler.ts +++ b/cli/src/modules/common/permission/BasePermissionHandler.ts @@ -19,14 +19,18 @@ export type AutoApprovalRuleSet = { const AUTO_APPROVE_TOOL_NAME_HINTS = [ 'change_title', + 'display_image', + 'display_video', 'happy__change_title', 'hapi_change_title', // OpenCode MCP tool pattern + 'hapi_display_image', + 'hapi_display_video', 'geminireasoning', 'codexreasoning', 'think', 'save_memory' ]; -const AUTO_APPROVE_TOOL_ID_HINTS = ['change_title', 'save_memory']; +const AUTO_APPROVE_TOOL_ID_HINTS = ['change_title', 'display_image', 'display_video', 'save_memory']; const AUTO_APPROVE_WRITE_TOOL_HINTS = ['write', 'edit', 'create', 'delete', 'patch', 'fs-edit']; export function resolveToolAutoApprovalDecision( diff --git a/cli/src/opencode/opencodeRemoteLauncher.ts b/cli/src/opencode/opencodeRemoteLauncher.ts index b911fbca07..d3d2c796d7 100644 --- a/cli/src/opencode/opencodeRemoteLauncher.ts +++ b/cli/src/opencode/opencodeRemoteLauncher.ts @@ -10,7 +10,7 @@ import type { OpencodeMode, PermissionMode } from './types'; import { RPC_METHODS } from '@hapi/protocol/rpcMethods'; import { createOpencodeBackend } from './utils/opencodeBackend'; import { OpencodePermissionHandler } from './utils/permissionHandler'; -import { PLAN_MODE_INSTRUCTION, TITLE_INSTRUCTION } from './utils/systemPrompt'; +import { PLAN_MODE_INSTRUCTION } from './utils/systemPrompt'; import { resolveThoughtLevelEffort } from './thoughtLevelEffort'; type OpencodeRemoteLauncherOptions = { @@ -24,7 +24,6 @@ class OpencodeRemoteLauncher extends RemoteLauncherBase { private happyServer: { stop: () => void } | null = null; private abortController = new AbortController(); private displayPermissionMode: PermissionMode | null = null; - private instructionsSent = false; private currentBackendModel: string | null = null; private defaultBackendModel: string | null = null; private currentBackendEffort: string | null = null; @@ -266,19 +265,14 @@ class OpencodeRemoteLauncher extends RemoteLauncherBase { this.applyDisplayMode(batch.mode.permissionMode); messageBuffer.addMessage(batch.message, 'user'); - // Inject title instructions on first prompt let messageText = batch.message; if (batch.mode.permissionMode === 'plan') { messageText = `${PLAN_MODE_INSTRUCTION}\n\n${messageText}`; } - if (!this.instructionsSent) { - messageText = `${TITLE_INSTRUCTION}\n\n${messageText}`; - this.instructionsSent = true; - } const promptContent: PromptContent[] = [{ type: 'text', - text: messageText + text: messageText, }]; session.onThinkingChange(true); @@ -360,6 +354,9 @@ class OpencodeRemoteLauncher extends RemoteLauncherBase { case 'error': this.messageBuffer.addMessage(message.message, 'status'); break; + case 'generated_image': + this.messageBuffer.addMessage(`Generated image: ${message.fileName}`, 'assistant'); + break; case 'turn_complete': this.messageBuffer.addMessage('Turn complete', 'status'); break; diff --git a/cli/src/opencode/utils/opencodeBackend.ts b/cli/src/opencode/utils/opencodeBackend.ts index b49a345596..988035a738 100644 --- a/cli/src/opencode/utils/opencodeBackend.ts +++ b/cli/src/opencode/utils/opencodeBackend.ts @@ -21,6 +21,7 @@ export function createOpencodeBackend(opts: { return new AcpSdkBackend({ command: 'opencode', args, - env: filterEnv(env) + env: filterEnv(env), + flavor: 'opencode', }); } diff --git a/cli/src/opencode/utils/systemPrompt.ts b/cli/src/opencode/utils/systemPrompt.ts index e968759626..feb6d02d29 100644 --- a/cli/src/opencode/utils/systemPrompt.ts +++ b/cli/src/opencode/utils/systemPrompt.ts @@ -1,19 +1,17 @@ /** - * OpenCode-specific system prompt for change_title tool. + * OpenCode-specific system prompt for hapi MCP tools (change_title, display_image, display_video). * * OpenCode exposes MCP tools with the naming pattern: _ - * The hapi MCP server exposes `change_title`, so it's called as `hapi_change_title`. + * The hapi MCP server exposes `change_title`, `display_image`, and `display_video`. */ import { trimIdent } from '@/utils/trimIdent'; +import { HAPI_MCP_BRIDGE_PROMPT } from '@/modules/common/hapiMcpBridgePrompt'; /** - * Title instruction for OpenCode to call the hapi MCP tool. + * Title and display_image instructions for OpenCode to call the hapi MCP tools. */ -export const TITLE_INSTRUCTION = trimIdent(` - Use the title tool sparingly. For a new chat, call the tool "hapi_change_title" once after the user's initial request is clear, and set a concise task title. Do not rename the chat for routine progress, substeps, implementation details, or a slightly better wording. Rename only when the user's primary objective changes substantially and the existing title would be misleading. - When you create or find a local image file that the user should see, call the tool "hapi_display_image" with the image path so HAPI can show it inline. -`); +export const TITLE_INSTRUCTION = HAPI_MCP_BRIDGE_PROMPT; /** * The system prompt to inject for OpenCode sessions. diff --git a/cli/src/ui/doctorInlineMedia.test.ts b/cli/src/ui/doctorInlineMedia.test.ts new file mode 100644 index 0000000000..35c1644349 --- /dev/null +++ b/cli/src/ui/doctorInlineMedia.test.ts @@ -0,0 +1,11 @@ +import { describe, expect, it } from 'vitest' +import { formatInlineMediaCommand, inlineMediaHelperScriptPath } from './doctorInlineMedia' + +describe('doctorInlineMedia', () => { + it('formatInlineMediaCommand uses repo scripts path', () => { + const script = inlineMediaHelperScriptPath() + expect(formatInlineMediaCommand(script, '341fe421')).toContain( + 'bun scripts/tooling/hapi-display-image.mjs 341fe421' + ) + }) +}) diff --git a/cli/src/ui/doctorInlineMedia.ts b/cli/src/ui/doctorInlineMedia.ts new file mode 100644 index 0000000000..62c2228a16 --- /dev/null +++ b/cli/src/ui/doctorInlineMedia.ts @@ -0,0 +1,246 @@ +/** + * Inline media bridge diagnostics (display_image / display_video + helper script). + */ + +import chalk from 'chalk' +import { existsSync } from 'node:fs' +import { join, resolve } from 'node:path' +import { configuration } from '@/configuration' +import { readSettings } from '@/persistence' +import { projectPath } from '@/projectPath' + +export type InlineMediaDoctorCheck = { + ok: boolean + label: string + detail: string +} + +export type InlineMediaSessionBridge = { + id: string + prefix: string + flavor: string | null + hapiMcpUrl: string | null + listShowsMcpUrl: boolean + path: string | null + name: string | null +} + +function repoRootFromCli(): string { + return resolve(projectPath(), '..') +} + +export function inlineMediaHelperScriptPath(): string { + return join(repoRootFromCli(), 'scripts/tooling/hapi-display-image.mjs') +} + +function mcpSdkResolvable(): boolean { + const candidates = [ + join(projectPath(), 'node_modules/@modelcontextprotocol/sdk/package.json'), + join(repoRootFromCli(), 'node_modules/@modelcontextprotocol/sdk/package.json'), + ] + return candidates.some((p) => existsSync(p)) +} + +async function hubJwt(): Promise { + const settings = await readSettings() + const token = process.env.CLI_API_TOKEN ?? settings.cliApiToken + if (!token) { + return null + } + const res = await fetch(`${configuration.apiUrl}/api/auth`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ accessToken: token }), + }) + if (!res.ok) { + return null + } + const body = (await res.json()) as { token?: string } + return body.token ?? null +} + +function asRecord(value: unknown): Record | null { + return value !== null && typeof value === 'object' ? value as Record : null +} + +function sessionDisplayName(metadata: Record | null): string | null { + if (!metadata) return null + const name = metadata.name + return typeof name === 'string' ? name : null +} + +export function formatInlineMediaCommand( + scriptPath: string, + sessionPrefix: string, + samplePath = '/absolute/path/to/image.png' +): string { + const scriptDir = resolve(scriptPath, '..', '..', '..') + return `cd ${scriptDir} && bun scripts/tooling/hapi-display-image.mjs ${sessionPrefix} ${samplePath} "title"` +} + +export async function collectInlineMediaSessionBridges(jwt: string): Promise { + const listRes = await fetch(`${configuration.apiUrl}/api/sessions?limit=200`, { + headers: { Authorization: `Bearer ${jwt}` }, + }) + if (!listRes.ok) { + throw new Error(`sessions list failed: ${listRes.status}`) + } + const listBody = (await listRes.json()) as { sessions?: unknown[] } + const sessions = Array.isArray(listBody.sessions) ? listBody.sessions : [] + const active = sessions.filter((s) => asRecord(s)?.active === true) + + const bridges: InlineMediaSessionBridge[] = [] + for (const row of active) { + const summary = asRecord(row) + if (!summary || typeof summary.id !== 'string') continue + const listMeta = asRecord(summary.metadata) + const listMcp = listMeta && typeof listMeta.hapiMcpUrl === 'string' ? listMeta.hapiMcpUrl : null + + const detailRes = await fetch( + `${configuration.apiUrl}/api/sessions/${encodeURIComponent(summary.id)}`, + { headers: { Authorization: `Bearer ${jwt}` } } + ) + if (!detailRes.ok) continue + const detailBody = (await detailRes.json()) as { session?: unknown } + const detailRow = asRecord(detailBody.session) ?? asRecord(detailBody) + const detailMeta = asRecord(detailRow?.metadata) + const detailMcp = detailMeta && typeof detailMeta.hapiMcpUrl === 'string' ? detailMeta.hapiMcpUrl : null + const flavor = detailMeta && typeof detailMeta.flavor === 'string' ? detailMeta.flavor : null + const path = detailMeta && typeof detailMeta.path === 'string' ? detailMeta.path : null + + bridges.push({ + id: summary.id, + prefix: summary.id.slice(0, 8), + flavor, + hapiMcpUrl: detailMcp, + listShowsMcpUrl: listMcp !== null, + path, + name: sessionDisplayName(detailMeta), + }) + } + return bridges +} + +export async function runDoctorInlineMedia(): Promise { + console.log(chalk.bold.cyan('\n๐Ÿ–ผ๏ธ hapi inline media doctor\n')) + + const checks: InlineMediaDoctorCheck[] = [] + const scriptPath = inlineMediaHelperScriptPath() + const scriptExists = existsSync(scriptPath) + checks.push({ + ok: scriptExists, + label: 'Helper script', + detail: scriptExists ? scriptPath : `missing: ${scriptPath}`, + }) + + const sdkOk = mcpSdkResolvable() + checks.push({ + ok: sdkOk, + label: '@modelcontextprotocol/sdk', + detail: sdkOk ? 'resolvable from cli or repo root' : 'not found โ€” run bun install from repo root', + }) + + const envSessionId = process.env.HAPI_SESSION_ID + if (envSessionId) { + checks.push({ + ok: true, + label: 'HAPI_SESSION_ID', + detail: envSessionId, + }) + } + + let jwt: string | null = null + try { + jwt = await hubJwt() + } catch { + jwt = null + } + checks.push({ + ok: jwt !== null, + label: 'Hub auth', + detail: jwt ? configuration.apiUrl : 'CLI_API_TOKEN missing or auth failed', + }) + + for (const check of checks) { + const mark = check.ok ? chalk.green('โœ“') : chalk.red('โœ—') + console.log(`${mark} ${check.label}: ${chalk.gray(check.detail)}`) + } + + if (!jwt) { + console.log(chalk.red('\nCannot probe sessions without hub auth.\n')) + return 1 + } + + let bridges: InlineMediaSessionBridge[] = [] + try { + bridges = await collectInlineMediaSessionBridges(jwt) + } catch (error) { + const msg = error instanceof Error ? error.message : String(error) + console.log(chalk.red(`\nโœ— Session probe failed: ${msg}\n`)) + return 1 + } + + const withBridge = bridges.filter((b) => b.hapiMcpUrl) + const listOmitsMcp = bridges.some((b) => b.hapiMcpUrl && !b.listShowsMcpUrl) + + console.log(chalk.bold('\nActive sessions')) + if (bridges.length === 0) { + console.log(chalk.yellow(' No active sessions on hub.')) + } else { + for (const b of bridges) { + const bridgeMark = b.hapiMcpUrl ? chalk.green('bridge') : chalk.yellow('no bridge') + const title = b.name ?? b.path ?? b.id + console.log( + ` ${chalk.blue(b.prefix)} ${bridgeMark} ${chalk.gray(title)}` + + (b.flavor ? chalk.gray(` (${b.flavor})`) : '') + ) + if (b.hapiMcpUrl) { + console.log(chalk.gray(` mcp: ${b.hapiMcpUrl}`)) + console.log(chalk.gray(` ${formatInlineMediaCommand(scriptPath, b.prefix)}`)) + } + } + } + + if (listOmitsMcp) { + console.log(chalk.yellow( + '\nโš  Some active sessions have hapiMcpUrl on detail GET but not on list โ€” upgrade hub or use per-session GET.' + )) + } + + const cursorSessions = withBridge.filter((b) => b.flavor === 'cursor') + if (cursorSessions.length > 0) { + console.log(chalk.bold('\nCursor ACP')) + console.log(chalk.gray(' Cursor ignores session/new mcpServers. Remote sessions use .cursor/mcp.json + `agent mcp enable hapi`.')) + console.log(chalk.gray(' Tool names are bare: display_image, display_video, change_title (not hapi_display_image).')) + console.log(chalk.gray(' Verify on the session machine: agent mcp list-tools hapi')) + } + + console.log(chalk.bold('\nAgent inline path')) + console.log(chalk.gray(' 1. MCP tool display_image / display_video in the running session (ACP flavors via hapi bridge)')) + console.log(chalk.gray(' 2. Shell fallback (HAPI session id prefix, not cursorSessionId):')) + if (withBridge.length > 0) { + console.log(chalk.green(` ${formatInlineMediaCommand(scriptPath, withBridge[0].prefix)}`)) + } else if (envSessionId) { + console.log(chalk.green(` ${formatInlineMediaCommand(scriptPath, envSessionId.slice(0, 8))}`)) + } else { + console.log(chalk.gray(` ${formatInlineMediaCommand(scriptPath, '')}`)) + } + + const ok = + scriptExists + && sdkOk + && jwt !== null + && (withBridge.length > 0 || Boolean(envSessionId)) + + if (ok) { + console.log(chalk.green('\nโœ“ Inline media path available\n')) + return 0 + } + + if (withBridge.length === 0 && !envSessionId) { + console.log(chalk.yellow('\nโš  No active session with hapiMcpUrl โ€” start or resume a remote session first.\n')) + } else { + console.log(chalk.red('\nโœ— Inline media checks failed โ€” fix items marked โœ— above.\n')) + } + return withBridge.length > 0 ? 0 : 1 +} diff --git a/package.json b/package.json index c33c2acc89..9bdde30e25 100644 --- a/package.json +++ b/package.json @@ -29,9 +29,9 @@ "release-all": "cd cli && bun run release-all" }, "devDependencies": { - "@playwright/test": "^1.60.0", + "@playwright/test": "^1.61.0", "concurrently": "^9.2.1", - "playwright": "1.60.0", + "playwright": "1.61.0", "react-devtools-core": "^7.0.1", "vite-plugin-pwa": "^1.2.0" } diff --git a/playwright.config.ts b/playwright.config.ts index 779efd169c..b3b0a41eb1 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -1,21 +1,32 @@ import { defineConfig, devices } from '@playwright/test' +import { + annotatedVideoUseOption, + shouldRecordAnnotatedVideo, +} from './scripts/dev/playwright-annotated-video.mjs' const PORT = 5179 const BASE_URL = `http://localhost:${PORT}` +const peerWebUrl = process.env.HAPI_PEER_WEB_URL?.replace(/\/$/, '') +const usePeerStack = Boolean(peerWebUrl) +const baseURL = peerWebUrl ?? BASE_URL + export default defineConfig({ testDir: './e2e', - timeout: 30_000, - expect: { timeout: 5_000 }, + timeout: usePeerStack ? 60_000 : 30_000, + expect: { timeout: usePeerStack ? 10_000 : 5_000 }, fullyParallel: false, forbidOnly: !!process.env.CI, retries: process.env.CI ? 1 : 0, workers: 1, reporter: process.env.CI ? [['list'], ['html', { open: 'never' }]] : 'list', use: { - baseURL: BASE_URL, + baseURL, trace: 'retain-on-failure', screenshot: 'only-on-failure', + video: shouldRecordAnnotatedVideo() + ? annotatedVideoUseOption('on', usePeerStack ? { width: 1440, height: 900 } : undefined) + : 'off', }, projects: [ { @@ -23,26 +34,21 @@ export default defineConfig({ use: { ...devices['Desktop Chrome'], launchOptions: { - // The CI runner and most sandboxed dev environments - // run as root or under restricted user namespaces; - // without --no-sandbox chromium silently exits 0 a - // few seconds after launch and the page handshake - // times out. Keep the flag scoped to launchOptions - // so this is the only place a future maintainer has - // to revisit if they harden the runner. - args: ['--no-sandbox'], + args: usePeerStack + ? ['--no-sandbox', '--disable-dev-shm-usage'] + : ['--no-sandbox'], }, }, }, ], - webServer: { - // The fixture page mounts ScratchlistPanel in isolation; no hub - // is required, which is why this dev server doesn't proxy /api. - command: `bun run --cwd web dev -- --port ${PORT} --strictPort`, - url: `${BASE_URL}/e2e-fixtures/scratchlist-fixture.html`, - timeout: 60_000, - reuseExistingServer: !process.env.CI, - stdout: 'ignore', - stderr: 'pipe', - }, + webServer: usePeerStack + ? undefined + : { + command: `bun run --cwd web dev -- --port ${PORT} --strictPort`, + url: `${BASE_URL}/e2e-fixtures/scratchlist-fixture.html`, + timeout: 60_000, + reuseExistingServer: !process.env.CI, + stdout: 'ignore', + stderr: 'pipe', + }, }) diff --git a/scripts/dev/playwright-annotated-video.mjs b/scripts/dev/playwright-annotated-video.mjs new file mode 100644 index 0000000000..74ae3af3e3 --- /dev/null +++ b/scripts/dev/playwright-annotated-video.mjs @@ -0,0 +1,70 @@ +/** + * Playwright screencast helpers โ€” click highlights + animated pointer on recorded video. + * + * Requires Playwright >= 1.59 (screencast.showActions); cursor animation needs >= 1.61. + * + * @playwright/test fixtures: + * import { annotatedVideoUseOption } from './scripts/dev/playwright-annotated-video.mjs' + * use: { video: process.env.PLAYWRIGHT_RECORD_VIDEO === '1' ? annotatedVideoUseOption('on') : 'off' } + * + * Programmatic (handoff .mjs scripts): + * import { startAnnotatedScreencast, stopAnnotatedScreencast } from './playwright-annotated-video.mjs' + * await startAnnotatedScreencast(page, { path: 'localdocs/playwright-runs/demo.webm' }) + * // ... interactions ... + * await stopAnnotatedScreencast(page) + */ + +/** Default overlays: element outline, action title, pointer glide between clicks. */ +export const ANNOTATED_SHOW_ACTIONS = { + position: 'top-right', + cursor: 'pointer', + duration: 800, + fontSize: 22, +} + +/** + * `use.video` value for @playwright/test when recording with action annotations. + * @param {import('@playwright/test').VideoMode} mode + * @param {import('@playwright/test').ViewportSize | undefined} size + */ +export function annotatedVideoUseOption(mode = 'on', size) { + const option = { + mode, + show: { + actions: { + position: ANNOTATED_SHOW_ACTIONS.position, + duration: ANNOTATED_SHOW_ACTIONS.duration, + fontSize: ANNOTATED_SHOW_ACTIONS.fontSize, + }, + }, + } + if (size) option.size = size + return option +} + +export function shouldRecordAnnotatedVideo() { + return process.env.HAPI_PEER_RECORD_VIDEO === '1' || process.env.PLAYWRIGHT_RECORD_VIDEO === '1' +} + +/** + * Start annotated screencast on a page (replaces raw `recordVideo` on browser context). + * @param {import('playwright').Page} page + * @param {{ path: string, showActions?: typeof ANNOTATED_SHOW_ACTIONS, size?: { width: number, height: number } }} options + */ +export async function startAnnotatedScreencast(page, options) { + const { path, showActions = ANNOTATED_SHOW_ACTIONS, size } = options + await page.screencast.start({ path, size }) + await page.screencast.showActions(showActions) +} + +/** Stop screencast and finalize the file written by {@link startAnnotatedScreencast}. */ +export async function stopAnnotatedScreencast(page) { + await page.screencast.stop() +} + +/** Resolve webm/mp4 paths under a handoff output directory. */ +export function annotatedVideoPaths(dir, basename) { + const webm = `${dir.replace(/\/$/, '')}/${basename}.webm` + const mp4 = `${dir.replace(/\/$/, '')}/${basename}.mp4` + return { webm, mp4 } +} diff --git a/scripts/tooling/hapi-display-image.mjs b/scripts/tooling/hapi-display-image.mjs index c2670585d2..16d1a75d80 100644 --- a/scripts/tooling/hapi-display-image.mjs +++ b/scripts/tooling/hapi-display-image.mjs @@ -1,30 +1,72 @@ #!/usr/bin/env bun /** - * Post a local image inline to a HAPI session via the session CLI's display_image MCP tool. + * Post a local image or video inline to a HAPI session via display_image / display_video MCP. * * Uses session.metadata.hapiMcpUrl (published at MCP server start) so we hit the MCP * endpoint, not the session hook server on another loopback port in the same process. * * Usage: - * bun scripts/tooling/hapi-display-image.mjs [title] + * bun scripts/tooling/hapi-display-image.mjs [title] + * + * Picks display_video for mp4/webm (ftyp / webm magic), else display_image. */ -import { readFileSync, lstatSync } from 'node:fs' +import { readFileSync, lstatSync, existsSync } from 'node:fs' import { Client } from '@modelcontextprotocol/sdk/client/index.js' import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js' const HAPI_HOST = process.env.HAPI_HOST ?? 'http://localhost:3006' const SETTINGS = process.env.HAPI_SETTINGS ?? `${process.env.HOME}/.hapi/settings.json` -const sessionArg = process.argv[2] -const imagePath = process.argv[3] -const title = process.argv[4] +const envSessionPrefix = process.env.HAPI_SESSION_ID ? process.env.HAPI_SESSION_ID.slice(0, 8) : undefined +let sessionArg = process.argv[2] +let imagePath = process.argv[3] +let title = process.argv[4] + +// HAPI_SESSION_ID + path-only: hapi-display-image.mjs [title] +if (envSessionPrefix && sessionArg && existsSync(sessionArg) && lstatSync(sessionArg).isFile()) { + imagePath = sessionArg + title = process.argv[3] + sessionArg = envSessionPrefix +} else if (!sessionArg && envSessionPrefix) { + sessionArg = envSessionPrefix +} if (!sessionArg || !imagePath) { - console.error('usage: hapi-display-image.mjs [title]') + console.error('usage: hapi-display-image.mjs [title]') + console.error(' or: HAPI_SESSION_ID= hapi-display-image.mjs [title]') process.exit(2) } +function sessionMatchesPrefix(session, prefix) { + if (typeof session.id === 'string' && session.id.startsWith(prefix)) { + return true + } + const meta = session.metadata ?? {} + const agentIds = [ + meta.agentSessionId, + meta.cursorSessionId, + meta.codexSessionId, + meta.claudeSessionId, + meta.geminiSessionId, + meta.opencodeSessionId, + meta.kimiSessionId, + ] + return agentIds.some((id) => typeof id === 'string' && id.startsWith(prefix)) +} + +function detectMediaTool(path) { + const head = readFileSync(path).subarray(0, 16) + if (head.length >= 12 && head.subarray(4, 8).toString('ascii') === 'ftyp') { + const brand = head.subarray(8, 12).toString('ascii') + return brand === 'avif' || brand === 'avis' ? 'display_image' : 'display_video' + } + if (head.length >= 4 && head[0] === 0x1a && head[1] === 0x45 && head[2] === 0xdf && head[3] === 0xa3) { + return 'display_video' + } + return 'display_image' +} + if (!lstatSync(imagePath).isFile()) { console.error(`not a file: ${imagePath}`) process.exit(2) @@ -51,25 +93,39 @@ const sessionsRes = await fetch(`${HAPI_HOST}/api/sessions?limit=500`, { }) const sessionsBody = await sessionsRes.json() const sessions = sessionsBody.sessions ?? sessionsBody -const session = sessions.find((s) => s.id.startsWith(sessionArg)) +const session = sessions.find((s) => sessionMatchesPrefix(s, sessionArg)) if (!session) { - console.error(`no session for prefix ${sessionArg}`) + console.error(`no session for prefix ${sessionArg} (use HAPI session id from /sessions/, not cursorSessionId alone)`) process.exit(4) } -const mcpUrl = session.metadata?.hapiMcpUrl +// List endpoint may omit hapiMcpUrl until hub summary includes it; per-session GET always has it. +let mcpUrl = session.metadata?.hapiMcpUrl +if (!mcpUrl) { + const detailRes = await fetch(`${HAPI_HOST}/api/sessions/${encodeURIComponent(session.id)}`, { + headers: { Authorization: `Bearer ${jwt}` }, + }) + if (!detailRes.ok) { + console.error('session detail fetch failed', detailRes.status) + process.exit(5) + } + const detailBody = await detailRes.json() + const detail = detailBody.session ?? detailBody + mcpUrl = detail.metadata?.hapiMcpUrl +} if (!mcpUrl) { - console.error('session has no hapiMcpUrl metadata (restart session CLI after MCP fix lands)') + console.error('session has no hapiMcpUrl metadata (happy MCP not running in that session CLI โ€” check GET /api/sessions/:id)') process.exit(5) } console.error(`hapi-display-image: session=${session.id} mcp=${mcpUrl}`) +const mediaTool = detectMediaTool(imagePath) const client = new Client({ name: 'hapi-display-image', version: '1.0.0' }, { capabilities: {} }) const transport = new StreamableHTTPClientTransport(new URL(mcpUrl)) await client.connect(transport) const result = await client.callTool({ - name: 'display_image', + name: mediaTool, arguments: { path: imagePath, title: title ?? undefined }, }) await client.close() diff --git a/shared/src/sessionSummary.test.ts b/shared/src/sessionSummary.test.ts index c490c1bc75..b8365d054a 100644 --- a/shared/src/sessionSummary.test.ts +++ b/shared/src/sessionSummary.test.ts @@ -93,6 +93,18 @@ describe('toSessionSummary', () => { expect(summary.metadata?.lifecycleState).toBe('archived') }) + it('includes hapiMcpUrl in summary metadata when session bridge is live', () => { + const summary = toSessionSummary(makeSession({ + metadata: { + path: '/proj', + host: 'local', + hapiMcpUrl: 'http://127.0.0.1:42133/' + } + })) + + expect(summary.metadata?.hapiMcpUrl).toBe('http://127.0.0.1:42133/') + }) + it('includes structured pendingRequests for hover-tooltip copy', () => { const summary = toSessionSummary(makeSession({ updatedAt: 5000, diff --git a/shared/src/sessionSummary.ts b/shared/src/sessionSummary.ts index 32a3a48c23..859cb4b222 100644 --- a/shared/src/sessionSummary.ts +++ b/shared/src/sessionSummary.ts @@ -38,6 +38,8 @@ export type SessionSummaryMetadata = { worktree?: WorktreeMetadata agentSessionId?: string lifecycleState?: string + /** Loopback MCP URL when session CLI happy server is running (#956). */ + hapiMcpUrl?: string } export type SessionSummary = { @@ -122,7 +124,8 @@ export function toSessionSummary(session: Session): SessionSummary { ?? session.metadata.cursorSessionId ?? session.metadata.kimiSessionId ?? undefined, - lifecycleState: session.metadata.lifecycleState + lifecycleState: session.metadata.lifecycleState, + hapiMcpUrl: session.metadata.hapiMcpUrl ?? undefined } : null const todoProgress = session.todos?.length ? { diff --git a/web/src/api/client.ts b/web/src/api/client.ts index d37e0828bc..502a484685 100644 --- a/web/src/api/client.ts +++ b/web/src/api/client.ts @@ -326,9 +326,13 @@ export class ApiClient { if (authToken) { headers.set('authorization', `Bearer ${authToken}`) } - const res = await fetch(this.buildUrl(`/api/sessions/${encodeURIComponent(sessionId)}/generated-images/${encodeURIComponent(imageId)}`), { - headers - }) + const url = this.buildUrl(`/api/sessions/${encodeURIComponent(sessionId)}/generated-images/${encodeURIComponent(imageId)}`) + let res = await fetch(url, { headers }) + // Hub returns ETag + immutable Cache-Control (#927). Default fetch cache stores 200 + // responses so remounts avoid RPC; on 304 the body is empty โ€” read from cache. + if (res.status === 304) { + res = await fetch(url, { headers, cache: 'force-cache' }) + } if (res.status === 401 && attempt === 0 && this.onUnauthorized) { const refreshed = await this.onUnauthorized() if (refreshed) { diff --git a/web/src/chat/inlineMediaSource.ts b/web/src/chat/inlineMediaSource.ts new file mode 100644 index 0000000000..7b0191f8f8 --- /dev/null +++ b/web/src/chat/inlineMediaSource.ts @@ -0,0 +1,28 @@ +/** v1 inline media provenance (wire + chat blocks). See cli/src/modules/common/inlineMediaSource.ts */ +export type InlineMediaIngress = 'mcp' | 'acp' | 'tool_result' + +export type InlineMediaSource = { + ingress: InlineMediaIngress + flavor?: string + toolCallId?: string + toolName?: string +} + +export function inlineMediaSourceFromWire(value: unknown): InlineMediaSource | undefined { + if (!value || typeof value !== 'object' || Array.isArray(value)) return undefined + const record = value as Record + const ingress = record.ingress ?? record.path + if (ingress !== 'mcp' && ingress !== 'acp' && ingress !== 'tool_result') return undefined + const flavor = typeof record.flavor === 'string' ? record.flavor : undefined + const toolCallId = typeof record.toolCallId === 'string' + ? record.toolCallId + : typeof record.tool_call_id === 'string' + ? record.tool_call_id + : undefined + const toolName = typeof record.toolName === 'string' + ? record.toolName + : typeof record.tool_name === 'string' + ? record.tool_name + : undefined + return { ingress, flavor, toolCallId, toolName } +} diff --git a/web/src/chat/normalizeAgent.ts b/web/src/chat/normalizeAgent.ts index 3f2bf8814d..b5b90d5b6f 100644 --- a/web/src/chat/normalizeAgent.ts +++ b/web/src/chat/normalizeAgent.ts @@ -1,4 +1,5 @@ import type { AgentEvent, CodexReview, CodexReviewFinding, NormalizedAgentContent, NormalizedMessage, ToolResultPermission } from '@/chat/types' +import { inlineMediaSourceFromWire } from '@/chat/inlineMediaSource' import { AGENT_MESSAGE_PAYLOAD_TYPE, asNumber, asString, isObject } from '@hapi/protocol' import { isClaudeChatVisibleMessage } from '@hapi/protocol/messages' @@ -559,6 +560,7 @@ export function normalizeAgentRecord( const imageId = asString(data.imageId ?? data.image_id) if (!imageId) return null const uuid = asString(data.id) ?? messageId + const source = inlineMediaSourceFromWire(data.source) return { id: messageId, localId, @@ -571,7 +573,8 @@ export function normalizeAgentRecord( fileName: asString(data.fileName ?? data.file_name) ?? 'generated-image', mimeType: asString(data.mimeType ?? data.mime_type), uuid, - parentUUID: null + parentUUID: null, + source, }], meta } diff --git a/web/src/chat/reconcile.ts b/web/src/chat/reconcile.ts index e4d9a480a2..eeb5338a6e 100644 --- a/web/src/chat/reconcile.ts +++ b/web/src/chat/reconcile.ts @@ -143,6 +143,7 @@ function areGeneratedImageBlocksEqual(left: GeneratedImageBlock, right: Generate && left.imageId === right.imageId && left.fileName === right.fileName && left.mimeType === right.mimeType + && left.source === right.source && left.meta === right.meta } diff --git a/web/src/chat/reducerTimeline.ts b/web/src/chat/reducerTimeline.ts index 6b95d193b4..d7437f780c 100644 --- a/web/src/chat/reducerTimeline.ts +++ b/web/src/chat/reducerTimeline.ts @@ -757,6 +757,7 @@ export function reduceTimeline( imageId: c.imageId, fileName: c.fileName, mimeType: c.mimeType, + source: c.source, meta: msg.meta }) continue diff --git a/web/src/chat/types.ts b/web/src/chat/types.ts index f42d0540a6..13f2df6836 100644 --- a/web/src/chat/types.ts +++ b/web/src/chat/types.ts @@ -1,5 +1,6 @@ import type { AttachmentMetadata, MessageStatus } from '@/types/api' import type { ThreadGoal } from '@/types/api' +import type { InlineMediaSource } from '@/chat/inlineMediaSource' export type UsageData = { input_tokens: number @@ -64,6 +65,7 @@ export type GeneratedImageContent = { mimeType: string | null uuid: string parentUUID: string | null + source?: InlineMediaSource } export type CodexReviewFinding = { @@ -232,6 +234,7 @@ export type GeneratedImageBlock = { imageId: string fileName: string mimeType: string | null + source?: InlineMediaSource meta?: unknown } diff --git a/web/src/components/AssistantChat/messages/ToolMessage.tsx b/web/src/components/AssistantChat/messages/ToolMessage.tsx index 871b463bcb..26b62b2d33 100644 --- a/web/src/components/AssistantChat/messages/ToolMessage.tsx +++ b/web/src/components/AssistantChat/messages/ToolMessage.tsx @@ -1,4 +1,4 @@ -import { useEffect, useState } from 'react' +import { useEffect, useRef, useState, type CSSProperties } from 'react' import type { ToolCallMessagePartProps } from '@assistant-ui/react' import type { ChatBlock } from '@/chat/types' import type { GeneratedImageBlock, ToolCallBlock } from '@/chat/types' @@ -15,6 +15,7 @@ import { useHappyChatContext } from '@/components/AssistantChat/context' import { CliOutputBlock } from '@/components/CliOutputBlock' import { UserBubbleContent, getUserBubbleClassName, shouldShowMessageStatus } from '@/components/AssistantChat/messages/user-bubble' import { ImagePreview } from '@/components/ImagePreview' +import { generatedInlineMediaLabel, isInlineVideoMimeType } from '@/lib/generatedInlineMedia' function isToolCallBlock(value: unknown): value is ToolCallBlock { if (!isObject(value)) return false @@ -49,52 +50,105 @@ function isGeneratedImageBlock(value: unknown): value is GeneratedImageBlock { return true } +const MIN_INLINE_IMAGE_DIMENSION = 64 + +function computeTinyImageScale(width: number, height: number): number { + const minDim = Math.min(width, height) + if (minDim <= 0 || minDim >= MIN_INLINE_IMAGE_DIMENSION) { + return 1 + } + return Math.min(MIN_INLINE_IMAGE_DIMENSION / minDim, 16) +} + function GeneratedImageCard(props: { block: GeneratedImageBlock }) { const ctx = useHappyChatContext() const [objectUrl, setObjectUrl] = useState(null) const [error, setError] = useState(null) + const [imageStyle, setImageStyle] = useState(undefined) + const objectUrlRef = useRef(null) + const isVideo = isInlineVideoMimeType(props.block.mimeType) + const mediaLabel = generatedInlineMediaLabel(props.block.mimeType) + + useEffect(() => { + return () => { + if (objectUrlRef.current) { + URL.revokeObjectURL(objectUrlRef.current) + objectUrlRef.current = null + } + } + }, []) useEffect(() => { let disposed = false - let nextObjectUrl: string | null = null + if (objectUrlRef.current) { + URL.revokeObjectURL(objectUrlRef.current) + objectUrlRef.current = null + } setObjectUrl(null) + setImageStyle(undefined) setError(null) + void ctx.api.getGeneratedImageBlob(ctx.sessionId, props.block.imageId) .then((blob) => { if (disposed) return - nextObjectUrl = URL.createObjectURL(blob) + const nextObjectUrl = URL.createObjectURL(blob) + if (objectUrlRef.current) { + URL.revokeObjectURL(objectUrlRef.current) + } + objectUrlRef.current = nextObjectUrl setObjectUrl(nextObjectUrl) + if (!isVideo) { + setImageStyle(undefined) + const probe = new Image() + probe.onload = () => { + if (disposed) return + const scale = computeTinyImageScale(probe.naturalWidth, probe.naturalHeight) + setImageStyle(scale === 1 ? undefined : { transform: `scale(${scale})` }) + } + probe.src = nextObjectUrl + } }) .catch((err: unknown) => { if (disposed) return - setError(err instanceof Error ? err.message : 'Failed to load generated image') + setError(err instanceof Error ? err.message : 'Failed to load inline media') }) return () => { disposed = true - if (nextObjectUrl) { - URL.revokeObjectURL(nextObjectUrl) - } } - }, [ctx.api, ctx.sessionId, props.block.imageId]) + }, [ctx.api, ctx.sessionId, props.block.imageId, isVideo]) return (
- Generated image ยท {props.block.fileName} + {mediaLabel} ยท {props.block.fileName}
{objectUrl ? ( - + isVideo ? ( +
+
+ ) : ( +
+ +
+ ) ) : error ? (
- Generated image is unavailable. {error} + {mediaLabel} is unavailable. {error}
) : (
diff --git a/web/src/components/ImagePreview.tsx b/web/src/components/ImagePreview.tsx index 49db13b66f..04187b524e 100644 --- a/web/src/components/ImagePreview.tsx +++ b/web/src/components/ImagePreview.tsx @@ -1,4 +1,4 @@ -import { useCallback, useEffect, useRef, useState, type PointerEvent, type ReactNode, type SyntheticEvent, type WheelEvent } from 'react' +import { useCallback, useEffect, useRef, useState, type CSSProperties, type PointerEvent, type ReactNode, type SyntheticEvent, type WheelEvent } from 'react' import { CloseIcon } from '@/components/icons' const MIN_IMAGE_SCALE = 0.25 @@ -29,6 +29,7 @@ export function ImagePreview(props: { label: string buttonClassName?: string imageClassName?: string + imageStyle?: CSSProperties caption?: ReactNode }) { const [viewerOpen, setViewerOpen] = useState(false) @@ -227,6 +228,7 @@ export function ImagePreview(props: { src={props.src} alt={props.label} className={props.imageClassName ?? 'max-h-[calc(100vh-14rem)] max-w-full object-contain transition-transform group-hover:scale-[1.01]'} + style={props.imageStyle} draggable={false} /> {props.caption} diff --git a/web/src/lib/generatedInlineMedia.test.ts b/web/src/lib/generatedInlineMedia.test.ts new file mode 100644 index 0000000000..e7155f0adb --- /dev/null +++ b/web/src/lib/generatedInlineMedia.test.ts @@ -0,0 +1,16 @@ +import { describe, expect, it } from 'vitest' +import { generatedInlineMediaLabel, isInlineVideoMimeType } from './generatedInlineMedia' + +describe('generatedInlineMedia', () => { + it('detects inline video MIME types', () => { + expect(isInlineVideoMimeType('video/mp4')).toBe(true) + expect(isInlineVideoMimeType('video/webm')).toBe(true) + expect(isInlineVideoMimeType('image/png')).toBe(false) + expect(isInlineVideoMimeType(null)).toBe(false) + }) + + it('labels generated inline media by MIME type', () => { + expect(generatedInlineMediaLabel('video/mp4')).toBe('Generated video') + expect(generatedInlineMediaLabel('image/png')).toBe('Generated image') + }) +}) diff --git a/web/src/lib/generatedInlineMedia.ts b/web/src/lib/generatedInlineMedia.ts new file mode 100644 index 0000000000..6d8e02875d --- /dev/null +++ b/web/src/lib/generatedInlineMedia.ts @@ -0,0 +1,7 @@ +export function isInlineVideoMimeType(mimeType: string | null | undefined): boolean { + return typeof mimeType === 'string' && mimeType.startsWith('video/') +} + +export function generatedInlineMediaLabel(mimeType: string | null | undefined): 'Generated video' | 'Generated image' { + return isInlineVideoMimeType(mimeType) ? 'Generated video' : 'Generated image' +}