From c7e06a146c07bc32cfa5bdbb3ce9046a82eeb6c2 Mon Sep 17 00:00:00 2001 From: HeavyGee <133152184+heavygee@users.noreply.github.com> Date: Fri, 19 Jun 2026 17:54:31 +0100 Subject: [PATCH 01/13] feat(cli): cross-flavor inline image display via MCP and ACP Share display_image prompt across MCP-bridge flavors (Cursor, Gemini, Kimi, Codex, Claude, OpenCode), auto-approve the tool in buildHapiMcpBridge, handle ACP image content blocks, and harden generated-image registration with content sniffing. Closes tiann/hapi#956 Co-authored-by: Cursor --- .../backends/acp/AcpMessageHandler.test.ts | 29 ++++++ .../agent/backends/acp/AcpMessageHandler.ts | 29 +++++- cli/src/agent/messageConverter.test.ts | 17 ++++ cli/src/agent/messageConverter.ts | 17 +++- cli/src/agent/types.ts | 1 + cli/src/claude/utils/startHappyServer.ts | 4 +- cli/src/claude/utils/systemPrompt.ts | 3 +- cli/src/codex/codexRemoteLauncher.ts | 40 +++----- .../codex/utils/buildHapiMcpBridge.test.ts | 30 ++++++ cli/src/codex/utils/buildHapiMcpBridge.ts | 14 +-- cli/src/codex/utils/codexMcpConfig.test.ts | 4 + cli/src/codex/utils/systemPrompt.ts | 13 ++- cli/src/cursor/cursorAcpRemoteLauncher.ts | 13 ++- cli/src/kimi/kimiRemoteLauncher.ts | 13 ++- cli/src/modules/common/displayImagePrompt.ts | 17 ++++ .../modules/common/generatedImages.test.ts | 30 +++++- cli/src/modules/common/generatedImages.ts | 97 ++++++++++++++++++- cli/src/modules/common/hapiMcpBridgePrompt.ts | 15 +++ .../permission/BasePermissionHandler.ts | 4 +- cli/src/opencode/opencodeRemoteLauncher.ts | 3 + cli/src/opencode/utils/systemPrompt.ts | 12 +-- 21 files changed, 349 insertions(+), 56 deletions(-) create mode 100644 cli/src/codex/utils/buildHapiMcpBridge.test.ts create mode 100644 cli/src/modules/common/displayImagePrompt.ts create mode 100644 cli/src/modules/common/hapiMcpBridgePrompt.ts diff --git a/cli/src/agent/backends/acp/AcpMessageHandler.test.ts b/cli/src/agent/backends/acp/AcpMessageHandler.test.ts index ed699f590a..e82799f874 100644 --- a/cli/src/agent/backends/acp/AcpMessageHandler.test.ts +++ b/cli/src/agent/backends/acp/AcpMessageHandler.test.ts @@ -4,6 +4,7 @@ import { fileURLToPath } from 'node:url'; import type { AgentMessage } from '@/agent/types'; import { AcpMessageHandler } from './AcpMessageHandler'; import { ACP_SESSION_UPDATE_TYPES } from './constants'; +import { clearGeneratedImages } from '@/modules/common/generatedImages'; function getToolResult(messages: AgentMessage[], id: string): Extract { const result = messages.find((message): message is Extract => @@ -2093,4 +2094,32 @@ describe('AcpMessageHandler', () => { }); }); }); + + it('emits generated_image agent messages from ACP image content blocks', async () => { + const messages: AgentMessage[] = []; + const handler = new AcpMessageHandler((message) => messages.push(message)); + const pngHeader = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x00]); + + handler.handleUpdate({ + sessionUpdate: ACP_SESSION_UPDATE_TYPES.agentMessageChunk, + content: { + type: 'image', + mimeType: 'image/png', + data: pngHeader.toString('base64') + } + }); + + await vi.waitFor(() => { + expect(messages.some((message) => message.type === 'generated_image')).toBe(true); + }); + + const imageMessage = messages.find( + (message): message is Extract => + message.type === 'generated_image' + ); + expect(imageMessage?.mimeType).toBe('image/png'); + expect(imageMessage?.fileName).toBeTruthy(); + expect(imageMessage?.imageId).toBeTruthy(); + clearGeneratedImages(); + }); }); diff --git a/cli/src/agent/backends/acp/AcpMessageHandler.ts b/cli/src/agent/backends/acp/AcpMessageHandler.ts index 68a9ac00b1..ffc0c09a9f 100644 --- a/cli/src/agent/backends/acp/AcpMessageHandler.ts +++ b/cli/src/agent/backends/acp/AcpMessageHandler.ts @@ -1,5 +1,7 @@ -import type { AgentMessage, PlanItem } from '@/agent/types'; import { randomUUID } from 'node:crypto'; +import { logger } from '@/ui/logger'; +import type { AgentMessage, PlanItem } from '@/agent/types'; +import { registerGeneratedImageFromAcpBlock } from '@/modules/common/generatedImages'; import { asString, isObject } from '@hapi/protocol'; import { deriveToolNameWithSource, isPlaceholderToolName } from '@/agent/utils'; import { parseRateLimitText } from '@/agent/rateLimitParser'; @@ -554,6 +556,11 @@ export class AcpMessageHandler { if (updateType === ACP_SESSION_UPDATE_TYPES.agentMessageChunk) { const content = update.content; + if (isObject(content) && content.type === 'image') { + this.flushReasoning(); + void this.emitGeneratedImageFromAcpContent(content); + return; + } const text = extractTextContent(content); if (text) { // Check once whether the buffered text is a prefix of this @@ -629,6 +636,26 @@ export class AcpMessageHandler { } } + private async emitGeneratedImageFromAcpContent(content: Record): Promise { + try { + const image = await registerGeneratedImageFromAcpBlock(content); + if (!image) { + return; + } + this.onMessage({ + type: 'generated_image', + imageId: image.id, + fileName: image.fileName, + mimeType: image.mimeType + }); + } catch (error) { + logger.debug( + '[AcpMessageHandler] Failed to register ACP image block:', + error instanceof Error ? error.message : String(error) + ); + } + } + private handleToolCall(update: Record): void { const toolCallId = asString(update.toolCallId); if (!toolCallId) return; diff --git a/cli/src/agent/messageConverter.test.ts b/cli/src/agent/messageConverter.test.ts index 86ca29cc28..d15f53fb0d 100644 --- a/cli/src/agent/messageConverter.test.ts +++ b/cli/src/agent/messageConverter.test.ts @@ -89,4 +89,21 @@ describe('convertAgentMessage', () => { } }); }); + + it('converts generated_image messages into generated-image wire payloads', () => { + const converted = convertAgentMessage({ + type: 'generated_image', + imageId: 'img-1', + fileName: 'inline.png', + mimeType: 'image/png' + }); + + expect(converted).toMatchObject({ + type: 'generated-image', + imageId: 'img-1', + fileName: 'inline.png', + mimeType: 'image/png' + }); + expect(converted && 'id' in converted && typeof converted.id === 'string').toBe(true); + }); }); diff --git a/cli/src/agent/messageConverter.ts b/cli/src/agent/messageConverter.ts index 4e1dbed50d..67e048505c 100644 --- a/cli/src/agent/messageConverter.ts +++ b/cli/src/agent/messageConverter.ts @@ -32,7 +32,14 @@ export type CodexMessage = is_error?: boolean; } | { type: 'plan'; entries: PlanItem[] } - | { type: 'error'; message: string }; + | { type: 'error'; message: string } + | { + type: 'generated-image'; + imageId: string; + fileName: string; + mimeType: string; + id: string; + }; export function convertAgentMessage(message: AgentMessage): CodexMessage | null { switch (message.type) { @@ -78,6 +85,14 @@ export function convertAgentMessage(message: AgentMessage): CodexMessage | null type: 'plan', entries: message.items }; + case 'generated_image': + return { + type: 'generated-image', + imageId: message.imageId, + fileName: message.fileName, + mimeType: message.mimeType, + id: randomUUID() + }; case 'error': return { type: 'error', message: message.message }; case 'turn_complete': diff --git a/cli/src/agent/types.ts b/cli/src/agent/types.ts index 03d8543187..ce0bc1b066 100644 --- a/cli/src/agent/types.ts +++ b/cli/src/agent/types.ts @@ -44,6 +44,7 @@ export type AgentMessage = contextWindow?: number; } | { type: 'plan'; items: PlanItem[] } + | { type: 'generated_image'; imageId: string; fileName: string; mimeType: string } | { type: 'turn_complete'; stopReason: string } | { type: 'error'; message: string }; diff --git a/cli/src/claude/utils/startHappyServer.ts b/cli/src/claude/utils/startHappyServer.ts index b2001a5703..6a53867884 100644 --- a/cli/src/claude/utils/startHappyServer.ts +++ b/cli/src/claude/utils/startHappyServer.ts @@ -12,7 +12,7 @@ import { z } from "zod"; import { logger } from "@/ui/logger"; import { ApiSessionClient } from "@/api/apiSession"; import { randomUUID } from "node:crypto"; -import { detectImageMimeType, registerGeneratedImage } from "@/modules/common/generatedImages"; +import { detectImageMimeType, registerGeneratedImage, MAX_GENERATED_IMAGE_BYTES } from "@/modules/common/generatedImages"; type StartHappyServerOptions = { emitTitleSummary?: boolean; @@ -94,7 +94,7 @@ function createHapiMcpServer(client: ApiSessionClient, emitTitleSummary: boolean throw new Error('Path is not a regular file'); } - const maxImageBytes = 25 * 1024 * 1024; + const maxImageBytes = MAX_GENERATED_IMAGE_BYTES; if (info.size > maxImageBytes) { throw new Error('Image is too large to display inline'); } diff --git a/cli/src/claude/utils/systemPrompt.ts b/cli/src/claude/utils/systemPrompt.ts index 9e90179d22..941648998e 100644 --- a/cli/src/claude/utils/systemPrompt.ts +++ b/cli/src/claude/utils/systemPrompt.ts @@ -1,12 +1,13 @@ import { trimIdent } from "@/utils/trimIdent"; import { shouldIncludeCoAuthoredBy } from "./claudeSettings"; +import { DISPLAY_IMAGE_PROMPT_CLAUDE } from "@/modules/common/displayImagePrompt"; /** * Base system prompt shared across all configurations */ const BASE_SYSTEM_PROMPT = (() => trimIdent(` Use the title tool sparingly. For a new chat, call the tool "mcp__hapi__change_title" once after the user's initial request is clear, and set a concise task title. Do not rename the chat for routine progress, substeps, implementation details, or a slightly better wording. Rename only when the user's primary objective changes substantially and the existing title would be misleading. - When you create or find a local image file that the user should see, call the tool "mcp__hapi__display_image" with the image path so HAPI can show it inline. + ${DISPLAY_IMAGE_PROMPT_CLAUDE} `))(); /** diff --git a/cli/src/codex/codexRemoteLauncher.ts b/cli/src/codex/codexRemoteLauncher.ts index 2cb80fe17f..3684ab47f1 100644 --- a/cli/src/codex/codexRemoteLauncher.ts +++ b/cli/src/codex/codexRemoteLauncher.ts @@ -1,6 +1,5 @@ import React from 'react'; import { randomUUID } from 'node:crypto'; -import { lstat, readFile } from 'node:fs/promises'; import { CodexAppServerClient } from './codexAppServerClient'; import { CodexPermissionHandler } from './utils/permissionHandler'; @@ -14,7 +13,7 @@ import type { CodexSession } from './session'; import type { EnhancedMode } from './loop'; import { hasCodexCliOverrides } from './utils/codexCliOverrides'; import { AppServerEventConverter } from './utils/appServerEventConverter'; -import { detectImageMimeType, registerGeneratedImage } from '@/modules/common/generatedImages'; +import { registerGeneratedImageFromPath } from '@/modules/common/generatedImages'; import { registerAppServerPermissionHandlers } from './utils/appServerPermissionAdapter'; import { buildThreadStartParams, buildTurnStartParams } from './utils/appServerConfig'; import type { ThreadGoal, ThreadGoalStatus } from './appServerTypes'; @@ -27,32 +26,17 @@ import { } from '@/modules/common/remote/RemoteLauncherBase'; -async function registerGeneratedImageFromPath(args: { id: string; path: string; fileName?: string | null }): Promise | null> { - try { - const info = await lstat(args.path); - if (!info.isFile()) { - throw new Error('Path is not a regular file'); - } - const maxImageBytes = 25 * 1024 * 1024; - if (info.size > maxImageBytes) { - throw new Error('Image is too large to display inline'); - } - const bytes = await readFile(args.path); - const mimeType = detectImageMimeType(bytes); - if (!mimeType) { - throw new Error('Unsupported image content'); - } - return registerGeneratedImage({ - id: args.id, - path: args.path, - fileName: args.fileName, - mimeType, - bytes - }); - } catch (error) { - logger.debug('[CodexRemoteLauncher] Failed to register generated image:', error instanceof Error ? error.message : String(error)); - return null; + +async function registerGeneratedImageFromPathWrapper(args: { id: string; path: string; fileName?: string | null }): Promise> | null> { + const image = await registerGeneratedImageFromPath({ + id: args.id, + path: args.path, + fileName: args.fileName + }); + if (!image) { + logger.debug('[CodexRemoteLauncher] Failed to register generated image from path'); } + return image; } type HappyServer = Awaited>['server']; @@ -2215,7 +2199,7 @@ class CodexRemoteLauncher extends RemoteLauncherBase { const imageId = randomUUID(); const savedPath = asString(msg.saved_path ?? msg.savedPath); if (savedPath) { - const image = await registerGeneratedImageFromPath({ + const image = await registerGeneratedImageFromPathWrapper({ id: imageId, path: savedPath, fileName: asString(msg.file_name ?? msg.fileName) diff --git a/cli/src/codex/utils/buildHapiMcpBridge.test.ts b/cli/src/codex/utils/buildHapiMcpBridge.test.ts new file mode 100644 index 0000000000..efd2314a31 --- /dev/null +++ b/cli/src/codex/utils/buildHapiMcpBridge.test.ts @@ -0,0 +1,30 @@ +import { describe, expect, it, vi } from 'vitest'; +import { buildHapiMcpBridge } from './buildHapiMcpBridge'; + +vi.mock('@/claude/utils/startHappyServer', () => ({ + startHappyServer: vi.fn(async () => ({ + url: 'http://127.0.0.1:63995/', + stop: vi.fn(), + toolNames: ['change_title', 'display_image'] + })) +})); + +vi.mock('@/utils/spawnHappyCLI', () => ({ + getHappyCliCommand: vi.fn(() => ({ + command: 'hapi', + args: ['mcp', '--url', 'http://127.0.0.1:63995/'] + })) +})); + +describe('buildHapiMcpBridge', () => { + it('auto-approves change_title and display_image MCP tools', async () => { + const client = {} as never; + const bridge = await buildHapiMcpBridge(client); + + expect(bridge.mcpServers.hapi.tools).toEqual({ + change_title: { approval_mode: 'approve' }, + display_image: { approval_mode: 'approve' } + }); + expect(bridge.server.url).toBe('http://127.0.0.1:63995/'); + }); +}); diff --git a/cli/src/codex/utils/buildHapiMcpBridge.ts b/cli/src/codex/utils/buildHapiMcpBridge.ts index 72ef85eaa1..86efc9353c 100644 --- a/cli/src/codex/utils/buildHapiMcpBridge.ts +++ b/cli/src/codex/utils/buildHapiMcpBridge.ts @@ -1,8 +1,8 @@ /** - * Unified MCP bridge setup for Codex local and remote modes. + * Unified MCP bridge setup for all flavors that wire HAPI tools through Codex-style MCP config. * - * This module provides a single source of truth for starting the hapi MCP - * bridge server and generating the MCP server configuration that Codex needs. + * Starts the hapi MCP bridge server and returns MCP server configuration for + * Gemini, Kimi, Cursor, OpenCode, and Codex launchers. */ import { startHappyServer } from '@/claude/utils/startHappyServer'; @@ -48,10 +48,9 @@ export interface HapiMcpBridgeOptions { /** * Start the hapi MCP bridge server and return the configuration - * needed to connect Codex to it. + * needed to connect agent flavors to it. * - * This is the single source of truth for MCP bridge setup, - * used by both local and remote launchers. + * Single source of truth for MCP bridge setup across local and remote launchers. */ export async function buildHapiMcpBridge( client: ApiSessionClient, @@ -74,6 +73,9 @@ export async function buildHapiMcpBridge( tools: { change_title: { approval_mode: 'approve' + }, + display_image: { + approval_mode: 'approve' } } } diff --git a/cli/src/codex/utils/codexMcpConfig.test.ts b/cli/src/codex/utils/codexMcpConfig.test.ts index 298f2dbd0e..c0b4597ec0 100644 --- a/cli/src/codex/utils/codexMcpConfig.test.ts +++ b/cli/src/codex/utils/codexMcpConfig.test.ts @@ -31,6 +31,9 @@ describe('codexMcpConfig', () => { tools: { change_title: { approval_mode: 'approve' as const + }, + display_image: { + approval_mode: 'approve' as const } } } @@ -39,6 +42,7 @@ describe('codexMcpConfig', () => { const args = buildMcpServerConfigArgs(mcpServers); expect(args).toContain('mcp_servers.hapi.tools.change_title.approval_mode="approve"'); + expect(args).toContain('mcp_servers.hapi.tools.display_image.approval_mode="approve"'); }); it('builds config args for multiple MCP servers', () => { diff --git a/cli/src/codex/utils/systemPrompt.ts b/cli/src/codex/utils/systemPrompt.ts index a6057be03b..ed003fbb6f 100644 --- a/cli/src/codex/utils/systemPrompt.ts +++ b/cli/src/codex/utils/systemPrompt.ts @@ -1,24 +1,29 @@ /** * Codex-specific system prompt for local mode. * - * This prompt instructs Codex to call the hapi__change_title function - * to set appropriate chat session titles. + * This prompt instructs Codex to call the hapi MCP tools for session title + * and inline image display. */ import { trimIdent } from '@/utils/trimIdent'; +import { DISPLAY_IMAGE_PROMPT_CODEX } from '@/modules/common/displayImagePrompt'; /** * Title instruction for Codex to call the hapi MCP tool. * Note: Codex exposes MCP tools under the `functions.` namespace, * so the tool is called as `functions.hapi__change_title`. */ -export const TITLE_INSTRUCTION = trimIdent(` +const CODEX_TITLE_INSTRUCTION = trimIdent(` Use the title tool sparingly. For a new chat, call it once after the user's initial request is clear, and set a concise task title. Prefer calling functions.hapi__change_title. If that exact tool name is unavailable, call an equivalent alias such as hapi__change_title, mcp__hapi__change_title, or hapi_change_title. Do not rename the chat for routine progress, substeps, implementation details, or a slightly better wording. Rename only when the user's primary objective changes substantially and the existing title would be misleading. - When you create or find a local image file that the user should see, call functions.hapi__display_image with the image path. If that exact tool name is unavailable, use an equivalent alias such as hapi__display_image, mcp__hapi__display_image, or hapi_display_image. +`); + +export const TITLE_INSTRUCTION = trimIdent(` + ${CODEX_TITLE_INSTRUCTION} + ${DISPLAY_IMAGE_PROMPT_CODEX} `); /** diff --git a/cli/src/cursor/cursorAcpRemoteLauncher.ts b/cli/src/cursor/cursorAcpRemoteLauncher.ts index 5e3eaf75b5..add205fd8c 100644 --- a/cli/src/cursor/cursorAcpRemoteLauncher.ts +++ b/cli/src/cursor/cursorAcpRemoteLauncher.ts @@ -20,6 +20,7 @@ import { applyCursorAcpMode, applyCursorAcpModel, wireIdForCursorSessionState } import { buildCursorModelsSeedPayload, seedCursorModelsCache } from '@/modules/common/cursorModels'; import { readSharedCursorModelsCache } from '@/modules/common/cursorModelsSharedCache'; import type { AcpSdkBackend } from '@/agent/backends/acp'; +import { HAPI_MCP_BRIDGE_PROMPT } from '@/modules/common/hapiMcpBridgePrompt'; class CursorAcpRemoteLauncher extends RemoteLauncherBase { private readonly session: CursorSession; @@ -33,6 +34,7 @@ class CursorAcpRemoteLauncher extends RemoteLauncherBase { private defaultBackendModel: string | null = null; private unregisterModelApplyHandler: (() => void) | null = null; private modelApplySeq = 0; + private instructionsSent = false; constructor(session: CursorSession) { super(process.env.DEBUG ? session.logPath : undefined); @@ -211,9 +213,15 @@ class CursorAcpRemoteLauncher extends RemoteLauncherBase { this.applyDisplayMode(batch.mode.permissionMode as PermissionMode); messageBuffer.addMessage(batch.message, 'user'); + let messageText = batch.message; + if (!this.instructionsSent) { + messageText = `${HAPI_MCP_BRIDGE_PROMPT}\n\n${messageText}`; + this.instructionsSent = true; + } + const promptContent: PromptContent[] = [{ type: 'text', - text: batch.message + text: messageText }]; session.onThinkingChange(true); @@ -296,6 +304,9 @@ class CursorAcpRemoteLauncher extends RemoteLauncherBase { case 'error': this.messageBuffer.addMessage(message.message, 'status'); break; + case 'generated_image': + this.messageBuffer.addMessage(`Generated image: ${message.fileName}`, 'assistant'); + break; case 'turn_complete': break; default: diff --git a/cli/src/kimi/kimiRemoteLauncher.ts b/cli/src/kimi/kimiRemoteLauncher.ts index 21fb438f8a..c5ca62d2bb 100644 --- a/cli/src/kimi/kimiRemoteLauncher.ts +++ b/cli/src/kimi/kimiRemoteLauncher.ts @@ -10,6 +10,7 @@ import type { PermissionMode } from './types'; import { createKimiBackend } from './utils/kimiBackend'; import { KimiPermissionHandler } from './utils/permissionHandler'; import { resolveKimiRuntimeConfig } from './utils/config'; +import { HAPI_MCP_BRIDGE_PROMPT } from '@/modules/common/hapiMcpBridgePrompt'; class KimiRemoteLauncher extends RemoteLauncherBase { private readonly session: KimiSession; @@ -23,6 +24,7 @@ class KimiRemoteLauncher extends RemoteLauncherBase { private currentBackendModel: string | null = null; private setModelSupported: boolean | undefined = undefined; private lastDisplayedToolCall = new Map(); + private instructionsSent = false; constructor(session: KimiSession, opts: { model?: string }) { super(process.env.DEBUG ? session.logPath : undefined); @@ -156,9 +158,15 @@ class KimiRemoteLauncher extends RemoteLauncherBase { this.applyDisplayMode(batch.mode.permissionMode, batch.mode.model); messageBuffer.addMessage(batch.message, 'user'); + let messageText = batch.message; + if (!this.instructionsSent) { + messageText = `${HAPI_MCP_BRIDGE_PROMPT}\n\n${messageText}`; + this.instructionsSent = true; + } + const promptContent: PromptContent[] = [{ type: 'text', - text: batch.message + text: messageText }]; session.onThinkingChange(true); @@ -236,6 +244,9 @@ class KimiRemoteLauncher extends RemoteLauncherBase { case 'error': this.messageBuffer.addMessage(message.message, 'status'); break; + case 'generated_image': + this.messageBuffer.addMessage(`Generated image: ${message.fileName}`, 'assistant'); + break; case 'turn_complete': this.messageBuffer.addMessage('Turn complete', 'status'); break; diff --git a/cli/src/modules/common/displayImagePrompt.ts b/cli/src/modules/common/displayImagePrompt.ts new file mode 100644 index 0000000000..507c0903aa --- /dev/null +++ b/cli/src/modules/common/displayImagePrompt.ts @@ -0,0 +1,17 @@ +import { trimIdent } from '@/utils/trimIdent'; + +/** + * Shared display_image MCP tool hints — one export per tool naming convention. + * Inject into flavor system prompts and first-prompt bridge instructions. + */ +export const DISPLAY_IMAGE_PROMPT_CLAUDE = trimIdent(` + When you create or find a local image file that the user should see, call the tool "mcp__hapi__display_image" with the image path so HAPI can show it inline. +`); + +export const DISPLAY_IMAGE_PROMPT_CODEX = trimIdent(` + When you create or find a local image file that the user should see, call functions.hapi__display_image with the image path. If that exact tool name is unavailable, use an equivalent alias such as hapi__display_image, mcp__hapi__display_image, or hapi_display_image. +`); + +export const DISPLAY_IMAGE_PROMPT_HAPI_MCP = trimIdent(` + When you create or find a local image file that the user should see, call the tool "hapi_display_image" with the image path so HAPI can show it inline. +`); diff --git a/cli/src/modules/common/generatedImages.test.ts b/cli/src/modules/common/generatedImages.test.ts index 471539e41e..839c4a44a8 100644 --- a/cli/src/modules/common/generatedImages.test.ts +++ b/cli/src/modules/common/generatedImages.test.ts @@ -1,5 +1,8 @@ import { describe, expect, it } from 'vitest' -import { clearGeneratedImages, detectImageMimeType, getGeneratedImage, registerGeneratedImage } from './generatedImages' +import { mkdirSync, writeFileSync } from 'node:fs' +import { join } from 'node:path' +import { tmpdir } from 'node:os' +import { clearGeneratedImages, detectImageMimeType, getGeneratedImage, registerGeneratedImage, registerGeneratedImageFromAcpBlock, registerGeneratedImageFromPath } from './generatedImages' describe('generatedImages', () => { it('detects supported image MIME types from file bytes', () => { @@ -67,4 +70,29 @@ describe('generatedImages', () => { clearGeneratedImages() }) + it('registers images from ACP base64 image blocks after MIME sniffing', async () => { + const pngHeader = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x00]) + const image = await registerGeneratedImageFromAcpBlock({ + type: 'image', + mimeType: 'image/png', + data: pngHeader.toString('base64') + }) + + expect(image?.mimeType).toBe('image/png') + expect(getGeneratedImage(image!.id)?.content.subarray(0, 8)).toEqual(pngHeader.subarray(0, 8)) + clearGeneratedImages() + }) + + it('registers images from local file paths in ACP uri blocks', async () => { + const dir = join(tmpdir(), `hapi-acp-image-${Date.now()}`) + mkdirSync(dir, { recursive: true }) + const path = join(dir, 'inline.png') + const bytes = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00]) + writeFileSync(path, bytes) + + const image = await registerGeneratedImageFromPath({ path }) + expect(image?.mimeType).toBe('image/png') + clearGeneratedImages() + }) + }) diff --git a/cli/src/modules/common/generatedImages.ts b/cli/src/modules/common/generatedImages.ts index fdf6cb9441..dc3f53fb71 100644 --- a/cli/src/modules/common/generatedImages.ts +++ b/cli/src/modules/common/generatedImages.ts @@ -1,4 +1,8 @@ -import { basename } from 'path' +import { basename } from 'node:path' +import { fileURLToPath } from 'node:url' +import { lstat, readFile } from 'node:fs/promises' +import { randomUUID } from 'node:crypto' +import { asString, isObject } from '@hapi/protocol' export type GeneratedImageMetadata = { id: string @@ -8,7 +12,7 @@ export type GeneratedImageMetadata = { createdAt: number } -const MAX_GENERATED_IMAGE_BYTES = 25 * 1024 * 1024 +export const MAX_GENERATED_IMAGE_BYTES = 25 * 1024 * 1024 const MAX_GENERATED_IMAGE_TOTAL_BYTES = 100 * 1024 * 1024 const MAX_GENERATED_IMAGE_COUNT = 100 @@ -105,3 +109,92 @@ export function clearGeneratedImages(): void { generatedImages.clear() generatedImageBytes = 0 } + +export async function registerGeneratedImageFromPath(args: { + id?: string + path: string + fileName?: string | null +}): Promise { + try { + const info = await lstat(args.path) + if (!info.isFile()) { + throw new Error('Path is not a regular file') + } + if (info.size > MAX_GENERATED_IMAGE_BYTES) { + throw new Error('Image is too large to display inline') + } + const bytes = await readFile(args.path) + const mimeType = detectImageMimeType(bytes) + if (!mimeType) { + throw new Error('Unsupported image content') + } + return registerGeneratedImage({ + id: args.id ?? randomUUID(), + path: args.path, + fileName: args.fileName, + mimeType, + bytes + }) + } catch { + return null + } +} + +function parseAcpImageUri(uri: string): string | null { + if (uri.startsWith('file://')) { + try { + return fileURLToPath(uri) + } catch { + return null + } + } + if (/^https?:\/\//i.test(uri)) { + return null + } + return uri +} + +export async function registerGeneratedImageFromAcpBlock(block: unknown): Promise { + if (!isObject(block) || block.type !== 'image') { + return null + } + + const data = asString(block.data) + const declaredMimeType = asString(block.mimeType ?? block.mime_type) + const uri = asString(block.uri ?? block.url) + + if (data) { + const bytes = Buffer.from(data, 'base64') + if (bytes.byteLength > MAX_GENERATED_IMAGE_BYTES) { + return null + } + const sniffedMimeType = detectImageMimeType(bytes) + if (!sniffedMimeType) { + return null + } + if (declaredMimeType && declaredMimeType !== sniffedMimeType) { + return null + } + const path = uri ? parseAcpImageUri(uri) ?? uri : `${randomUUID()}.bin` + return registerGeneratedImage({ + id: randomUUID(), + path, + fileName: basename(path), + mimeType: sniffedMimeType, + bytes + }) + } + + if (uri) { + const path = parseAcpImageUri(uri) + if (!path) { + return null + } + return registerGeneratedImageFromPath({ + path, + fileName: basename(path) + }) + } + + return null +} diff --git a/cli/src/modules/common/hapiMcpBridgePrompt.ts b/cli/src/modules/common/hapiMcpBridgePrompt.ts new file mode 100644 index 0000000000..74bf4b029b --- /dev/null +++ b/cli/src/modules/common/hapiMcpBridgePrompt.ts @@ -0,0 +1,15 @@ +import { trimIdent } from '@/utils/trimIdent'; +import { DISPLAY_IMAGE_PROMPT_HAPI_MCP } from './displayImagePrompt'; + +/** + * Title + display_image instructions for ACP flavors wired through buildHapiMcpBridge + * (Gemini, Kimi, Cursor, OpenCode). Prepended on the first user prompt. + */ +export const HAPI_MCP_TITLE_INSTRUCTION = trimIdent(` + Use the title tool sparingly. For a new chat, call the tool "hapi_change_title" once after the user's initial request is clear, and set a concise task title. Do not rename the chat for routine progress, substeps, implementation details, or a slightly better wording. Rename only when the user's primary objective changes substantially and the existing title would be misleading. +`); + +export const HAPI_MCP_BRIDGE_PROMPT = trimIdent(` + ${HAPI_MCP_TITLE_INSTRUCTION} + ${DISPLAY_IMAGE_PROMPT_HAPI_MCP} +`); diff --git a/cli/src/modules/common/permission/BasePermissionHandler.ts b/cli/src/modules/common/permission/BasePermissionHandler.ts index f70908f5eb..b68dc79235 100644 --- a/cli/src/modules/common/permission/BasePermissionHandler.ts +++ b/cli/src/modules/common/permission/BasePermissionHandler.ts @@ -19,14 +19,16 @@ export type AutoApprovalRuleSet = { const AUTO_APPROVE_TOOL_NAME_HINTS = [ 'change_title', + 'display_image', 'happy__change_title', 'hapi_change_title', // OpenCode MCP tool pattern + 'hapi_display_image', 'geminireasoning', 'codexreasoning', 'think', 'save_memory' ]; -const AUTO_APPROVE_TOOL_ID_HINTS = ['change_title', 'save_memory']; +const AUTO_APPROVE_TOOL_ID_HINTS = ['change_title', 'display_image', 'save_memory']; const AUTO_APPROVE_WRITE_TOOL_HINTS = ['write', 'edit', 'create', 'delete', 'patch', 'fs-edit']; export function resolveToolAutoApprovalDecision( diff --git a/cli/src/opencode/opencodeRemoteLauncher.ts b/cli/src/opencode/opencodeRemoteLauncher.ts index b911fbca07..0abca7e06b 100644 --- a/cli/src/opencode/opencodeRemoteLauncher.ts +++ b/cli/src/opencode/opencodeRemoteLauncher.ts @@ -360,6 +360,9 @@ class OpencodeRemoteLauncher extends RemoteLauncherBase { case 'error': this.messageBuffer.addMessage(message.message, 'status'); break; + case 'generated_image': + this.messageBuffer.addMessage(`Generated image: ${message.fileName}`, 'assistant'); + break; case 'turn_complete': this.messageBuffer.addMessage('Turn complete', 'status'); break; diff --git a/cli/src/opencode/utils/systemPrompt.ts b/cli/src/opencode/utils/systemPrompt.ts index e968759626..9a2e556b26 100644 --- a/cli/src/opencode/utils/systemPrompt.ts +++ b/cli/src/opencode/utils/systemPrompt.ts @@ -1,19 +1,17 @@ /** - * OpenCode-specific system prompt for change_title tool. + * OpenCode-specific system prompt for hapi MCP tools (change_title, display_image). * * OpenCode exposes MCP tools with the naming pattern: _ - * The hapi MCP server exposes `change_title`, so it's called as `hapi_change_title`. + * The hapi MCP server exposes `change_title` and `display_image`. */ import { trimIdent } from '@/utils/trimIdent'; +import { HAPI_MCP_BRIDGE_PROMPT } from '@/modules/common/hapiMcpBridgePrompt'; /** - * Title instruction for OpenCode to call the hapi MCP tool. + * Title and display_image instructions for OpenCode to call the hapi MCP tools. */ -export const TITLE_INSTRUCTION = trimIdent(` - Use the title tool sparingly. For a new chat, call the tool "hapi_change_title" once after the user's initial request is clear, and set a concise task title. Do not rename the chat for routine progress, substeps, implementation details, or a slightly better wording. Rename only when the user's primary objective changes substantially and the existing title would be misleading. - When you create or find a local image file that the user should see, call the tool "hapi_display_image" with the image path so HAPI can show it inline. -`); +export const TITLE_INSTRUCTION = HAPI_MCP_BRIDGE_PROMPT; /** * The system prompt to inject for OpenCode sessions. From cb773762b9fd6298f95c83f242ebb22d385105c9 Mon Sep 17 00:00:00 2001 From: HeavyGee <133152184+heavygee@users.noreply.github.com> Date: Fri, 19 Jun 2026 23:07:34 +0100 Subject: [PATCH 02/13] fix(web): render generated-image cards reliably in chat Keep object URLs stable across refetch, upscale tiny inline images, fetch generated-image bytes with cache no-store (avoid empty 304 bodies), and load hapiMcpUrl from per-session API in hapi-display-image tooling. Co-authored-by: Cursor --- scripts/tooling/hapi-display-image.mjs | 15 ++++- web/src/api/client.ts | 5 +- .../AssistantChat/messages/ToolMessage.tsx | 59 ++++++++++++++----- web/src/components/ImagePreview.tsx | 4 +- 4 files changed, 66 insertions(+), 17 deletions(-) diff --git a/scripts/tooling/hapi-display-image.mjs b/scripts/tooling/hapi-display-image.mjs index c2670585d2..c490cc467d 100644 --- a/scripts/tooling/hapi-display-image.mjs +++ b/scripts/tooling/hapi-display-image.mjs @@ -57,7 +57,20 @@ if (!session) { process.exit(4) } -const mcpUrl = session.metadata?.hapiMcpUrl +// List endpoint omits metadata; per-session GET includes hapiMcpUrl. +let mcpUrl = session.metadata?.hapiMcpUrl +if (!mcpUrl) { + const detailRes = await fetch(`${HAPI_HOST}/api/sessions/${encodeURIComponent(session.id)}`, { + headers: { Authorization: `Bearer ${jwt}` }, + }) + if (!detailRes.ok) { + console.error('session detail fetch failed', detailRes.status) + process.exit(5) + } + const detailBody = await detailRes.json() + const detail = detailBody.session ?? detailBody + mcpUrl = detail.metadata?.hapiMcpUrl +} if (!mcpUrl) { console.error('session has no hapiMcpUrl metadata (restart session CLI after MCP fix lands)') process.exit(5) diff --git a/web/src/api/client.ts b/web/src/api/client.ts index d37e0828bc..febc565b55 100644 --- a/web/src/api/client.ts +++ b/web/src/api/client.ts @@ -327,7 +327,10 @@ export class ApiClient { headers.set('authorization', `Bearer ${authToken}`) } const res = await fetch(this.buildUrl(`/api/sessions/${encodeURIComponent(sessionId)}/generated-images/${encodeURIComponent(imageId)}`), { - headers + headers, + // Hub answers with ETag + Cache-Control; browser 304 responses have no body and + // fail res.ok — use no-store so GeneratedImageCard always gets bytes (issue #927). + cache: 'no-store' }) if (res.status === 401 && attempt === 0 && this.onUnauthorized) { const refreshed = await this.onUnauthorized() diff --git a/web/src/components/AssistantChat/messages/ToolMessage.tsx b/web/src/components/AssistantChat/messages/ToolMessage.tsx index 871b463bcb..919707e10c 100644 --- a/web/src/components/AssistantChat/messages/ToolMessage.tsx +++ b/web/src/components/AssistantChat/messages/ToolMessage.tsx @@ -1,4 +1,4 @@ -import { useEffect, useState } from 'react' +import { useEffect, useRef, useState, type CSSProperties } from 'react' import type { ToolCallMessagePartProps } from '@assistant-ui/react' import type { ChatBlock } from '@/chat/types' import type { GeneratedImageBlock, ToolCallBlock } from '@/chat/types' @@ -49,22 +49,53 @@ function isGeneratedImageBlock(value: unknown): value is GeneratedImageBlock { return true } +const MIN_INLINE_IMAGE_DIMENSION = 64 + +function computeTinyImageScale(width: number, height: number): number { + const minDim = Math.min(width, height) + if (minDim <= 0 || minDim >= MIN_INLINE_IMAGE_DIMENSION) { + return 1 + } + return Math.min(MIN_INLINE_IMAGE_DIMENSION / minDim, 16) +} + function GeneratedImageCard(props: { block: GeneratedImageBlock }) { const ctx = useHappyChatContext() const [objectUrl, setObjectUrl] = useState(null) const [error, setError] = useState(null) + const [imageStyle, setImageStyle] = useState(undefined) + const objectUrlRef = useRef(null) + + useEffect(() => { + return () => { + if (objectUrlRef.current) { + URL.revokeObjectURL(objectUrlRef.current) + objectUrlRef.current = null + } + } + }, []) useEffect(() => { let disposed = false - let nextObjectUrl: string | null = null - setObjectUrl(null) setError(null) void ctx.api.getGeneratedImageBlob(ctx.sessionId, props.block.imageId) .then((blob) => { if (disposed) return - nextObjectUrl = URL.createObjectURL(blob) + const nextObjectUrl = URL.createObjectURL(blob) + if (objectUrlRef.current) { + URL.revokeObjectURL(objectUrlRef.current) + } + objectUrlRef.current = nextObjectUrl setObjectUrl(nextObjectUrl) + setImageStyle(undefined) + const probe = new Image() + probe.onload = () => { + if (disposed) return + const scale = computeTinyImageScale(probe.naturalWidth, probe.naturalHeight) + setImageStyle(scale === 1 ? undefined : { transform: `scale(${scale})` }) + } + probe.src = nextObjectUrl }) .catch((err: unknown) => { if (disposed) return @@ -73,9 +104,6 @@ function GeneratedImageCard(props: { block: GeneratedImageBlock }) { return () => { disposed = true - if (nextObjectUrl) { - URL.revokeObjectURL(nextObjectUrl) - } } }, [ctx.api, ctx.sessionId, props.block.imageId]) @@ -85,13 +113,16 @@ function GeneratedImageCard(props: { block: GeneratedImageBlock }) { Generated image · {props.block.fileName} {objectUrl ? ( - +
+ +
) : error ? (
Generated image is unavailable. {error} diff --git a/web/src/components/ImagePreview.tsx b/web/src/components/ImagePreview.tsx index 49db13b66f..04187b524e 100644 --- a/web/src/components/ImagePreview.tsx +++ b/web/src/components/ImagePreview.tsx @@ -1,4 +1,4 @@ -import { useCallback, useEffect, useRef, useState, type PointerEvent, type ReactNode, type SyntheticEvent, type WheelEvent } from 'react' +import { useCallback, useEffect, useRef, useState, type CSSProperties, type PointerEvent, type ReactNode, type SyntheticEvent, type WheelEvent } from 'react' import { CloseIcon } from '@/components/icons' const MIN_IMAGE_SCALE = 0.25 @@ -29,6 +29,7 @@ export function ImagePreview(props: { label: string buttonClassName?: string imageClassName?: string + imageStyle?: CSSProperties caption?: ReactNode }) { const [viewerOpen, setViewerOpen] = useState(false) @@ -227,6 +228,7 @@ export function ImagePreview(props: { src={props.src} alt={props.label} className={props.imageClassName ?? 'max-h-[calc(100vh-14rem)] max-w-full object-contain transition-transform group-hover:scale-[1.01]'} + style={props.imageStyle} draggable={false} /> {props.caption} From 3aa0aa50fbf33ba74f292db90fffe67bb1f99a08 Mon Sep 17 00:00:00 2001 From: HeavyGee <133152184+heavygee@users.noreply.github.com> Date: Sat, 20 Jun 2026 22:01:11 +0100 Subject: [PATCH 03/13] feat(cli+web): display_video MCP for inline mp4/webm (#956) Add display_video alongside display_image, video MIME sniffing with avif guard, web GeneratedImageCard video player, and hapi-display-image auto-routing. Co-authored-by: Cursor --- cli/src/claude/utils/startHappyServer.ts | 114 +++++++++++++----- cli/src/claude/utils/systemPrompt.ts | 4 +- cli/src/codex/happyMcpStdioBridge.ts | 28 +++++ cli/src/codex/utils/buildHapiMcpBridge.ts | 14 ++- cli/src/codex/utils/systemPrompt.ts | 14 +-- .../modules/common/generatedImages.test.ts | 10 +- cli/src/modules/common/generatedImages.ts | 30 ++++- scripts/tooling/hapi-display-image.mjs | 36 ++++-- .../AssistantChat/messages/ToolMessage.tsx | 70 ++++------- 9 files changed, 215 insertions(+), 105 deletions(-) diff --git a/cli/src/claude/utils/startHappyServer.ts b/cli/src/claude/utils/startHappyServer.ts index 6a53867884..91eab8741a 100644 --- a/cli/src/claude/utils/startHappyServer.ts +++ b/cli/src/claude/utils/startHappyServer.ts @@ -12,7 +12,7 @@ import { z } from "zod"; import { logger } from "@/ui/logger"; import { ApiSessionClient } from "@/api/apiSession"; import { randomUUID } from "node:crypto"; -import { detectImageMimeType, registerGeneratedImage, MAX_GENERATED_IMAGE_BYTES } from "@/modules/common/generatedImages"; +import { detectImageMimeType, detectVideoMimeType, registerGeneratedImage } from "@/modules/common/generatedImages"; type StartHappyServerOptions = { emitTitleSummary?: boolean; @@ -50,6 +50,50 @@ function createHapiMcpServer(client: ApiSessionClient, emitTitleSummary: boolean title: z.string().optional().describe('Optional display title or filename for the image'), }); + const displayVideoInputSchema: z.ZodTypeAny = z.object({ + path: z.string().describe('Local filesystem path of the video to display inline (mp4 or webm)'), + title: z.string().optional().describe('Optional display title or filename for the video'), + }); + + const maxInlineMediaBytes = 25 * 1024 * 1024; + + async function displayInlineMedia(args: { path: string; title?: string }, mediaKind: 'image' | 'video') { + const info = await lstat(args.path); + if (!info.isFile()) { + throw new Error('Path is not a regular file'); + } + + if (info.size > maxInlineMediaBytes) { + throw new Error('File is too large to display inline'); + } + + const bytes = await readFile(args.path); + const mimeType = mediaKind === 'video' + ? detectVideoMimeType(bytes) + : detectImageMimeType(bytes); + if (!mimeType) { + throw new Error(mediaKind === 'video' ? 'Unsupported video content' : 'Unsupported image content'); + } + + const media = registerGeneratedImage({ + id: randomUUID(), + path: args.path, + fileName: args.title, + mimeType, + bytes + }); + + client.sendAgentMessage({ + type: 'generated-image', + imageId: media.id, + fileName: media.fileName, + mimeType: media.mimeType, + id: randomUUID() + }); + + return media; + } + mcp.registerTool('change_title', { description: 'Change the title of the current chat session', title: 'Change Chat Title', @@ -89,37 +133,7 @@ function createHapiMcpServer(client: ApiSessionClient, emitTitleSummary: boolean logger.debug('[hapiMCP] Display image:', args.path); try { - const info = await lstat(args.path); - if (!info.isFile()) { - throw new Error('Path is not a regular file'); - } - - const maxImageBytes = MAX_GENERATED_IMAGE_BYTES; - if (info.size > maxImageBytes) { - throw new Error('Image is too large to display inline'); - } - - const bytes = await readFile(args.path); - const mimeType = detectImageMimeType(bytes); - if (!mimeType) { - throw new Error('Unsupported image content'); - } - - const image = registerGeneratedImage({ - id: randomUUID(), - path: args.path, - fileName: args.title, - mimeType, - bytes - }); - - client.sendAgentMessage({ - type: 'generated-image', - imageId: image.id, - fileName: image.fileName, - mimeType: image.mimeType, - id: randomUUID() - }); + const image = await displayInlineMedia(args, 'image'); return { content: [ @@ -145,6 +159,40 @@ function createHapiMcpServer(client: ApiSessionClient, emitTitleSummary: boolean } }); + mcp.registerTool('display_video', { + description: 'Display a local mp4 or webm file inline in the current HAPI chat session', + title: 'Display Video', + inputSchema: displayVideoInputSchema, + }, async (args: { path: string; title?: string }) => { + logger.debug('[hapiMCP] Display video:', args.path); + + try { + const video = await displayInlineMedia(args, 'video'); + + return { + content: [ + { + type: 'text' as const, + text: `Displayed video: ${video.fileName}`, + }, + ], + isError: false, + }; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + logger.debug('[hapiMCP] Failed to display video:', message); + return { + content: [ + { + type: 'text' as const, + text: `Failed to display video: ${message}`, + }, + ], + isError: true, + }; + } + }); + return mcp; } @@ -221,7 +269,7 @@ export async function startHappyServer(client: ApiSessionClient, options: StartH return { url: mcpUrl, - toolNames: ['change_title', 'display_image'], + toolNames: ['change_title', 'display_image', 'display_video'], stop: () => { logger.debug('[hapiMCP] Stopping server'); for (const mcp of mcps.values()) { diff --git a/cli/src/claude/utils/systemPrompt.ts b/cli/src/claude/utils/systemPrompt.ts index 941648998e..804001d975 100644 --- a/cli/src/claude/utils/systemPrompt.ts +++ b/cli/src/claude/utils/systemPrompt.ts @@ -1,13 +1,13 @@ import { trimIdent } from "@/utils/trimIdent"; import { shouldIncludeCoAuthoredBy } from "./claudeSettings"; -import { DISPLAY_IMAGE_PROMPT_CLAUDE } from "@/modules/common/displayImagePrompt"; /** * Base system prompt shared across all configurations */ const BASE_SYSTEM_PROMPT = (() => trimIdent(` Use the title tool sparingly. For a new chat, call the tool "mcp__hapi__change_title" once after the user's initial request is clear, and set a concise task title. Do not rename the chat for routine progress, substeps, implementation details, or a slightly better wording. Rename only when the user's primary objective changes substantially and the existing title would be misleading. - ${DISPLAY_IMAGE_PROMPT_CLAUDE} + When you create or find a local image file that the user should see, call the tool "mcp__hapi__display_image" with the image path so HAPI can show it inline. + When you create or find a local mp4 or webm recording the user should see, call the tool "mcp__hapi__display_video" with the file path so HAPI can show it inline. `))(); /** diff --git a/cli/src/codex/happyMcpStdioBridge.ts b/cli/src/codex/happyMcpStdioBridge.ts index 7c30617c2b..518eeffb9b 100644 --- a/cli/src/codex/happyMcpStdioBridge.ts +++ b/cli/src/codex/happyMcpStdioBridge.ts @@ -123,6 +123,34 @@ export async function runHappyMcpStdioBridge(argv: string[]): Promise { } ); + const displayVideoInputSchema: z.ZodTypeAny = z.object({ + path: z.string().describe('Local filesystem path of the video to display inline (mp4 or webm)'), + title: z.string().optional().describe('Optional display title or filename for the video'), + }); + + server.registerTool( + 'display_video', + { + description: 'Display a local mp4 or webm file inline in the current HAPI chat session', + title: 'Display Video', + inputSchema: displayVideoInputSchema, + }, + async (args: Record) => { + try { + const client = await ensureHttpClient(); + const response = await client.callTool({ name: 'display_video', arguments: args }); + return response as any; + } catch (error) { + return { + content: [ + { type: 'text' as const, text: `Failed to display video: ${error instanceof Error ? error.message : String(error)}` }, + ], + isError: true, + }; + } + } + ); + // Start STDIO transport const stdio = new StdioServerTransport(); await server.connect(stdio); diff --git a/cli/src/codex/utils/buildHapiMcpBridge.ts b/cli/src/codex/utils/buildHapiMcpBridge.ts index 86efc9353c..7fd7cf329c 100644 --- a/cli/src/codex/utils/buildHapiMcpBridge.ts +++ b/cli/src/codex/utils/buildHapiMcpBridge.ts @@ -1,8 +1,8 @@ /** - * Unified MCP bridge setup for all flavors that wire HAPI tools through Codex-style MCP config. + * Unified MCP bridge setup for Codex local and remote modes. * - * Starts the hapi MCP bridge server and returns MCP server configuration for - * Gemini, Kimi, Cursor, OpenCode, and Codex launchers. + * This module provides a single source of truth for starting the hapi MCP + * bridge server and generating the MCP server configuration that Codex needs. */ import { startHappyServer } from '@/claude/utils/startHappyServer'; @@ -48,9 +48,10 @@ export interface HapiMcpBridgeOptions { /** * Start the hapi MCP bridge server and return the configuration - * needed to connect agent flavors to it. + * needed to connect Codex to it. * - * Single source of truth for MCP bridge setup across local and remote launchers. + * This is the single source of truth for MCP bridge setup, + * used by both local and remote launchers. */ export async function buildHapiMcpBridge( client: ApiSessionClient, @@ -76,6 +77,9 @@ export async function buildHapiMcpBridge( }, display_image: { approval_mode: 'approve' + }, + display_video: { + approval_mode: 'approve' } } } diff --git a/cli/src/codex/utils/systemPrompt.ts b/cli/src/codex/utils/systemPrompt.ts index ed003fbb6f..3f58b216a7 100644 --- a/cli/src/codex/utils/systemPrompt.ts +++ b/cli/src/codex/utils/systemPrompt.ts @@ -1,29 +1,25 @@ /** * Codex-specific system prompt for local mode. * - * This prompt instructs Codex to call the hapi MCP tools for session title - * and inline image display. + * This prompt instructs Codex to call the hapi__change_title function + * to set appropriate chat session titles. */ import { trimIdent } from '@/utils/trimIdent'; -import { DISPLAY_IMAGE_PROMPT_CODEX } from '@/modules/common/displayImagePrompt'; /** * Title instruction for Codex to call the hapi MCP tool. * Note: Codex exposes MCP tools under the `functions.` namespace, * so the tool is called as `functions.hapi__change_title`. */ -const CODEX_TITLE_INSTRUCTION = trimIdent(` +export const TITLE_INSTRUCTION = trimIdent(` Use the title tool sparingly. For a new chat, call it once after the user's initial request is clear, and set a concise task title. Prefer calling functions.hapi__change_title. If that exact tool name is unavailable, call an equivalent alias such as hapi__change_title, mcp__hapi__change_title, or hapi_change_title. Do not rename the chat for routine progress, substeps, implementation details, or a slightly better wording. Rename only when the user's primary objective changes substantially and the existing title would be misleading. -`); - -export const TITLE_INSTRUCTION = trimIdent(` - ${CODEX_TITLE_INSTRUCTION} - ${DISPLAY_IMAGE_PROMPT_CODEX} + When you create or find a local image file that the user should see, call functions.hapi__display_image with the image path. If that exact tool name is unavailable, use an equivalent alias such as hapi__display_image, mcp__hapi__display_image, or hapi_display_image. + When you create or find a local mp4 or webm file the user should see, call functions.hapi__display_video with the file path. If unavailable, use hapi__display_video, mcp__hapi__display_video, or hapi_display_video. `); /** diff --git a/cli/src/modules/common/generatedImages.test.ts b/cli/src/modules/common/generatedImages.test.ts index 839c4a44a8..3da6492e0f 100644 --- a/cli/src/modules/common/generatedImages.test.ts +++ b/cli/src/modules/common/generatedImages.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it } from 'vitest' import { mkdirSync, writeFileSync } from 'node:fs' import { join } from 'node:path' import { tmpdir } from 'node:os' -import { clearGeneratedImages, detectImageMimeType, getGeneratedImage, registerGeneratedImage, registerGeneratedImageFromAcpBlock, registerGeneratedImageFromPath } from './generatedImages' +import { clearGeneratedImages, detectImageMimeType, detectVideoMimeType, getGeneratedImage, registerGeneratedImage, registerGeneratedImageFromAcpBlock, registerGeneratedImageFromPath } from './generatedImages' describe('generatedImages', () => { it('detects supported image MIME types from file bytes', () => { @@ -13,6 +13,12 @@ describe('generatedImages', () => { expect(detectImageMimeType(Buffer.from([0x00, 0x00, 0x00, 0x1c, 0x66, 0x74, 0x79, 0x70, 0x61, 0x76, 0x69, 0x66]))).toBe('image/avif') }) + it('detects supported video MIME types from file bytes', () => { + expect(detectVideoMimeType(Buffer.from([0x00, 0x00, 0x00, 0x18, 0x66, 0x74, 0x79, 0x70, 0x69, 0x73, 0x6f, 0x6d]))).toBe('video/mp4') + expect(detectVideoMimeType(Buffer.from([0x1a, 0x45, 0xdf, 0xa3]))).toBe('video/webm') + expect(detectVideoMimeType(Buffer.from([0x00, 0x00, 0x00, 0x1c, 0x66, 0x74, 0x79, 0x70, 0x61, 0x76, 0x69, 0x66]))).toBeNull() + }) + it('rejects non-image bytes even if the path has an image extension', () => { expect(detectImageMimeType(Buffer.from('not really a png'))).toBeNull() }) @@ -50,7 +56,7 @@ describe('generatedImages', () => { path: '/tmp/large.png', mimeType: 'image/png', bytes: new Uint8Array(25 * 1024 * 1024 + 1) - })).toThrow('Image is too large to display inline') + })).toThrow('File is too large to display inline') clearGeneratedImages() }) diff --git a/cli/src/modules/common/generatedImages.ts b/cli/src/modules/common/generatedImages.ts index dc3f53fb71..f1462a8f0e 100644 --- a/cli/src/modules/common/generatedImages.ts +++ b/cli/src/modules/common/generatedImages.ts @@ -59,6 +59,30 @@ export function detectImageMimeType(bytes: Uint8Array): string | null { return null } +export function detectVideoMimeType(bytes: Uint8Array): string | null { + if (bytes.length >= 12 && ascii(bytes, 4, 8) === 'ftyp') { + const brand = ascii(bytes, 8, 12) + if (brand === 'avif' || brand === 'avis') { + return null + } + return 'video/mp4' + } + + if (bytes.length >= 4 + && bytes[0] === 0x1a + && bytes[1] === 0x45 + && bytes[2] === 0xdf + && bytes[3] === 0xa3) { + return 'video/webm' + } + + return null +} + +export function isInlineMediaMimeType(mimeType: string): boolean { + return mimeType.startsWith('image/') || mimeType.startsWith('video/') +} + function ascii(bytes: Uint8Array, start: number, end: number): string { return String.fromCharCode(...bytes.subarray(start, end)) } @@ -66,7 +90,11 @@ function ascii(bytes: Uint8Array, start: number, end: number): string { export function registerGeneratedImage(args: { id: string; path: string; mimeType: string; bytes: Uint8Array; fileName?: string | null }): GeneratedImageMetadata { const content = Buffer.from(args.bytes) if (content.byteLength > MAX_GENERATED_IMAGE_BYTES) { - throw new Error('Image is too large to display inline') + throw new Error('File is too large to display inline') + } + + if (!isInlineMediaMimeType(args.mimeType)) { + throw new Error('Unsupported inline media MIME type') } const previous = generatedImages.get(args.id) diff --git a/scripts/tooling/hapi-display-image.mjs b/scripts/tooling/hapi-display-image.mjs index c490cc467d..454eddb139 100644 --- a/scripts/tooling/hapi-display-image.mjs +++ b/scripts/tooling/hapi-display-image.mjs @@ -1,17 +1,25 @@ #!/usr/bin/env bun /** - * Post a local image inline to a HAPI session via the session CLI's display_image MCP tool. + * Post a local image or video inline to a HAPI session via display_image / display_video MCP. * * Uses session.metadata.hapiMcpUrl (published at MCP server start) so we hit the MCP * endpoint, not the session hook server on another loopback port in the same process. * * Usage: - * bun scripts/tooling/hapi-display-image.mjs [title] + * bun scripts/tooling/hapi-display-image.mjs [title] + * + * Picks display_video for mp4/webm (ftyp / webm magic), else display_image. */ import { readFileSync, lstatSync } from 'node:fs' -import { Client } from '@modelcontextprotocol/sdk/client/index.js' -import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js' +import { fileURLToPath } from 'node:url' +import { dirname, join } from 'node:path' + +// MCP SDK is a cli workspace dep; Bun resolves imports from this script's dir, not cli/. +const cliRoot = join(dirname(fileURLToPath(import.meta.url)), '../../cli') +const sdkRoot = join(cliRoot, 'node_modules/@modelcontextprotocol/sdk/dist/esm') +const { Client } = await import(join(sdkRoot, 'client/index.js')) +const { StreamableHTTPClientTransport } = await import(join(sdkRoot, 'client/streamableHttp.js')) const HAPI_HOST = process.env.HAPI_HOST ?? 'http://localhost:3006' const SETTINGS = process.env.HAPI_SETTINGS ?? `${process.env.HOME}/.hapi/settings.json` @@ -21,10 +29,21 @@ const imagePath = process.argv[3] const title = process.argv[4] if (!sessionArg || !imagePath) { - console.error('usage: hapi-display-image.mjs [title]') + console.error('usage: hapi-display-image.mjs [title]') process.exit(2) } +function detectMediaTool(path) { + const head = readFileSync(path).subarray(0, 16) + if (head.length >= 12 && head.subarray(4, 8).toString('ascii') === 'ftyp') { + return 'display_video' + } + if (head.length >= 4 && head[0] === 0x1a && head[1] === 0x45 && head[2] === 0xdf && head[3] === 0xa3) { + return 'display_video' + } + return 'display_image' +} + if (!lstatSync(imagePath).isFile()) { console.error(`not a file: ${imagePath}`) process.exit(2) @@ -57,7 +76,7 @@ if (!session) { process.exit(4) } -// List endpoint omits metadata; per-session GET includes hapiMcpUrl. +// List endpoint omits metadata; per-session GET includes hapiMcpUrl (#956 / PR #958). let mcpUrl = session.metadata?.hapiMcpUrl if (!mcpUrl) { const detailRes = await fetch(`${HAPI_HOST}/api/sessions/${encodeURIComponent(session.id)}`, { @@ -72,17 +91,18 @@ if (!mcpUrl) { mcpUrl = detail.metadata?.hapiMcpUrl } if (!mcpUrl) { - console.error('session has no hapiMcpUrl metadata (restart session CLI after MCP fix lands)') + console.error('session has no hapiMcpUrl metadata (happy MCP not running in that session CLI — check GET /api/sessions/:id)') process.exit(5) } console.error(`hapi-display-image: session=${session.id} mcp=${mcpUrl}`) +const mediaTool = detectMediaTool(imagePath) const client = new Client({ name: 'hapi-display-image', version: '1.0.0' }, { capabilities: {} }) const transport = new StreamableHTTPClientTransport(new URL(mcpUrl)) await client.connect(transport) const result = await client.callTool({ - name: 'display_image', + name: mediaTool, arguments: { path: imagePath, title: title ?? undefined }, }) await client.close() diff --git a/web/src/components/AssistantChat/messages/ToolMessage.tsx b/web/src/components/AssistantChat/messages/ToolMessage.tsx index 919707e10c..efef78ab6d 100644 --- a/web/src/components/AssistantChat/messages/ToolMessage.tsx +++ b/web/src/components/AssistantChat/messages/ToolMessage.tsx @@ -1,4 +1,4 @@ -import { useEffect, useRef, useState, type CSSProperties } from 'react' +import { useEffect, useState } from 'react' import type { ToolCallMessagePartProps } from '@assistant-ui/react' import type { ChatBlock } from '@/chat/types' import type { GeneratedImageBlock, ToolCallBlock } from '@/chat/types' @@ -49,53 +49,24 @@ function isGeneratedImageBlock(value: unknown): value is GeneratedImageBlock { return true } -const MIN_INLINE_IMAGE_DIMENSION = 64 - -function computeTinyImageScale(width: number, height: number): number { - const minDim = Math.min(width, height) - if (minDim <= 0 || minDim >= MIN_INLINE_IMAGE_DIMENSION) { - return 1 - } - return Math.min(MIN_INLINE_IMAGE_DIMENSION / minDim, 16) -} - function GeneratedImageCard(props: { block: GeneratedImageBlock }) { const ctx = useHappyChatContext() const [objectUrl, setObjectUrl] = useState(null) const [error, setError] = useState(null) - const [imageStyle, setImageStyle] = useState(undefined) - const objectUrlRef = useRef(null) - - useEffect(() => { - return () => { - if (objectUrlRef.current) { - URL.revokeObjectURL(objectUrlRef.current) - objectUrlRef.current = null - } - } - }, []) + const isVideo = props.block.mimeType?.startsWith('video/') ?? false + const mediaLabel = isVideo ? 'Generated video' : 'Generated image' useEffect(() => { let disposed = false + let nextObjectUrl: string | null = null + setObjectUrl(null) setError(null) void ctx.api.getGeneratedImageBlob(ctx.sessionId, props.block.imageId) .then((blob) => { if (disposed) return - const nextObjectUrl = URL.createObjectURL(blob) - if (objectUrlRef.current) { - URL.revokeObjectURL(objectUrlRef.current) - } - objectUrlRef.current = nextObjectUrl + nextObjectUrl = URL.createObjectURL(blob) setObjectUrl(nextObjectUrl) - setImageStyle(undefined) - const probe = new Image() - probe.onload = () => { - if (disposed) return - const scale = computeTinyImageScale(probe.naturalWidth, probe.naturalHeight) - setImageStyle(scale === 1 ? undefined : { transform: `scale(${scale})` }) - } - probe.src = nextObjectUrl }) .catch((err: unknown) => { if (disposed) return @@ -104,28 +75,37 @@ function GeneratedImageCard(props: { block: GeneratedImageBlock }) { return () => { disposed = true + if (nextObjectUrl) { + URL.revokeObjectURL(nextObjectUrl) + } } }, [ctx.api, ctx.sessionId, props.block.imageId]) return (
- Generated image · {props.block.fileName} + {mediaLabel} · {props.block.fileName}
{objectUrl ? ( -
- -
+ ) : ( + + ) ) : error ? (
- Generated image is unavailable. {error} + {mediaLabel} is unavailable. {error}
) : (
From 2ef487a8b7ea514dd63e529a9009ea2dd270da53 Mon Sep 17 00:00:00 2001 From: HeavyGee <133152184+heavygee@users.noreply.github.com> Date: Sat, 20 Jun 2026 22:05:03 +0100 Subject: [PATCH 04/13] feat(cli+web): cross-flavor display_video parity with images (#956) Share display_video prompts across MCP-bridge flavors, auto-approve the tool, register mp4/webm via path sniffing, render inline video in web on the existing generated-image RPC path, and restore robust media card fetch. Co-authored-by: Cursor --- cli/src/claude/utils/systemPrompt.ts | 5 +- .../codex/utils/buildHapiMcpBridge.test.ts | 7 +- cli/src/codex/utils/codexMcpConfig.test.ts | 4 + cli/src/codex/utils/systemPrompt.ts | 5 +- cli/src/modules/common/displayImagePrompt.ts | 12 +++ .../modules/common/generatedImages.test.ts | 12 +++ cli/src/modules/common/generatedImages.ts | 4 +- cli/src/modules/common/hapiMcpBridgePrompt.ts | 5 +- .../permission/BasePermissionHandler.ts | 4 +- cli/src/opencode/utils/systemPrompt.ts | 4 +- .../AssistantChat/messages/ToolMessage.tsx | 84 +++++++++++++------ web/src/lib/generatedInlineMedia.test.ts | 16 ++++ web/src/lib/generatedInlineMedia.ts | 7 ++ 13 files changed, 131 insertions(+), 38 deletions(-) create mode 100644 web/src/lib/generatedInlineMedia.test.ts create mode 100644 web/src/lib/generatedInlineMedia.ts diff --git a/cli/src/claude/utils/systemPrompt.ts b/cli/src/claude/utils/systemPrompt.ts index 804001d975..eb0e07aef7 100644 --- a/cli/src/claude/utils/systemPrompt.ts +++ b/cli/src/claude/utils/systemPrompt.ts @@ -1,13 +1,14 @@ import { trimIdent } from "@/utils/trimIdent"; import { shouldIncludeCoAuthoredBy } from "./claudeSettings"; +import { DISPLAY_IMAGE_PROMPT_CLAUDE, DISPLAY_VIDEO_PROMPT_CLAUDE } from "@/modules/common/displayImagePrompt"; /** * Base system prompt shared across all configurations */ const BASE_SYSTEM_PROMPT = (() => trimIdent(` Use the title tool sparingly. For a new chat, call the tool "mcp__hapi__change_title" once after the user's initial request is clear, and set a concise task title. Do not rename the chat for routine progress, substeps, implementation details, or a slightly better wording. Rename only when the user's primary objective changes substantially and the existing title would be misleading. - When you create or find a local image file that the user should see, call the tool "mcp__hapi__display_image" with the image path so HAPI can show it inline. - When you create or find a local mp4 or webm recording the user should see, call the tool "mcp__hapi__display_video" with the file path so HAPI can show it inline. + ${DISPLAY_IMAGE_PROMPT_CLAUDE} + ${DISPLAY_VIDEO_PROMPT_CLAUDE} `))(); /** diff --git a/cli/src/codex/utils/buildHapiMcpBridge.test.ts b/cli/src/codex/utils/buildHapiMcpBridge.test.ts index efd2314a31..164cb88d53 100644 --- a/cli/src/codex/utils/buildHapiMcpBridge.test.ts +++ b/cli/src/codex/utils/buildHapiMcpBridge.test.ts @@ -5,7 +5,7 @@ vi.mock('@/claude/utils/startHappyServer', () => ({ startHappyServer: vi.fn(async () => ({ url: 'http://127.0.0.1:63995/', stop: vi.fn(), - toolNames: ['change_title', 'display_image'] + toolNames: ['change_title', 'display_image', 'display_video'] })) })); @@ -17,13 +17,14 @@ vi.mock('@/utils/spawnHappyCLI', () => ({ })); describe('buildHapiMcpBridge', () => { - it('auto-approves change_title and display_image MCP tools', async () => { + it('auto-approves change_title, display_image, and display_video MCP tools', async () => { const client = {} as never; const bridge = await buildHapiMcpBridge(client); expect(bridge.mcpServers.hapi.tools).toEqual({ change_title: { approval_mode: 'approve' }, - display_image: { approval_mode: 'approve' } + display_image: { approval_mode: 'approve' }, + display_video: { approval_mode: 'approve' } }); expect(bridge.server.url).toBe('http://127.0.0.1:63995/'); }); diff --git a/cli/src/codex/utils/codexMcpConfig.test.ts b/cli/src/codex/utils/codexMcpConfig.test.ts index c0b4597ec0..6ced105183 100644 --- a/cli/src/codex/utils/codexMcpConfig.test.ts +++ b/cli/src/codex/utils/codexMcpConfig.test.ts @@ -34,6 +34,9 @@ describe('codexMcpConfig', () => { }, display_image: { approval_mode: 'approve' as const + }, + display_video: { + approval_mode: 'approve' as const } } } @@ -43,6 +46,7 @@ describe('codexMcpConfig', () => { expect(args).toContain('mcp_servers.hapi.tools.change_title.approval_mode="approve"'); expect(args).toContain('mcp_servers.hapi.tools.display_image.approval_mode="approve"'); + expect(args).toContain('mcp_servers.hapi.tools.display_video.approval_mode="approve"'); }); it('builds config args for multiple MCP servers', () => { diff --git a/cli/src/codex/utils/systemPrompt.ts b/cli/src/codex/utils/systemPrompt.ts index 3f58b216a7..82e3da28f3 100644 --- a/cli/src/codex/utils/systemPrompt.ts +++ b/cli/src/codex/utils/systemPrompt.ts @@ -6,6 +6,7 @@ */ import { trimIdent } from '@/utils/trimIdent'; +import { DISPLAY_IMAGE_PROMPT_CODEX, DISPLAY_VIDEO_PROMPT_CODEX } from '@/modules/common/displayImagePrompt'; /** * Title instruction for Codex to call the hapi MCP tool. @@ -18,8 +19,8 @@ export const TITLE_INSTRUCTION = trimIdent(` If that exact tool name is unavailable, call an equivalent alias such as hapi__change_title, mcp__hapi__change_title, or hapi_change_title. Do not rename the chat for routine progress, substeps, implementation details, or a slightly better wording. Rename only when the user's primary objective changes substantially and the existing title would be misleading. - When you create or find a local image file that the user should see, call functions.hapi__display_image with the image path. If that exact tool name is unavailable, use an equivalent alias such as hapi__display_image, mcp__hapi__display_image, or hapi_display_image. - When you create or find a local mp4 or webm file the user should see, call functions.hapi__display_video with the file path. If unavailable, use hapi__display_video, mcp__hapi__display_video, or hapi_display_video. + ${DISPLAY_IMAGE_PROMPT_CODEX} + ${DISPLAY_VIDEO_PROMPT_CODEX} `); /** diff --git a/cli/src/modules/common/displayImagePrompt.ts b/cli/src/modules/common/displayImagePrompt.ts index 507c0903aa..d3953eff1b 100644 --- a/cli/src/modules/common/displayImagePrompt.ts +++ b/cli/src/modules/common/displayImagePrompt.ts @@ -15,3 +15,15 @@ export const DISPLAY_IMAGE_PROMPT_CODEX = trimIdent(` export const DISPLAY_IMAGE_PROMPT_HAPI_MCP = trimIdent(` When you create or find a local image file that the user should see, call the tool "hapi_display_image" with the image path so HAPI can show it inline. `); + +export const DISPLAY_VIDEO_PROMPT_CLAUDE = trimIdent(` + When you create or find a local mp4 or webm recording the user should see, call the tool "mcp__hapi__display_video" with the file path so HAPI can show it inline. +`); + +export const DISPLAY_VIDEO_PROMPT_CODEX = trimIdent(` + When you create or find a local mp4 or webm file the user should see, call functions.hapi__display_video with the file path. If that exact tool name is unavailable, use an equivalent alias such as hapi__display_video, mcp__hapi__display_video, or hapi_display_video. +`); + +export const DISPLAY_VIDEO_PROMPT_HAPI_MCP = trimIdent(` + When you create or find a local mp4 or webm recording the user should see, call the tool "hapi_display_video" with the file path so HAPI can show it inline. +`); diff --git a/cli/src/modules/common/generatedImages.test.ts b/cli/src/modules/common/generatedImages.test.ts index 3da6492e0f..427fb32af8 100644 --- a/cli/src/modules/common/generatedImages.test.ts +++ b/cli/src/modules/common/generatedImages.test.ts @@ -101,4 +101,16 @@ describe('generatedImages', () => { clearGeneratedImages() }) + it('registers mp4 from local file paths after MIME sniffing', async () => { + const dir = join(tmpdir(), `hapi-inline-mp4-${Date.now()}`) + mkdirSync(dir, { recursive: true }) + const path = join(dir, 'inline.mp4') + const bytes = Buffer.from([0x00, 0x00, 0x00, 0x18, 0x66, 0x74, 0x79, 0x70, 0x69, 0x73, 0x6f, 0x6d]) + writeFileSync(path, bytes) + + const video = await registerGeneratedImageFromPath({ path }) + expect(video?.mimeType).toBe('video/mp4') + clearGeneratedImages() + }) + }) diff --git a/cli/src/modules/common/generatedImages.ts b/cli/src/modules/common/generatedImages.ts index f1462a8f0e..247f4608bd 100644 --- a/cli/src/modules/common/generatedImages.ts +++ b/cli/src/modules/common/generatedImages.ts @@ -152,9 +152,9 @@ export async function registerGeneratedImageFromPath(args: { throw new Error('Image is too large to display inline') } const bytes = await readFile(args.path) - const mimeType = detectImageMimeType(bytes) + const mimeType = detectImageMimeType(bytes) ?? detectVideoMimeType(bytes) if (!mimeType) { - throw new Error('Unsupported image content') + throw new Error('Unsupported inline media content') } return registerGeneratedImage({ id: args.id ?? randomUUID(), diff --git a/cli/src/modules/common/hapiMcpBridgePrompt.ts b/cli/src/modules/common/hapiMcpBridgePrompt.ts index 74bf4b029b..92f2af8697 100644 --- a/cli/src/modules/common/hapiMcpBridgePrompt.ts +++ b/cli/src/modules/common/hapiMcpBridgePrompt.ts @@ -1,8 +1,8 @@ import { trimIdent } from '@/utils/trimIdent'; -import { DISPLAY_IMAGE_PROMPT_HAPI_MCP } from './displayImagePrompt'; +import { DISPLAY_IMAGE_PROMPT_HAPI_MCP, DISPLAY_VIDEO_PROMPT_HAPI_MCP } from './displayImagePrompt'; /** - * Title + display_image instructions for ACP flavors wired through buildHapiMcpBridge + * Title + display_image / display_video instructions for ACP flavors wired through buildHapiMcpBridge * (Gemini, Kimi, Cursor, OpenCode). Prepended on the first user prompt. */ export const HAPI_MCP_TITLE_INSTRUCTION = trimIdent(` @@ -12,4 +12,5 @@ export const HAPI_MCP_TITLE_INSTRUCTION = trimIdent(` export const HAPI_MCP_BRIDGE_PROMPT = trimIdent(` ${HAPI_MCP_TITLE_INSTRUCTION} ${DISPLAY_IMAGE_PROMPT_HAPI_MCP} + ${DISPLAY_VIDEO_PROMPT_HAPI_MCP} `); diff --git a/cli/src/modules/common/permission/BasePermissionHandler.ts b/cli/src/modules/common/permission/BasePermissionHandler.ts index b68dc79235..e03908405e 100644 --- a/cli/src/modules/common/permission/BasePermissionHandler.ts +++ b/cli/src/modules/common/permission/BasePermissionHandler.ts @@ -20,15 +20,17 @@ export type AutoApprovalRuleSet = { const AUTO_APPROVE_TOOL_NAME_HINTS = [ 'change_title', 'display_image', + 'display_video', 'happy__change_title', 'hapi_change_title', // OpenCode MCP tool pattern 'hapi_display_image', + 'hapi_display_video', 'geminireasoning', 'codexreasoning', 'think', 'save_memory' ]; -const AUTO_APPROVE_TOOL_ID_HINTS = ['change_title', 'display_image', 'save_memory']; +const AUTO_APPROVE_TOOL_ID_HINTS = ['change_title', 'display_image', 'display_video', 'save_memory']; const AUTO_APPROVE_WRITE_TOOL_HINTS = ['write', 'edit', 'create', 'delete', 'patch', 'fs-edit']; export function resolveToolAutoApprovalDecision( diff --git a/cli/src/opencode/utils/systemPrompt.ts b/cli/src/opencode/utils/systemPrompt.ts index 9a2e556b26..feb6d02d29 100644 --- a/cli/src/opencode/utils/systemPrompt.ts +++ b/cli/src/opencode/utils/systemPrompt.ts @@ -1,8 +1,8 @@ /** - * OpenCode-specific system prompt for hapi MCP tools (change_title, display_image). + * OpenCode-specific system prompt for hapi MCP tools (change_title, display_image, display_video). * * OpenCode exposes MCP tools with the naming pattern: _ - * The hapi MCP server exposes `change_title` and `display_image`. + * The hapi MCP server exposes `change_title`, `display_image`, and `display_video`. */ import { trimIdent } from '@/utils/trimIdent'; diff --git a/web/src/components/AssistantChat/messages/ToolMessage.tsx b/web/src/components/AssistantChat/messages/ToolMessage.tsx index efef78ab6d..bc0d7b5489 100644 --- a/web/src/components/AssistantChat/messages/ToolMessage.tsx +++ b/web/src/components/AssistantChat/messages/ToolMessage.tsx @@ -1,4 +1,4 @@ -import { useEffect, useState } from 'react' +import { useEffect, useRef, useState, type CSSProperties } from 'react' import type { ToolCallMessagePartProps } from '@assistant-ui/react' import type { ChatBlock } from '@/chat/types' import type { GeneratedImageBlock, ToolCallBlock } from '@/chat/types' @@ -15,6 +15,7 @@ import { useHappyChatContext } from '@/components/AssistantChat/context' import { CliOutputBlock } from '@/components/CliOutputBlock' import { UserBubbleContent, getUserBubbleClassName, shouldShowMessageStatus } from '@/components/AssistantChat/messages/user-bubble' import { ImagePreview } from '@/components/ImagePreview' +import { generatedInlineMediaLabel, isInlineVideoMimeType } from '@/lib/generatedInlineMedia' function isToolCallBlock(value: unknown): value is ToolCallBlock { if (!isObject(value)) return false @@ -49,37 +50,67 @@ function isGeneratedImageBlock(value: unknown): value is GeneratedImageBlock { return true } +const MIN_INLINE_IMAGE_DIMENSION = 64 + +function computeTinyImageScale(width: number, height: number): number { + const minDim = Math.min(width, height) + if (minDim <= 0 || minDim >= MIN_INLINE_IMAGE_DIMENSION) { + return 1 + } + return Math.min(MIN_INLINE_IMAGE_DIMENSION / minDim, 16) +} + function GeneratedImageCard(props: { block: GeneratedImageBlock }) { const ctx = useHappyChatContext() const [objectUrl, setObjectUrl] = useState(null) const [error, setError] = useState(null) - const isVideo = props.block.mimeType?.startsWith('video/') ?? false - const mediaLabel = isVideo ? 'Generated video' : 'Generated image' + const [imageStyle, setImageStyle] = useState(undefined) + const objectUrlRef = useRef(null) + const isVideo = isInlineVideoMimeType(props.block.mimeType) + const mediaLabel = generatedInlineMediaLabel(props.block.mimeType) + + useEffect(() => { + return () => { + if (objectUrlRef.current) { + URL.revokeObjectURL(objectUrlRef.current) + objectUrlRef.current = null + } + } + }, []) useEffect(() => { let disposed = false - let nextObjectUrl: string | null = null - setObjectUrl(null) setError(null) void ctx.api.getGeneratedImageBlob(ctx.sessionId, props.block.imageId) .then((blob) => { if (disposed) return - nextObjectUrl = URL.createObjectURL(blob) + const nextObjectUrl = URL.createObjectURL(blob) + if (objectUrlRef.current) { + URL.revokeObjectURL(objectUrlRef.current) + } + objectUrlRef.current = nextObjectUrl setObjectUrl(nextObjectUrl) + if (!isVideo) { + setImageStyle(undefined) + const probe = new Image() + probe.onload = () => { + if (disposed) return + const scale = computeTinyImageScale(probe.naturalWidth, probe.naturalHeight) + setImageStyle(scale === 1 ? undefined : { transform: `scale(${scale})` }) + } + probe.src = nextObjectUrl + } }) .catch((err: unknown) => { if (disposed) return - setError(err instanceof Error ? err.message : 'Failed to load generated image') + setError(err instanceof Error ? err.message : 'Failed to load inline media') }) return () => { disposed = true - if (nextObjectUrl) { - URL.revokeObjectURL(nextObjectUrl) - } } - }, [ctx.api, ctx.sessionId, props.block.imageId]) + }, [ctx.api, ctx.sessionId, props.block.imageId, isVideo]) return (
@@ -88,20 +119,25 @@ function GeneratedImageCard(props: { block: GeneratedImageBlock }) {
{objectUrl ? ( isVideo ? ( -