-
Notifications
You must be signed in to change notification settings - Fork 0
feat(hub+cli+web): mermaid parse-failure feedback — detect broken diagrams, notify agent, show error placeholder #71
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
85363fd
3817efe
97651ea
4619cf5
d4e6227
4c518ee
914788c
8e7c4f5
202bef4
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,193 @@ | ||
| import { describe, expect, it } from 'bun:test' | ||
| import { extractFailingMermaidBlocks, buildMermaidRenderIssueHint } from './mermaid' | ||
|
|
||
| // Bare { type:'assistant', message:... } envelope (e.g. test / older formats) | ||
| function assistantMessage(text: string): unknown { | ||
| return { | ||
| type: 'assistant', | ||
| message: { | ||
| role: 'assistant', | ||
| content: [{ type: 'text', text }] | ||
| } | ||
| } | ||
| } | ||
|
|
||
| // role:'agent' + type:'output' — the real live-session format for Claude | ||
| function agentOutputMessage(text: string): unknown { | ||
| return { | ||
| role: 'agent', | ||
| content: { | ||
| type: 'output', | ||
| data: { | ||
| type: 'assistant', | ||
| message: { | ||
| role: 'assistant', | ||
| content: [{ type: 'text', text }] | ||
| } | ||
| } | ||
| }, | ||
| meta: { sentFrom: 'cli' } | ||
| } | ||
| } | ||
|
|
||
| // role:'agent' + type:'codex' — Codex/Gemini/OpenCode format | ||
| function agentCodexMessage(text: string): unknown { | ||
| return { | ||
| role: 'agent', | ||
| content: { | ||
| type: 'codex', | ||
| data: { | ||
| type: 'message', | ||
| message: text | ||
| } | ||
| }, | ||
| meta: { sentFrom: 'cli' } | ||
| } | ||
| } | ||
|
|
||
| describe('extractFailingMermaidBlocks', () => { | ||
| it('returns [] for non-assistant messages', () => { | ||
| expect(extractFailingMermaidBlocks({ type: 'user', message: { role: 'user', content: 'hello' } })).toEqual([]) | ||
| expect(extractFailingMermaidBlocks(null)).toEqual([]) | ||
| expect(extractFailingMermaidBlocks('plain string')).toEqual([]) | ||
| }) | ||
|
|
||
| it('returns an issue for invalid mermaid in live-session agent output (Claude format)', () => { | ||
| const msg = agentOutputMessage('```mermaid\ngrph TD\n A --> B\n```') | ||
| const issues = extractFailingMermaidBlocks(msg) | ||
| expect(issues).toHaveLength(1) | ||
| expect(issues[0].snippet).toContain('grph TD') | ||
| }) | ||
|
|
||
| it('returns [] for valid mermaid in live-session agent output (Claude format)', () => { | ||
| const msg = agentOutputMessage('```mermaid\ngraph TD\n A --> B\n```') | ||
| expect(extractFailingMermaidBlocks(msg)).toEqual([]) | ||
| }) | ||
|
|
||
| it('returns an issue for invalid mermaid in codex agent format', () => { | ||
| const msg = agentCodexMessage('```mermaid\ngrph TD\n A --> B\n```') | ||
| const issues = extractFailingMermaidBlocks(msg) | ||
| expect(issues).toHaveLength(1) | ||
| expect(issues[0].snippet).toContain('grph TD') | ||
| }) | ||
|
|
||
| it('returns [] for valid mermaid in codex agent format', () => { | ||
| const msg = agentCodexMessage('```mermaid\nflowchart LR\n A --> B\n```') | ||
| expect(extractFailingMermaidBlocks(msg)).toEqual([]) | ||
| }) | ||
|
|
||
| it('returns [] when no mermaid blocks present', () => { | ||
| expect(extractFailingMermaidBlocks(assistantMessage('Here is some text with no diagrams.'))).toEqual([]) | ||
| }) | ||
|
|
||
| it('returns [] for valid diagram types', () => { | ||
| const msg = assistantMessage('```mermaid\ngraph TD\n A --> B\n```') | ||
| expect(extractFailingMermaidBlocks(msg)).toEqual([]) | ||
| }) | ||
|
|
||
| it('returns [] for valid type with extra whitespace after type name', () => { | ||
| const msg = assistantMessage('```mermaid\nflowchart LR\n A --> B\n```') | ||
| expect(extractFailingMermaidBlocks(msg)).toEqual([]) | ||
| }) | ||
|
|
||
| it('returns an issue for an unknown diagram type', () => { | ||
| const msg = assistantMessage('```mermaid\ngrph TD\n A --> B\n```') | ||
| const issues = extractFailingMermaidBlocks(msg) | ||
| expect(issues).toHaveLength(1) | ||
| expect(issues[0].snippet).toContain('grph TD') | ||
| }) | ||
|
|
||
| it('is case-insensitive for known types', () => { | ||
| const msg = assistantMessage('```mermaid\nFLOWCHART TD\n A --> B\n```') | ||
| expect(extractFailingMermaidBlocks(msg)).toEqual([]) | ||
| }) | ||
|
|
||
| it('skips %%{init} config directives when determining diagram type', () => { | ||
| const text = '```mermaid\n%%{init: {"theme": "base"}}%%\ngraph TD\n A --> B\n```' | ||
| expect(extractFailingMermaidBlocks(assistantMessage(text))).toEqual([]) | ||
| }) | ||
|
|
||
| it('skips YAML frontmatter when determining diagram type', () => { | ||
| const text = '```mermaid\n---\ntitle: My diagram\n---\ngraph TD\n A --> B\n```' | ||
| expect(extractFailingMermaidBlocks(assistantMessage(text))).toEqual([]) | ||
| }) | ||
|
|
||
| it('returns [] for mermaid 11.12+ diagram types (radar-beta, treemap, treeview-beta, venn-beta, wardley-beta, ishikawa)', () => { | ||
| for (const type of ['radar-beta', 'treemap', 'treeview-beta', 'venn-beta', 'wardley-beta', 'ishikawa']) { | ||
| const msg = assistantMessage(`\`\`\`mermaid\n${type}\n A --> B\n\`\`\``) | ||
| expect(extractFailingMermaidBlocks(msg), `should accept ${type}`).toEqual([]) | ||
| } | ||
| }) | ||
|
|
||
| it('detects an empty block as invalid', () => { | ||
| const msg = assistantMessage('```mermaid\n\n```') | ||
| const issues = extractFailingMermaidBlocks(msg) | ||
| expect(issues).toHaveLength(1) | ||
| }) | ||
|
|
||
| it('returns one issue per failing block, skips valid ones', () => { | ||
| const text = [ | ||
| '```mermaid', | ||
| 'graph TD', | ||
| ' A --> B', | ||
| '```', | ||
| '', | ||
| 'Some text', | ||
| '', | ||
| '```mermaid', | ||
| 'grph TD', | ||
| ' A --> B', | ||
| '```', | ||
| ].join('\n') | ||
| const issues = extractFailingMermaidBlocks(assistantMessage(text)) | ||
| expect(issues).toHaveLength(1) | ||
| expect(issues[0].snippet).toContain('grph TD') | ||
| }) | ||
|
|
||
| it('returns multiple issues when multiple blocks are invalid', () => { | ||
| const text = [ | ||
| '```mermaid', | ||
| 'grph TD', | ||
| ' A --> B', | ||
| '```', | ||
| '', | ||
| '```mermaid', | ||
| 'sequenceDagram', | ||
| ' A->>B: hello', | ||
| '```', | ||
| ].join('\n') | ||
| const issues = extractFailingMermaidBlocks(assistantMessage(text)) | ||
| expect(issues).toHaveLength(2) | ||
| }) | ||
|
|
||
| it('truncates snippet to 500 chars', () => { | ||
| const longCode = 'unknowntype\n' + 'A --> B\n'.repeat(100) | ||
| const msg = assistantMessage('```mermaid\n' + longCode + '\n```') | ||
| const issues = extractFailingMermaidBlocks(msg) | ||
| expect(issues).toHaveLength(1) | ||
| expect(issues[0].snippet.length).toBeLessThanOrEqual(500) | ||
| }) | ||
| }) | ||
|
|
||
| describe('buildMermaidRenderIssueHint', () => { | ||
| it('wraps a single issue in render-issue tags', () => { | ||
| const hint = buildMermaidRenderIssueHint([{ snippet: 'grph TD\n A --> B' }]) | ||
| expect(hint).toMatch(/^<render-issue>/) | ||
| expect(hint).toMatch(/<\/render-issue>$/) | ||
| expect(hint).toContain('grph TD') | ||
| }) | ||
|
|
||
| it('labels blocks when multiple issues', () => { | ||
| const hint = buildMermaidRenderIssueHint([ | ||
| { snippet: 'grph TD\n A --> B' }, | ||
| { snippet: 'sequenceDagram\n A->>B: hi' } | ||
| ]) | ||
| expect(hint).toContain('Block 1:') | ||
| expect(hint).toContain('Block 2:') | ||
| }) | ||
|
|
||
| it('does not label block when only one issue', () => { | ||
| const hint = buildMermaidRenderIssueHint([{ snippet: 'grph TD' }]) | ||
| expect(hint).not.toContain('Block 1:') | ||
| }) | ||
| }) |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,142 @@ | ||
| import { isObject } from '@hapi/protocol' | ||
| import { unwrapRoleWrappedRecordEnvelope } from '@hapi/protocol/messages' | ||
|
|
||
| // Diagram types recognised by mermaid v11 (up to 11.14). Checked | ||
| // case-insensitively against the first non-blank, non-directive token. | ||
| const KNOWN_DIAGRAM_TYPES = new Set([ | ||
| 'flowchart', | ||
| 'graph', | ||
| 'sequencediagram', | ||
| 'classdiagram', | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Mermaid accepts Useful? React with 👍 / 👎. |
||
| 'classdiagram-v2', | ||
| 'statediagram', | ||
| 'statediagram-v2', | ||
| 'erdiagram', | ||
| 'gantt', | ||
| 'journey', | ||
| 'gitgraph', | ||
| 'pie', | ||
| 'quadrantchart', | ||
| 'requirementdiagram', | ||
| 'mindmap', | ||
| 'timeline', | ||
| 'sankey-beta', | ||
| 'xychart-beta', | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Mermaid 11.14 accepts Useful? React with 👍 / 👎. |
||
| 'block-beta', | ||
| 'packet-beta', | ||
| 'architecture-beta', | ||
| 'c4context', | ||
| 'c4container', | ||
| 'c4component', | ||
| 'c4dynamic', | ||
| 'c4deployment', | ||
| 'zenuml', | ||
| 'kanban', | ||
| // Added in 11.12–11.14 | ||
| 'radar-beta', | ||
| 'treemap', | ||
| 'treeview-beta', | ||
| 'venn-beta', | ||
| 'wardley-beta', | ||
| 'ishikawa', | ||
| ]) | ||
|
|
||
| export interface MermaidIssue { | ||
| snippet: string | ||
| } | ||
|
|
||
| function extractTextFromContent(content: unknown): string | null { | ||
| if (typeof content === 'string') return content | ||
| if (!Array.isArray(content)) return null | ||
| const parts: string[] = [] | ||
| for (const block of content) { | ||
| if (isObject(block) && block.type === 'text' && typeof block.text === 'string') { | ||
| parts.push(block.text) | ||
| } | ||
| } | ||
| return parts.length > 0 ? parts.join('\n') : null | ||
| } | ||
|
|
||
| function getDiagramType(code: string): string { | ||
| let inFrontmatter = false | ||
| for (const line of code.split('\n')) { | ||
| const trimmed = line.trim() | ||
| if (!trimmed) continue | ||
| if (trimmed === '---') { | ||
| inFrontmatter = !inFrontmatter | ||
| continue | ||
| } | ||
| if (inFrontmatter || trimmed.startsWith('%%')) continue | ||
| return trimmed.split(/\s+/)[0].toLowerCase() | ||
| } | ||
| return '' | ||
| } | ||
|
|
||
| function findInvalidMermaidBlocks(markdown: string): MermaidIssue[] { | ||
| const issues: MermaidIssue[] = [] | ||
| // Match ```mermaid fences; multiline, anchored to line start | ||
| const fenceRegex = /^```mermaid[ \t]*\r?\n([\s\S]*?)^```/gm | ||
| let match: RegExpExecArray | null | ||
| while ((match = fenceRegex.exec(markdown)) !== null) { | ||
| const code = match[1] | ||
| const diagramType = getDiagramType(code) | ||
| if (!diagramType || !KNOWN_DIAGRAM_TYPES.has(diagramType)) { | ||
| issues.push({ snippet: code.slice(0, 500) }) | ||
| } | ||
| } | ||
| return issues | ||
| } | ||
|
|
||
| function extractTextFromRecord(record: { role: string; content: unknown }): string | null { | ||
| if (record.role === 'assistant') { | ||
| return extractTextFromContent(record.content) | ||
| } | ||
| if (record.role === 'agent' && isObject(record.content)) { | ||
| if (record.content.type === 'output' && isObject(record.content.data)) { | ||
| // Claude: data is { type: 'assistant', message: { role: 'assistant', content: [...] } } | ||
| const inner = unwrapRoleWrappedRecordEnvelope(record.content.data) | ||
| if (inner?.role === 'assistant') return extractTextFromContent(inner.content) | ||
| } else if (record.content.type === 'codex' && isObject(record.content.data)) { | ||
| // Codex/Gemini/OpenCode: data is { type: 'message', message: <string> } | ||
| if (record.content.data.type === 'message' && typeof record.content.data.message === 'string') { | ||
| return record.content.data.message | ||
| } | ||
| } | ||
| } | ||
| return null | ||
| } | ||
|
|
||
| /** | ||
| * Inspects a message content blob (as stored in the hub DB) and returns one | ||
| * entry per ```mermaid block whose first token is not a recognised diagram | ||
| * type. Handles both the bare role:'assistant' format and the live-session | ||
| * role:'agent' wrapper (type:'output' for Claude, type:'codex' for Codex/Gemini). | ||
| */ | ||
| export function extractFailingMermaidBlocks(messageContent: unknown): MermaidIssue[] { | ||
| const record = unwrapRoleWrappedRecordEnvelope(messageContent) | ||
| if (!record) return [] | ||
|
|
||
| const text = extractTextFromRecord(record) | ||
| if (!text) return [] | ||
|
|
||
| return findInvalidMermaidBlocks(text) | ||
| } | ||
|
|
||
| export function buildMermaidRenderIssueHint(issues: MermaidIssue[]): string { | ||
| const blocks = issues | ||
| .map((issue, i) => { | ||
| const label = issues.length > 1 ? `Block ${i + 1}:\n` : '' | ||
| return `${label}\`\`\`\n${issue.snippet.trim()}\n\`\`\`` | ||
| }) | ||
| .join('\n\n') | ||
| const noun = issues.length === 1 ? 'block' : 'blocks' | ||
| return ( | ||
| `<render-issue>\n` + | ||
| `A mermaid ${noun} in your previous response failed to render — ` + | ||
| `the diagram type was not recognised. The user sees an error placeholder, not a diagram.\n\n` + | ||
| `${blocks}\n\n` + | ||
| `You MUST re-emit the corrected diagram now. Do not explain, apologise, or comment — ` + | ||
| `just output the fixed mermaid block with a valid diagram type. This is required.\n` + | ||
| `</render-issue>` | ||
| ) | ||
| } | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
In Claude sessions where the echoed user turn is serialized as text blocks, this prefix only makes
isExternalUserMessagewrap the echo asrole:'agent'/content.type:'output'; the web normalizer still maps non-sidechaindata.type === 'user'messages with all-text array content back torole:'user'(web/src/chat/normalizeAgent.ts:349-364). That means the internal<render-issue>prompt and diagram snippet can appear in the visible transcript even though it is meant to be a system injection; suppress this prefix in the web normalizer or avoid storing the echo.Useful? React with 👍 / 👎.