From 7afce3dc7c88526e2c98fefd327f927a3b86322f Mon Sep 17 00:00:00 2001 From: Mister Nemo Date: Wed, 3 Jun 2026 23:14:12 -0400 Subject: [PATCH 1/2] Preserve inherited prosodic nod keyframes --- .../__tests__/snippetPreloader.test.ts | 9 +++ .../snippets/speaking/headNodBig.json | 8 ++- .../snippets/speaking/headNodSmall.json | 8 ++- .../__tests__/prosodicService.test.ts | 71 +++++++++++++++++++ src/prosodic/prosodicMachine.ts | 3 +- src/prosodic/types.ts | 1 + 6 files changed, 97 insertions(+), 3 deletions(-) create mode 100644 src/prosodic/__tests__/prosodicService.test.ts diff --git a/src/animation/__tests__/snippetPreloader.test.ts b/src/animation/__tests__/snippetPreloader.test.ts index fa56fb2..e688338 100644 --- a/src/animation/__tests__/snippetPreloader.test.ts +++ b/src/animation/__tests__/snippetPreloader.test.ts @@ -66,6 +66,15 @@ describe('snippetPreloader', () => { expect(storage.removeItem).not.toHaveBeenCalled(); }); + it('bundles head nod snippets with inherited head-pitch anchors', async () => { + const resolved = await resolveSnippetEntry('speakingAnimationsList', 'headNodSmall'); + + expect(resolved?.source).toBe('bundled'); + expect(resolved?.data.curves['54'][0]).toEqual({ time: 0, intensity: 0, inherit: true }); + expect(resolved?.data.curves['53'][0]).toEqual({ time: 0, intensity: 0, inherit: true }); + expect(resolved?.data.curves['53'].at(-1)?.intensity).toBe(0); + }); + it('ignores legacy preloaded bundle entries when reading custom snippet names', () => { const storage = createStorageMock({ bundledAnimationSnippetsManifest: JSON.stringify({ diff --git a/src/animation/snippets/speaking/headNodBig.json b/src/animation/snippets/speaking/headNodBig.json index 344806a..689f23c 100644 --- a/src/animation/snippets/speaking/headNodBig.json +++ b/src/animation/snippets/speaking/headNodBig.json @@ -3,7 +3,8 @@ "54": [ { "time": 0.0, - "intensity": 0.0 + "intensity": 0.0, + "inherit": true }, { "time": 0.4, @@ -15,6 +16,11 @@ } ], "53": [ + { + "time": 0.0, + "intensity": 0.0, + "inherit": true + }, { "time": 0.8, "intensity": 0.0 diff --git a/src/animation/snippets/speaking/headNodSmall.json b/src/animation/snippets/speaking/headNodSmall.json index 9fc8925..99a857e 100644 --- a/src/animation/snippets/speaking/headNodSmall.json +++ b/src/animation/snippets/speaking/headNodSmall.json @@ -3,7 +3,8 @@ "54": [ { "time": 0.0, - "intensity": 0.0 + "intensity": 0.0, + "inherit": true }, { "time": 0.15, @@ -15,6 +16,11 @@ } ], "53": [ + { + "time": 0.0, + "intensity": 0.0, + "inherit": true + }, { "time": 0.3, "intensity": 0.0 diff --git a/src/prosodic/__tests__/prosodicService.test.ts b/src/prosodic/__tests__/prosodicService.test.ts new file mode 100644 index 0000000..74704f9 --- /dev/null +++ b/src/prosodic/__tests__/prosodicService.test.ts @@ -0,0 +1,71 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { createProsodicService, type ProsodicServiceAPI } from '../prosodicService'; + +function createStorageMock(seed: Record = {}) { + const data = { ...seed }; + + return { + data, + getItem: vi.fn((key: string) => data[key] ?? null), + setItem: vi.fn((key: string, value: string) => { + data[key] = value; + }), + removeItem: vi.fn((key: string) => { + delete data[key]; + }), + }; +} + +describe('createProsodicService', () => { + let warnSpy: ReturnType; + let service: ProsodicServiceAPI | null; + + beforeEach(() => { + warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}); + vi.stubGlobal('requestAnimationFrame', vi.fn(() => 1)); + vi.stubGlobal('cancelAnimationFrame', vi.fn()); + service = null; + }); + + afterEach(() => { + service?.dispose(); + vi.unstubAllGlobals(); + warnSpy.mockRestore(); + }); + + it('preserves inherited keyframes when scheduling head nod snippets', () => { + const storage = createStorageMock({ + 'speakingAnimationsList/headNodSmall': JSON.stringify({ + name: 'headNodSmall', + curves: { + '53': [ + { time: 0, intensity: 0, inherit: true }, + { time: 0.15, intensity: 0.4 }, + { time: 0.4, intensity: 0 }, + ], + }, + }), + }); + const scheduled: any[] = []; + + vi.stubGlobal('localStorage', storage); + vi.stubGlobal('window', { localStorage: storage }); + + service = createProsodicService( + { headLoopKey: 'speakingAnimationsList/headNodSmall' }, + {}, + { + scheduleSnippet: (snippet) => { + scheduled.push(snippet); + return snippet.name; + }, + removeSnippet: vi.fn(), + }, + ); + + service.startTalking(); + + expect(scheduled).toHaveLength(1); + expect(scheduled[0].curves['53'][0]).toEqual({ time: 0, intensity: 0, inherit: true }); + }); +}); diff --git a/src/prosodic/prosodicMachine.ts b/src/prosodic/prosodicMachine.ts index 5b26182..3a5ac38 100644 --- a/src/prosodic/prosodicMachine.ts +++ b/src/prosodic/prosodicMachine.ts @@ -10,7 +10,7 @@ import { createMachine, assign } from 'xstate'; export interface ProsodicSnippet { name: string; - curves: Record>; + curves: Record>; category: 'brow' | 'head'; priority: number; intensityScale: number; @@ -57,6 +57,7 @@ function normalizeSnippet(data: any, category: 'brow' | 'head', priority: number curves[key] = arr.map((k: any) => ({ time: k.time ?? k.t ?? 0, intensity: k.intensity ?? k.v ?? 0, + ...(typeof k.inherit === 'boolean' ? { inherit: k.inherit } : {}), })); curves[key].sort((a, b) => a.time - b.time); } diff --git a/src/prosodic/types.ts b/src/prosodic/types.ts index f5f9373..cf3d1cf 100644 --- a/src/prosodic/types.ts +++ b/src/prosodic/types.ts @@ -19,6 +19,7 @@ export interface AnimationSnippet { export interface AnimationCurve { time: number; intensity: number; + inherit?: boolean; } export type ProsodicChannel = 'brow' | 'head' | 'both'; From c5277c7303c94faede39c0ff6efc70f845f104bc Mon Sep 17 00:00:00 2001 From: Mister Nemo Date: Thu, 4 Jun 2026 09:23:41 -0400 Subject: [PATCH 2/2] Trigger prosodic agency from vocal speech timing --- AGENTS.md | 14 ++++++++++++++ src/tts/ttsService.ts | 3 +++ src/tts/types.ts | 4 ++++ src/vocal/__tests__/service.test.ts | 28 ++++++++++++++++++++++++++++ src/vocal/index.ts | 1 + src/vocal/service.ts | 5 +++++ src/vocal/snippetBuilder.ts | 2 +- src/vocal/types.ts | 13 ++++++++++++- 8 files changed, 68 insertions(+), 2 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index a5109ef..862a0c8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -19,6 +19,20 @@ The desired architecture is data-first: - runtime outputs become observable streams at the JS boundary - LoomLarge bridges those streams into Effect-managed state +## Agency Ownership Boundaries + +When a change concerns character behavior, first identify the Polyester agency +that owns that behavior. Do not move agency behavior into LoomLarge call sites +because a LoomLarge module exposes the symptom. + +In particular, speech-time prosody is owned by Polyester: + +- speech/vocal timing code triggers the prosodic expression agency +- `src/prosodic/` owns prosodic state, pulse handling, scheduling, and fade-out +- bundled head-nod snippets live under `src/animation/snippets/speaking/` +- LoomLarge should configure or pass agencies, not author prosodic keyframes or + schedule head-nod snippets directly + ## Documentation Discipline When implementation work exposes a recurring misunderstanding, missing diff --git a/src/tts/ttsService.ts b/src/tts/ttsService.ts index e840a29..89ff373 100644 --- a/src/tts/ttsService.ts +++ b/src/tts/ttsService.ts @@ -103,6 +103,7 @@ export class TTSService { lipsyncIntensity: config.lipsyncIntensity ?? 1.0, jawScale: config.jawScale ?? 1.0, animationAgency: config.animationAgency ?? undefined, + prosodicService: config.prosodicService ?? undefined, }; this.callbacks = callbacks; @@ -146,6 +147,7 @@ export class TTSService { speechRate: this.config.rate, jawScale: this.config.jawScale, animationAgency: this.config.animationAgency, + prosodicService: this.config.prosodicService, }); } @@ -960,6 +962,7 @@ export class TTSService { speechRate: this.config.rate, intensity: this.config.lipsyncIntensity, jawScale: this.config.jawScale, + prosodicService: this.config.prosodicService, }); } } diff --git a/src/tts/types.ts b/src/tts/types.ts index 90ff521..7ffdab0 100644 --- a/src/tts/types.ts +++ b/src/tts/types.ts @@ -3,6 +3,8 @@ * Type definitions for Text-to-Speech functionality */ +import type { ProsodicExpressionAgency } from '../vocal'; + export type TTSEngine = 'webSpeech' | 'sapi' | 'azure'; export type WebSpeechReferenceMode = 'none' | 'displayMedia'; export type PlaybackReferenceStatus = @@ -44,6 +46,8 @@ export interface TTSConfig { setSnippetTime?: (name: string, timeSec: number) => void; seek?: (name: string, timeSec: number) => void; }; + /** Prosodic expression agency triggered by speech lifecycle and word boundaries. */ + prosodicService?: ProsodicExpressionAgency; } export interface TTSVoice { diff --git a/src/vocal/__tests__/service.test.ts b/src/vocal/__tests__/service.test.ts index 9d1b995..d8da901 100644 --- a/src/vocal/__tests__/service.test.ts +++ b/src/vocal/__tests__/service.test.ts @@ -106,4 +106,32 @@ describe('VocalService word-boundary sync', () => { expect(seek).toHaveBeenCalledWith('azure_test_timeline', 0.34); service.dispose(); }); + + it('triggers the prosodic expression agency from speech timing', () => { + const prosodicService = { + startTalking: vi.fn(), + stopTalking: vi.fn(), + pulse: vi.fn(), + }; + const service = new VocalService({ + animationAgency: { + schedule: (snippet) => snippet.name, + remove: vi.fn(), + seek: vi.fn(), + }, + prosodicService, + }); + + vi.spyOn(performance, 'now').mockReturnValue(0); + + service.startSentence('hello world'); + service.onWordBoundary('hello', 0, 0); + service.stopSentence(); + + expect(prosodicService.startTalking).toHaveBeenCalledTimes(1); + expect(prosodicService.pulse).toHaveBeenCalledWith(0); + expect(prosodicService.stopTalking).toHaveBeenCalledTimes(1); + + service.dispose(); + }); }); diff --git a/src/vocal/index.ts b/src/vocal/index.ts index cd67503..b2ba546 100644 --- a/src/vocal/index.ts +++ b/src/vocal/index.ts @@ -39,6 +39,7 @@ export type { VocalSource, VocalTimeline, VocalWordTiming, + ProsodicExpressionAgency, VocalConfig, VocalState, } from './types'; diff --git a/src/vocal/service.ts b/src/vocal/service.ts index d5b0d3a..be62051 100644 --- a/src/vocal/service.ts +++ b/src/vocal/service.ts @@ -115,6 +115,8 @@ export class VocalService { wordTimings: this.normalizeWordTimings(timeline.wordTimings), }; + this.config.prosodicService?.startTalking(); + return name; } @@ -182,6 +184,7 @@ export class VocalService { ctx.wordIndex = expectedIndex + 1; this.store.setCurrentWord(word); + this.config.prosodicService?.pulse(expectedIndex); } /** @@ -212,6 +215,7 @@ export class VocalService { this.currentSentence = null; this.store.stopSpeaking(); + this.config.prosodicService?.stopTalking(); } /** @@ -450,6 +454,7 @@ export class VocalService { // Clear sentence context if this was it if (this.currentSentence?.name === name) { this.currentSentence = null; + this.config.prosodicService?.stopTalking(); } // Update state if this was the last snippet diff --git a/src/vocal/snippetBuilder.ts b/src/vocal/snippetBuilder.ts index a8f6717..64d5b09 100644 --- a/src/vocal/snippetBuilder.ts +++ b/src/vocal/snippetBuilder.ts @@ -556,7 +556,7 @@ export function buildVocalSnippet( config?: Partial, name?: string ): VocalSnippet { - const cfg: Required> = { + const cfg: Required> = { ...DEFAULT_VOCAL_CONFIG, ...config, }; diff --git a/src/vocal/types.ts b/src/vocal/types.ts index c4c872e..0dedf02 100644 --- a/src/vocal/types.ts +++ b/src/vocal/types.ts @@ -74,6 +74,14 @@ export interface VocalTimeline { source?: VocalSource; } +/** Prosodic expression agency triggered by speech timing. */ +export interface ProsodicExpressionAgency { + startTalking: () => void; + stopTalking: () => void; + pulse: (wordIndex: number) => void; + stop?: () => void; +} + // ───────────────────────────────────────────────────────────────────────────── // Configuration // ───────────────────────────────────────────────────────────────────────────── @@ -112,6 +120,9 @@ export interface VocalConfig { resumeSnippet?: (name: string) => void; seek?: (name: string, offsetSec: number) => void; }; + + /** Prosodic expression agency for speech-time brow and head gestures. */ + prosodicService?: ProsodicExpressionAgency; } export interface VocalState { @@ -126,7 +137,7 @@ export interface VocalState { // Constants // ───────────────────────────────────────────────────────────────────────────── -export const DEFAULT_VOCAL_CONFIG: Required> = { +export const DEFAULT_VOCAL_CONFIG: Required> = { intensity: 1.0, speechRate: 1.0, jawScale: 1.0,