diff --git a/specs/multi-track-audio-timeline.md b/specs/multi-track-audio-timeline.md index 0edebcb..24628aa 100644 --- a/specs/multi-track-audio-timeline.md +++ b/specs/multi-track-audio-timeline.md @@ -44,3 +44,9 @@ SoundtrackPanel: ordered clip list, add / reorder / remove. Beat grid uses first `audio-analysis` module: RMS normalization to -18 dBFS. `loudnessCache` in slideshow.json (filename + byteLength). Manual `gainDb` on clip overrides auto offset. Per-clip dB input in SoundtrackPanel. Closes #47. + +## Slice 21 — Audio-driven duration + media loop (done) + +When audio exceeds visual timeline, `totalFrames` = sum of clip durations and slides loop in order until audio ends. Partial tail truncates last slide. Visual wins when longer than audio. + +Closes #46. diff --git a/src/composition/SlideshowComposition.tsx b/src/composition/SlideshowComposition.tsx index 9854ca7..3ee3a33 100644 --- a/src/composition/SlideshowComposition.tsx +++ b/src/composition/SlideshowComposition.tsx @@ -67,7 +67,7 @@ export function SlideshowComposition({ plan }: SlideshowProps) { ) : null} {plan.entries.map((entry) => ( - + {entry.transitionIn && entry.transitionIn.durationInFrames > 0 && ( { it('ducks music during a video slide between photos', () => { @@ -29,7 +29,7 @@ describe('plan — soundtrack ducking envelope', () => { const video = makeSlide('v', 'video', 120) const photo2 = makeSlide('b', 'image', 90) - const result = plan([photo, video, photo2], SETTINGS, undefined, AUDIO_CLIP) + const result = plan([photo, video, photo2], SETTINGS, undefined, SHORT_AUDIO) const envelope = result.duckingEnvelope expect(envelope).toBeDefined() @@ -50,7 +50,7 @@ describe('plan — soundtrack ducking envelope', () => { it('does not duck when video audio is muted', () => { const video = makeSlide('v', 'video', 120, { muteVideoAudio: true }) - const result = plan([video], SETTINGS, undefined, AUDIO_CLIP) + const result = plan([video], SETTINGS, undefined, SHORT_AUDIO) expect(result.duckingEnvelope?.segments).toEqual([]) expect(result.entries[0].videoVolume).toBe(0) @@ -59,7 +59,7 @@ describe('plan — soundtrack ducking envelope', () => { it('mutes soundtrack when muteMusic is set', () => { const video = makeSlide('v', 'video', 120, { muteMusic: true }) - const result = plan([video], SETTINGS, undefined, AUDIO_CLIP) + const result = plan([video], SETTINGS, undefined, SHORT_AUDIO) const { keyframes } = result.duckingEnvelope! expect(result.duckingEnvelope?.segments).toEqual([]) @@ -70,7 +70,7 @@ describe('plan — soundtrack ducking envelope', () => { it('uses custom musicVolume instead of default duck level', () => { const video = makeSlide('v', 'video', 120, { musicVolume: 0.5 }) - const result = plan([video], SETTINGS, undefined, AUDIO_CLIP) + const result = plan([video], SETTINGS, undefined, SHORT_AUDIO) expect(result.duckingEnvelope?.segments).toEqual([]) expect(result.duckingEnvelope?.keyframes).toContainEqual({ frame: 0, volume: 0.5 }) @@ -92,7 +92,7 @@ describe('plan — ducking envelope property', () => { ] for (const slides of cases) { - const result = plan(slides, SETTINGS, undefined, AUDIO_CLIP) + const result = plan(slides, SETTINGS, undefined, SHORT_AUDIO) const envelope = result.duckingEnvelope! const expectedSpans = getUnmutedVideoAudioSpans(result.entries).filter((span) => { const entry = result.entries.find((candidate) => candidate.startFrame === span.startFrame) diff --git a/src/sequence-planner/planner.test.ts b/src/sequence-planner/planner.test.ts index 507a9b5..40b14df 100644 --- a/src/sequence-planner/planner.test.ts +++ b/src/sequence-planner/planner.test.ts @@ -397,7 +397,7 @@ describe('plan — audio clips', () => { }, ]) expect(result.duckingEnvelope).toEqual(DUCKING) - expect(result.totalFrames).toBe(90) + expect(result.totalFrames).toBe(120) }) it('omits audio segments when not provided', () => { @@ -424,11 +424,63 @@ describe('plan — audio clips', () => { ]) }) - it('keeps slideshow totalFrames independent of audio duration', () => { + it('uses visual duration when audio is shorter than slides', () => { const shortClip = { blobUrl: 'blob:short', durationInFrames: 60 } const result = plan([IMG_90, IMG_90B], CROSSFADE, undefined, [shortClip]) expect(result.totalFrames).toBe(165) expect(result.audioSegments?.[0].durationInFrames).toBe(60) + expect(result.entries.length).toBe(2) + }) + + it('extends totalFrames to audio duration and loops the slide sequence', () => { + const longClip = { blobUrl: 'blob:long', durationInFrames: 400 } + const result = plan([IMG_90, IMG_90B], CROSSFADE, undefined, [longClip]) + expect(result.totalFrames).toBe(400) + expect(result.entries.length).toBeGreaterThan(2) + expect(result.entries[2].startFrame).toBe(165) + const lastEntry = result.entries.at(-1) + expect(lastEntry).toBeDefined() + expect(lastEntry!.startFrame + lastEntry!.durationInFrames).toBe(400) + }) +}) + +describe('plan — audio-driven loop golden snapshot', () => { + const CUT_SETTINGS: GlobalSettings = { ...DEFAULT_GLOBAL_SETTINGS, transitionType: 'cut' } + const LONG_AUDIO = [{ blobUrl: 'blob:long', durationInFrames: 250 }] + + it('matches golden snapshot for looped entries with a partial tail', () => { + const result = plan([IMG_90, IMG_90B], CUT_SETTINGS, undefined, LONG_AUDIO) + + expect(result.totalFrames).toBe(250) + expect(result.entries).toEqual([ + { + slide: IMG_90, + startFrame: 0, + durationInFrames: 90, + transitionIn: undefined, + fitMode: 'cover', + kenBurns: result.entries[0].kenBurns, + videoVolume: 1, + }, + { + slide: IMG_90B, + startFrame: 90, + durationInFrames: 90, + transitionIn: { type: 'cut', durationInFrames: 0 }, + fitMode: 'cover', + kenBurns: result.entries[1].kenBurns, + videoVolume: 1, + }, + { + slide: IMG_90, + startFrame: 180, + durationInFrames: 70, + transitionIn: undefined, + fitMode: 'cover', + kenBurns: result.entries[2].kenBurns, + videoVolume: 1, + }, + ]) }) }) diff --git a/src/sequence-planner/planner.ts b/src/sequence-planner/planner.ts index 856d090..0c65c26 100644 --- a/src/sequence-planner/planner.ts +++ b/src/sequence-planner/planner.ts @@ -41,6 +41,38 @@ const KB_PRESETS: KenBurnsVector[] = [ const KB_ZOOM_IN_PRESETS: KenBurnsVector[] = [KB_PRESETS[0], KB_PRESETS[2]] +function totalAudioFrames(audioClips?: AudioClipInput[]): number { + if (!audioClips?.length) return 0 + return audioClips.reduce((sum, clip) => sum + clip.durationInFrames, 0) +} + +function computePassDuration(slideDurations: number[], effectiveTrans: number[]): number { + let cursor = 0 + for (let index = 0; index < slideDurations.length; index++) { + if (index < slideDurations.length - 1) { + cursor += slideDurations[index] - effectiveTrans[index + 1] + } else { + cursor += slideDurations[index] + } + } + return cursor +} + +function loopBoundaryTransition( + prevDuration: number, + currDuration: number, + transitionType: TransitionType, + maxTransitionDur: number, +): TransitionSpec | undefined { + const transDur = Math.min( + maxTransitionDur, + Math.floor(prevDuration / 2), + Math.floor(currDuration / 2), + ) + if (transDur <= 0) return undefined + return { type: transitionType, durationInFrames: transDur } +} + function buildAudioOutput( entries: RenderPlanEntry[], audioClips?: AudioClipInput[], @@ -72,10 +104,11 @@ export function plan( beatGrid?: BeatGrid, ): RenderPlan { if (slides.length === 0) { + const audioTotal = totalAudioFrames(audioClips) return { entries: [], ...buildAudioOutput([], audioClips), - totalFrames: 0, + totalFrames: audioTotal, } } @@ -106,69 +139,89 @@ export function plan( return slide.type === 'video' ? 'contain' : resolved(slide).fitMode } - // Per-slide resolved transition duration (inbound = this slide's resolved type). function getTransitionDur(slide: Slide): number { return TRANSITION_FRAMES[resolved(slide).transitionType] } - // Pass 1: compute effective transition duration for each non-first slide. - // Clamp to half of each adjacent slide so the transition never consumes - // more than the slide it belongs to (Remotion requirement). - const effectiveTrans: number[] = slides.map((slide, i) => { - if (i === 0) return 0 - const prevDur = getDuration(slides[i - 1]) - const currDur = getDuration(slide) - const tDur = getTransitionDur(slide) - return Math.min(tDur, Math.floor(prevDur / 2), Math.floor(currDur / 2)) + const slideDurations = slides.map((slide) => getDuration(slide)) + const effectiveTrans = slides.map((slide, index) => { + if (index === 0) return 0 + const transitionDur = getTransitionDur(slide) + return Math.min( + transitionDur, + Math.floor(slideDurations[index - 1] / 2), + Math.floor(slideDurations[index] / 2), + ) }) - // Pass 2: build entries. - // photoIndex counts only non-video slides so Ken Burns presets alternate - // correctly even when photos and videos are interspersed. + const passDuration = computePassDuration(slideDurations, effectiveTrans) + const audioTotal = totalAudioFrames(audioClips) + const totalFrames = audioClips?.length + ? Math.max(passDuration, audioTotal) + : passDuration + const entries: RenderPlanEntry[] = [] let cursor = 0 let photoIndex = 0 - for (let i = 0; i < slides.length; i++) { - const slide = slides[i] - const durationInFrames = getDuration(slide) - const slideResolved = resolved(slide) - - const transitionIn: TransitionSpec | undefined = - i === 0 - ? undefined - : { type: slideResolved.transitionType, durationInFrames: effectiveTrans[i] } - - const kbPresets = - slideResolved.kenBurnsMode === 'zoom-in-only' ? KB_ZOOM_IN_PRESETS : KB_PRESETS - const kenBurns: KenBurnsVector | null = - !isTitleSlide(slide) && slideResolved.kenBurns && slide.type !== 'video' - ? kbPresets[photoIndex % kbPresets.length] - : null - - if (!isTitleSlide(slide) && slide.type !== 'video') photoIndex++ - - entries.push({ - slide, - startFrame: cursor, - durationInFrames, - transitionIn, - fitMode: getFitMode(slide), - kenBurns, - videoVolume: resolveVideoVolume(slide), - }) - - // Advance by this slide's duration minus the NEXT slide's inbound transition overlap. - if (i < slides.length - 1) { - cursor += durationInFrames - effectiveTrans[i + 1] - } else { - cursor += durationInFrames + while (cursor < totalFrames) { + for (let index = 0; index < slides.length; index++) { + if (cursor >= totalFrames) break + + const slide = slides[index] + const fullDuration = slideDurations[index] + const remaining = totalFrames - cursor + const durationInFrames = Math.min(fullDuration, remaining) + const slideResolved = resolved(slide) + const isLoopStart = entries.length > 0 && index === 0 + + let transitionIn: TransitionSpec | undefined + if (index === 0 && isLoopStart) { + transitionIn = loopBoundaryTransition( + slideDurations[slides.length - 1], + fullDuration, + slideResolved.transitionType, + getTransitionDur(slide), + ) + } else if (index > 0 && durationInFrames === fullDuration) { + transitionIn = { + type: slideResolved.transitionType, + durationInFrames: effectiveTrans[index], + } + } + + const kbPresets = + slideResolved.kenBurnsMode === 'zoom-in-only' ? KB_ZOOM_IN_PRESETS : KB_PRESETS + const kenBurns: KenBurnsVector | null = + !isTitleSlide(slide) && slideResolved.kenBurns && slide.type !== 'video' + ? kbPresets[photoIndex % kbPresets.length] + : null + + if (!isTitleSlide(slide) && slide.type !== 'video') photoIndex++ + + entries.push({ + slide, + startFrame: cursor, + durationInFrames, + transitionIn, + fitMode: getFitMode(slide), + kenBurns, + videoVolume: resolveVideoVolume(slide), + }) + + const hasNextInPass = index < slides.length - 1 + const isFullSlide = durationInFrames === fullDuration + if (hasNextInPass && isFullSlide && cursor + durationInFrames < totalFrames) { + cursor += durationInFrames - effectiveTrans[index + 1] + } else { + cursor += durationInFrames + } } } return { entries, ...buildAudioOutput(entries, audioClips), - totalFrames: cursor, + totalFrames, } }