From 563886776b086f952f138053095180e92531660e Mon Sep 17 00:00:00 2001 From: cbaucom Date: Mon, 22 Jun 2026 10:24:36 -0400 Subject: [PATCH 1/5] Extend beat grid across multi-clip audio timeline. Per-file beat grid cache, concatenated beat times for planning, and position-aware snap so beat sync works across clip boundaries and looped slides. Co-authored-by: Cursor --- specs/multi-track-audio-timeline.md | 6 ++ src/beat-grid/concat.test.ts | 53 +++++++++++++ src/beat-grid/concat.ts | 54 +++++++++++++ src/beat-grid/index.ts | 13 +++- src/beat-grid/manual.test.ts | 30 ++++---- src/beat-grid/manual.ts | 37 +++++++-- src/beat-grid/nudge-position.test.ts | 17 ++++ src/beat-grid/nudge-position.ts | 36 +++++++++ src/beat-grid/types.ts | 2 + src/editor-shell/App.tsx | 7 +- src/editor-shell/slidePersistence.test.ts | 45 ++++++++++- src/editor-shell/slidePersistence.ts | 22 +++++- src/editor-shell/useBeatGrid.ts | 94 +++++++++++++++++------ src/editor-shell/useProject.ts | 18 ++--- src/project-store/schema.ts | 4 +- src/sequence-planner/planner.test.ts | 42 ++++++++++ src/sequence-planner/planner.ts | 29 +++++-- 17 files changed, 434 insertions(+), 75 deletions(-) create mode 100644 src/beat-grid/concat.test.ts create mode 100644 src/beat-grid/concat.ts create mode 100644 src/beat-grid/nudge-position.test.ts create mode 100644 src/beat-grid/nudge-position.ts diff --git a/specs/multi-track-audio-timeline.md b/specs/multi-track-audio-timeline.md index 24628aa..bcf1597 100644 --- a/specs/multi-track-audio-timeline.md +++ b/specs/multi-track-audio-timeline.md @@ -50,3 +50,9 @@ Closes #47. When audio exceeds visual timeline, `totalFrames` = sum of clip durations and slides loop in order until audio ends. Partial tail truncates last slide. Visual wins when longer than audio. Closes #46. + +## Slice 23 — Beat grid across multi-clip timeline (done) + +Per-file `beatGridCache` in slideshow.json (migrates legacy single `BeatGrid`). `buildConcatenatedBeatTimes` shifts each clip's beats by clip start. Manual beat grid spans total audio duration. Planner uses position-aware `nudgeSlideEndFrame` when concatenated beat times are provided. `useBeatGrid` analyzes only clips missing from cache; reorder preserves cache. + +Closes #49. diff --git a/src/beat-grid/concat.test.ts b/src/beat-grid/concat.test.ts new file mode 100644 index 0000000..4896d36 --- /dev/null +++ b/src/beat-grid/concat.test.ts @@ -0,0 +1,53 @@ +import { describe, expect, it } from 'vitest' +import { buildConcatenatedBeatTimes } from './concat' +import { resolveConcatenatedBeatTimes } from './manual' +import type { BeatGrid, BeatGridCache } from './types' + +const MANUAL: BeatGrid = { bpm: 120, beatIntervalSecs: 0.5, firstBeatOffsetSecs: 0.1 } +const FPS = 30 + +describe('buildConcatenatedBeatTimes', () => { + it('shifts beat times by each clip start on the concatenated timeline', () => { + const cache: BeatGridCache = { + 'a.mp3': { bpm: 120, beatIntervalSecs: 0.5, firstBeatOffsetSecs: 0 }, + 'b.mp3': { bpm: 120, beatIntervalSecs: 0.5, firstBeatOffsetSecs: 0.1 }, + } + const clips = [ + { filename: 'a.mp3', durationInFrames: 60 }, + { filename: 'b.mp3', durationInFrames: 60 }, + ] + + expect(buildConcatenatedBeatTimes(clips, cache, FPS)).toEqual([ + 0, + 0.5, + 1, + 1.5, + 2.1, + 2.6, + 3.1, + 3.6, + ]) + }) +}) + +describe('resolveConcatenatedBeatTimes', () => { + const clips = [ + { filename: 'a.mp3', durationInFrames: 60 }, + { filename: 'b.mp3', durationInFrames: 60 }, + ] + + it('uses manual grid across total audio duration', () => { + const beatTimes = resolveConcatenatedBeatTimes(MANUAL, undefined, clips, FPS) + expect(beatTimes?.[0]).toBe(0.1) + expect(beatTimes?.at(-1)).toBeLessThan(4) + }) + + it('builds beat times from per-file cache when manual is absent', () => { + const cache: BeatGridCache = { + 'a.mp3': { bpm: 120, beatIntervalSecs: 0.5, firstBeatOffsetSecs: 0 }, + 'b.mp3': { bpm: 120, beatIntervalSecs: 0.5, firstBeatOffsetSecs: 0 }, + } + const beatTimes = resolveConcatenatedBeatTimes(undefined, cache, clips, FPS) + expect(beatTimes).toEqual(buildConcatenatedBeatTimes(clips, cache, FPS)) + }) +}) diff --git a/src/beat-grid/concat.ts b/src/beat-grid/concat.ts new file mode 100644 index 0000000..71bbc6a --- /dev/null +++ b/src/beat-grid/concat.ts @@ -0,0 +1,54 @@ +import type { BeatGrid, BeatGridCache } from './types' + +export type AudioClipTiming = { + durationInFrames: number + filename: string +} + +export function beatTimesFromGrid(grid: BeatGrid, durationSecs: number): number[] { + const beats: number[] = [] + let beatIndex = 0 + let beatTimeSecs = grid.firstBeatOffsetSecs + + while (beatTimeSecs < durationSecs) { + beats.push(beatTimeSecs) + beatIndex++ + beatTimeSecs = grid.firstBeatOffsetSecs + beatIndex * grid.beatIntervalSecs + } + + return beats +} + +export function buildConcatenatedBeatTimes( + clips: AudioClipTiming[], + cache: BeatGridCache, + fps = 30, +): number[] { + const beats: number[] = [] + let clipStartSecs = 0 + + for (const clip of clips) { + const grid = cache[clip.filename] + const clipDurationSecs = clip.durationInFrames / fps + if (!grid) { + clipStartSecs += clipDurationSecs + continue + } + + let beatIndex = 0 + let localBeatSecs = grid.firstBeatOffsetSecs + while (localBeatSecs < clipDurationSecs) { + beats.push(clipStartSecs + localBeatSecs) + beatIndex++ + localBeatSecs = grid.firstBeatOffsetSecs + beatIndex * grid.beatIntervalSecs + } + + clipStartSecs += clipDurationSecs + } + + return beats.sort((left, right) => left - right) +} + +export function totalAudioDurationSecs(clips: AudioClipTiming[], fps = 30): number { + return clips.reduce((sum, clip) => sum + clip.durationInFrames, 0) / fps +} diff --git a/src/beat-grid/index.ts b/src/beat-grid/index.ts index ab5b278..f429ccc 100644 --- a/src/beat-grid/index.ts +++ b/src/beat-grid/index.ts @@ -1,6 +1,15 @@ -export type { BeatGrid } from './types' +export type { BeatGrid, BeatGridCache } from './types' export { detectBeatGrid } from './detection' export { nudge } from './nudge' +export { nudgeSlideEndFrame } from './nudge-position' export { decodeMono } from './adapter' export { tapToBpm, manualBeatGridFromBpm } from './tap' -export { resolveEffectiveBeatGrid } from './manual' +export { + resolveConcatenatedBeatTimes, + resolveEffectiveBeatGrid, +} from './manual' +export { + buildConcatenatedBeatTimes, + beatTimesFromGrid, + totalAudioDurationSecs, +} from './concat' diff --git a/src/beat-grid/manual.test.ts b/src/beat-grid/manual.test.ts index a06c8a7..06f21a6 100644 --- a/src/beat-grid/manual.test.ts +++ b/src/beat-grid/manual.test.ts @@ -1,37 +1,33 @@ import { describe, expect, it } from 'vitest' import { resolveEffectiveBeatGrid } from './manual' -import type { BeatGrid } from './types' +import type { BeatGrid, BeatGridCache } from './types' const AUTO: BeatGrid = { bpm: 118, beatIntervalSecs: 60 / 118, firstBeatOffsetSecs: 0.05 } const MANUAL: BeatGrid = { bpm: 120, beatIntervalSecs: 0.5, firstBeatOffsetSecs: 0.1 } -describe('resolveEffectiveBeatGrid — priority (AC2)', () => { +describe('resolveEffectiveBeatGrid — priority', () => { + const cache: BeatGridCache = { 'track.mp3': AUTO } + it('returns manual grid when both manual and cache are present', () => { - const result = resolveEffectiveBeatGrid(MANUAL, AUTO) - expect(result).toEqual(MANUAL) + expect(resolveEffectiveBeatGrid(MANUAL, cache)).toEqual(MANUAL) }) it('returns auto-detected cache when no manual override is set', () => { - const result = resolveEffectiveBeatGrid(undefined, AUTO) - expect(result).toEqual(AUTO) + expect(resolveEffectiveBeatGrid(undefined, cache)).toEqual(AUTO) }) it('returns undefined when neither manual nor cache exists', () => { - const result = resolveEffectiveBeatGrid(undefined, undefined) - expect(result).toBeUndefined() + expect(resolveEffectiveBeatGrid(undefined, undefined)).toBeUndefined() }) - it('manual still wins after cache is updated (re-analysis does not override manual)', () => { - const updatedCache: BeatGrid = { bpm: 115, beatIntervalSecs: 60 / 115, firstBeatOffsetSecs: 0.02 } - // Simulate re-analysis: cache changes but manual stays - const result = resolveEffectiveBeatGrid(MANUAL, updatedCache) - expect(result).toEqual(MANUAL) - expect(result?.bpm).toBe(120) + it('manual still wins after cache is updated', () => { + const updatedCache: BeatGridCache = { + 'track.mp3': { bpm: 115, beatIntervalSecs: 60 / 115, firstBeatOffsetSecs: 0.02 }, + } + expect(resolveEffectiveBeatGrid(MANUAL, updatedCache)).toEqual(MANUAL) }) it('returns cache once manual is cleared', () => { - // Simulate clearing: pass undefined for manual - const result = resolveEffectiveBeatGrid(undefined, AUTO) - expect(result).toEqual(AUTO) + expect(resolveEffectiveBeatGrid(undefined, cache)).toEqual(AUTO) }) }) diff --git a/src/beat-grid/manual.ts b/src/beat-grid/manual.ts index 096d736..3c67514 100644 --- a/src/beat-grid/manual.ts +++ b/src/beat-grid/manual.ts @@ -1,13 +1,40 @@ -import type { BeatGrid } from './types' +import type { BeatGrid, BeatGridCache } from './types' +import { + beatTimesFromGrid, + buildConcatenatedBeatTimes, + totalAudioDurationSecs, + type AudioClipTiming, +} from './concat' /** * Return the effective BeatGrid for planning: manual override wins over - * auto-detected cache. Manual values persist across re-analysis until - * explicitly cleared (by passing undefined as manualBeatGrid). + * auto-detected cache. Manual values persist across re-analysis until cleared. */ export function resolveEffectiveBeatGrid( manualBeatGrid: BeatGrid | undefined, - beatGridCache: BeatGrid | undefined, + beatGridCache: BeatGridCache | undefined, ): BeatGrid | undefined { - return manualBeatGrid ?? beatGridCache + if (manualBeatGrid) return manualBeatGrid + if (!beatGridCache) return undefined + const filenames = Object.keys(beatGridCache) + if (filenames.length === 0) return undefined + return beatGridCache[filenames[0]] +} + +export function resolveConcatenatedBeatTimes( + manualBeatGrid: BeatGrid | undefined, + beatGridCache: BeatGridCache | undefined, + clips: AudioClipTiming[], + fps = 30, +): number[] | undefined { + if (clips.length === 0) return undefined + + const totalSecs = totalAudioDurationSecs(clips, fps) + if (manualBeatGrid) { + return beatTimesFromGrid(manualBeatGrid, totalSecs) + } + + if (!beatGridCache) return undefined + const beatTimes = buildConcatenatedBeatTimes(clips, beatGridCache, fps) + return beatTimes.length > 0 ? beatTimes : undefined } diff --git a/src/beat-grid/nudge-position.test.ts b/src/beat-grid/nudge-position.test.ts new file mode 100644 index 0000000..e8a1eaa --- /dev/null +++ b/src/beat-grid/nudge-position.test.ts @@ -0,0 +1,17 @@ +import { describe, expect, it } from 'vitest' +import { nudgeSlideEndFrame } from './nudge-position' + +const BEAT_TIMES = [0, 0.5, 1, 1.5, 2, 2.5, 3, 3.5] +const FPS = 30 + +describe('nudgeSlideEndFrame', () => { + it('snaps a slide end to the nearest concatenated beat', () => { + const duration = nudgeSlideEndFrame(45, 40, BEAT_TIMES, 'medium', FPS) + expect(45 + duration).toBe(90) + }) + + it('uses beats after the slide start when snapping', () => { + const duration = nudgeSlideEndFrame(60, 40, BEAT_TIMES, 'medium', FPS) + expect(60 + duration).toBe(105) + }) +}) diff --git a/src/beat-grid/nudge-position.ts b/src/beat-grid/nudge-position.ts new file mode 100644 index 0000000..92ea34b --- /dev/null +++ b/src/beat-grid/nudge-position.ts @@ -0,0 +1,36 @@ +import type { Energy } from '../timeline-core/settings' + +const ENERGY_MULTIPLIER: Record = { + calm: 1.5, + medium: 1.0, + punchy: 0.67, +} + +export function nudgeSlideEndFrame( + startFrame: number, + targetDurationFrames: number, + beatTimesSecs: number[], + energy: Energy, + fps = 30, +): number { + if (beatTimesSecs.length === 0) return Math.round(targetDurationFrames) + + const scaledTargetFrames = targetDurationFrames * ENERGY_MULTIPLIER[energy] + const startSecs = startFrame / fps + const targetEndSecs = startSecs + scaledTargetFrames / fps + + let nearestBeatSecs = beatTimesSecs[0] + let minDistance = Math.abs(nearestBeatSecs - targetEndSecs) + + for (const beatSecs of beatTimesSecs) { + if (beatSecs < startSecs) continue + const distance = Math.abs(beatSecs - targetEndSecs) + if (distance < minDistance) { + minDistance = distance + nearestBeatSecs = beatSecs + } + } + + const endFrame = Math.round(nearestBeatSecs * fps) + return Math.max(1, endFrame - startFrame) +} diff --git a/src/beat-grid/types.ts b/src/beat-grid/types.ts index 7f73924..5a298af 100644 --- a/src/beat-grid/types.ts +++ b/src/beat-grid/types.ts @@ -3,3 +3,5 @@ export type BeatGrid = { firstBeatOffsetSecs: number beatIntervalSecs: number } + +export type BeatGridCache = Record diff --git a/src/editor-shell/App.tsx b/src/editor-shell/App.tsx index e332cf7..9f8b940 100644 --- a/src/editor-shell/App.tsx +++ b/src/editor-shell/App.tsx @@ -61,13 +61,11 @@ export function App() { recentProjects, } = project - const primaryClipFilename = audioClips[0]?.filename ?? null - const beatGrid = useBeatGrid({ + audioClips, audioTracks, onPersistChange: updateBeatGridPersist, persisted: { beatGridCache, manualBeatGrid }, - primaryClipFilename, }) useLoudness({ @@ -133,8 +131,9 @@ export function App() { undefined, planAudioClips.length > 0 ? planAudioClips : undefined, beatGrid.effectiveBeatGrid, + beatGrid.concatenatedBeatTimes, ), - [beatGrid.effectiveBeatGrid, globalSettings, planAudioClips, slides], + [beatGrid.concatenatedBeatTimes, beatGrid.effectiveBeatGrid, globalSettings, planAudioClips, slides], ) const totalFrames = renderPlan.totalFrames > 0 ? renderPlan.totalFrames : FPS const canvas = dimensionsForAspectRatio(aspectRatio) diff --git a/src/editor-shell/slidePersistence.test.ts b/src/editor-shell/slidePersistence.test.ts index 840dd3b..6a66223 100644 --- a/src/editor-shell/slidePersistence.test.ts +++ b/src/editor-shell/slidePersistence.test.ts @@ -1,10 +1,10 @@ import { describe, expect, it } from 'vitest' import { DEFAULT_GLOBAL_SETTINGS } from '../timeline-core' import type { MediaSlide } from '../timeline-core/types' -import { SCHEMA_VERSION } from '../project-store' +import { SCHEMA_VERSION, type SlideshowJson } from '../project-store' import type { JamendoAttribution } from '../jamendo/types' import type { BeatGrid } from '../beat-grid/types' -import { audioClipsFromJson, slidesToJson } from './slidePersistence' +import { audioClipsFromJson, beatGridCacheFromJson, slidesToJson } from './slidePersistence' const BEAT_GRID: BeatGrid = { beatIntervalSecs: 0.5, @@ -85,6 +85,7 @@ describe('slidesToJson', () => { }) it('includes beatGridCache when provided', () => { + const beatGridCache = { 'track.mp3': BEAT_GRID } const json = slidesToJson( DEFAULT_GLOBAL_SETTINGS, [makeSlide('a.jpg')], @@ -92,9 +93,9 @@ describe('slidesToJson', () => { null, null, null, - BEAT_GRID, + beatGridCache, ) - expect(json.beatGridCache).toEqual(BEAT_GRID) + expect(json.beatGridCache).toEqual(beatGridCache) }) it('includes manualBeatGrid when provided', () => { @@ -184,3 +185,39 @@ describe('audioClipsFromJson', () => { expect(audioClipsFromJson({ schemaVersion: 1, slides: [] })).toEqual([]) }) }) + +describe('beatGridCacheFromJson', () => { + it('reads per-file beatGridCache', () => { + const cache = beatGridCacheFromJson({ + schemaVersion: 1, + slides: [], + audioClips: [{ filename: 'a.mp3' }], + beatGridCache: { 'a.mp3': BEAT_GRID }, + }) + expect(cache).toEqual({ 'a.mp3': BEAT_GRID }) + }) + + it('migrates legacy single BeatGrid to first clip filename', () => { + const cache = beatGridCacheFromJson({ + schemaVersion: 1, + slides: [], + audioClips: [{ filename: 'theme.mp3' }], + beatGridCache: BEAT_GRID, + } as unknown as SlideshowJson) + expect(cache).toEqual({ 'theme.mp3': BEAT_GRID }) + }) + + it('migrates legacy BeatGrid using soundtrackFilename when no audioClips', () => { + const cache = beatGridCacheFromJson({ + schemaVersion: 1, + slides: [], + soundtrackFilename: 'legacy.mp3', + beatGridCache: BEAT_GRID, + } as unknown as SlideshowJson) + expect(cache).toEqual({ 'legacy.mp3': BEAT_GRID }) + }) + + it('returns undefined when beatGridCache is absent', () => { + expect(beatGridCacheFromJson({ schemaVersion: 1, slides: [] })).toBeUndefined() + }) +}) diff --git a/src/editor-shell/slidePersistence.ts b/src/editor-shell/slidePersistence.ts index c807379..44fb8f2 100644 --- a/src/editor-shell/slidePersistence.ts +++ b/src/editor-shell/slidePersistence.ts @@ -1,4 +1,4 @@ -import type { BeatGrid } from '../beat-grid/types' +import type { BeatGrid, BeatGridCache } from '../beat-grid/types' import type { LoudnessCache } from '../audio-analysis/types' import type { AudioClip, MediaSlide, Slide } from '../timeline-core/types' import { isTitleSlide } from '../timeline-core/types' @@ -43,6 +43,22 @@ export function audioClipsFromJson(saved: SlideshowJson): AudioClip[] { return [] } +function isLegacyBeatGrid(value: unknown): value is BeatGrid { + if (typeof value !== 'object' || value === null) return false + const record = value as Record + return typeof record.bpm === 'number' +} + +export function beatGridCacheFromJson(saved: SlideshowJson): BeatGridCache | undefined { + const raw = saved.beatGridCache as BeatGrid | BeatGridCache | undefined + if (!raw) return undefined + if (isLegacyBeatGrid(raw)) { + const filename = saved.audioClips?.[0]?.filename ?? saved.soundtrackFilename + return filename ? { [filename]: raw } : undefined + } + return raw +} + export function slidesToJson( globalSettings: GlobalSettings, slides: Slide[], @@ -50,7 +66,7 @@ export function slidesToJson( themeName?: ThemeName | null, soundtrackAttribution?: JamendoAttribution | null, aspectRatio?: AspectRatio | null, - beatGridCache?: BeatGrid | null, + beatGridCache?: BeatGridCache | null, manualBeatGrid?: BeatGrid | null, loudnessCache?: LoudnessCache | null, ): SlideshowJson { @@ -66,7 +82,7 @@ export function slidesToJson( })), } : {}), - ...(beatGridCache ? { beatGridCache } : {}), + ...(beatGridCache && Object.keys(beatGridCache).length > 0 ? { beatGridCache } : {}), ...(loudnessCache && Object.keys(loudnessCache).length > 0 ? { loudnessCache } : {}), ...(manualBeatGrid ? { manualBeatGrid } : {}), ...(themeName ? { themeName } : {}), diff --git a/src/editor-shell/useBeatGrid.ts b/src/editor-shell/useBeatGrid.ts index 416ea2e..a8b612f 100644 --- a/src/editor-shell/useBeatGrid.ts +++ b/src/editor-shell/useBeatGrid.ts @@ -1,36 +1,50 @@ import { useCallback, useEffect, useMemo, useState } from 'react' +import type { AudioClip } from '../timeline-core/types' import type { AudioTrack } from '../project-store' import { decodeMono, detectBeatGrid, manualBeatGridFromBpm, + resolveConcatenatedBeatTimes, resolveEffectiveBeatGrid, tapToBpm, type BeatGrid, + type BeatGridCache, } from '../beat-grid' +import { FPS } from './PlayerPane' export type BeatGridAnalysisStatus = 'analyzing' | 'error' | 'idle' | 'ready' type PersistedBeatGrid = { - beatGridCache?: BeatGrid + beatGridCache?: BeatGridCache manualBeatGrid?: BeatGrid } type Options = { + audioClips: AudioClip[] audioTracks: AudioTrack[] onPersistChange: (update: PersistedBeatGrid) => void persisted: PersistedBeatGrid - primaryClipFilename: string | null } export function useBeatGrid({ + audioClips, audioTracks, onPersistChange, persisted, - primaryClipFilename, }: Options) { const [analysisFailedForFilename, setAnalysisFailedForFilename] = useState(null) + const clipTimings = useMemo( + () => audioClips.flatMap((clip) => { + const track = audioTracks.find((entry) => entry.filename === clip.filename) + if (!track) return [] + return [{ filename: clip.filename, durationInFrames: track.durationInFrames }] + }), + [audioClips, audioTracks], + ) + + const primaryClipFilename = audioClips[0]?.filename ?? null const soundtrack = primaryClipFilename ? audioTracks.find((track) => track.filename === primaryClipFilename) : undefined @@ -40,16 +54,38 @@ export function useBeatGrid({ persisted.beatGridCache, ) + const concatenatedBeatTimes = useMemo( + () => resolveConcatenatedBeatTimes( + persisted.manualBeatGrid, + persisted.beatGridCache, + clipTimings, + FPS, + ), + [clipTimings, persisted.beatGridCache, persisted.manualBeatGrid], + ) + + const pendingFilenames = useMemo( + () => audioClips + .map((clip) => clip.filename) + .filter((filename) => ( + !persisted.manualBeatGrid && !persisted.beatGridCache?.[filename] + )), + [audioClips, persisted.beatGridCache, persisted.manualBeatGrid], + ) + const analysisStatus = useMemo((): BeatGridAnalysisStatus => { - if (!soundtrack) return 'idle' - if (persisted.manualBeatGrid || persisted.beatGridCache) return 'ready' - if (analysisFailedForFilename === soundtrack.filename) return 'error' + if (audioClips.length === 0) return 'idle' + if (persisted.manualBeatGrid) return 'ready' + if (pendingFilenames.length === 0) return 'ready' + if (analysisFailedForFilename && pendingFilenames.includes(analysisFailedForFilename)) { + return 'error' + } return 'analyzing' }, [ analysisFailedForFilename, - persisted.beatGridCache, + audioClips.length, + pendingFilenames, persisted.manualBeatGrid, - soundtrack, ]) const setManualBeatGrid = useCallback((grid: BeatGrid | undefined) => { @@ -70,33 +106,46 @@ export function useBeatGrid({ }, [setManualBeatGrid]) useEffect(() => { - if (!soundtrack || persisted.manualBeatGrid || persisted.beatGridCache) { + if (persisted.manualBeatGrid || pendingFilenames.length === 0) { return } let cancelled = false - async function analyze() { - try { - const response = await fetch(soundtrack!.blobUrl) - const buffer = await response.arrayBuffer() - const { sampleRate, samples } = await decodeMono(buffer) - const grid = detectBeatGrid(samples, sampleRate) - if (cancelled) return - onPersistChange({ beatGridCache: grid }) - } catch { - if (cancelled) return - setAnalysisFailedForFilename(soundtrack!.filename) + async function analyzePending() { + const nextCache: BeatGridCache = { ...persisted.beatGridCache } + + for (const filename of pendingFilenames) { + const track = audioTracks.find((entry) => entry.filename === filename) + if (!track) continue + + try { + const response = await fetch(track.blobUrl) + const buffer = await response.arrayBuffer() + const { sampleRate, samples } = await decodeMono(buffer) + const grid = detectBeatGrid(samples, sampleRate) + if (cancelled) return + nextCache[filename] = grid + } catch { + if (cancelled) return + setAnalysisFailedForFilename(filename) + return + } + } + + if (!cancelled) { + onPersistChange({ beatGridCache: nextCache }) } } - void analyze() + void analyzePending() return () => { cancelled = true } }, [ + audioTracks, onPersistChange, + pendingFilenames, persisted.beatGridCache, persisted.manualBeatGrid, - soundtrack, ]) return { @@ -104,6 +153,7 @@ export function useBeatGrid({ applyManualBpm, applyTapTimestamps, clearManualBeatGrid, + concatenatedBeatTimes, effectiveBeatGrid, setManualBeatGrid, soundtrack, diff --git a/src/editor-shell/useProject.ts b/src/editor-shell/useProject.ts index 962198e..2cb29bc 100644 --- a/src/editor-shell/useProject.ts +++ b/src/editor-shell/useProject.ts @@ -19,10 +19,11 @@ import { type SlideshowJson, } from '../project-store' import { enumerateFolder, revokeSlideBlobUrls } from '../project-store/media-loader' -import { audioClipsFromJson, reconcileSlides, slidesToJson } from './slidePersistence' +import { audioClipsFromJson, beatGridCacheFromJson, reconcileSlides, slidesToJson } from './slidePersistence' import type { JamendoAttribution, JamendoTrack } from '../jamendo/types' import { downloadTrack, sanitizeFilename } from '../jamendo' import type { BeatGrid } from '../beat-grid' +import type { BeatGridCache } from '../beat-grid/types' import type { LoudnessCache } from '../audio-analysis/types' import { FPS } from './PlayerPane' @@ -53,7 +54,7 @@ export function useProject({ onFolderLoaded }: Options = {}) { const [slides, setSlides] = useState([]) const [soundtrackAttribution, setSoundtrackAttribution] = useState(null) const [themeName, setThemeName] = useState(null) - const [beatGridCache, setBeatGridCache] = useState() + const [beatGridCache, setBeatGridCache] = useState() const [loudnessCache, setLoudnessCache] = useState() const [manualBeatGrid, setManualBeatGrid] = useState() const [loading, setLoading] = useState(false) @@ -151,7 +152,9 @@ export function useProject({ onFolderLoaded }: Options = {}) { setThemeName(restoredThemeName) setSlides(finalSlides) setSoundtrackAttribution(restoredAttribution) - setBeatGridCache(restoredClips.length > 0 ? savedData?.beatGridCache : undefined) + setBeatGridCache( + savedData && restoredClips.length > 0 ? beatGridCacheFromJson(savedData) : undefined, + ) setLoudnessCache(savedData?.loudnessCache) setManualBeatGrid(restoredClips.length > 0 ? savedData?.manualBeatGrid : undefined) setProjectName(handle.name) @@ -245,23 +248,20 @@ export function useProject({ onFolderLoaded }: Options = {}) { }, []) const updateAudioClips = useCallback((clips: AudioClip[]) => { - const primaryChanged = audioClips[0]?.filename !== clips[0]?.filename - if (primaryChanged || clips.length === 0) { + if (clips.length === 0) { setBeatGridCache(undefined) setManualBeatGrid(undefined) - } - if (clips.length === 0) { setSoundtrackAttribution(null) } setAudioClips(clips) - }, [audioClips]) + }, []) const updateLoudnessCache = useCallback((cache: LoudnessCache) => { setLoudnessCache(cache) }, []) const updateBeatGridPersist = useCallback((update: { - beatGridCache?: BeatGrid + beatGridCache?: BeatGridCache manualBeatGrid?: BeatGrid }) => { if ('beatGridCache' in update) setBeatGridCache(update.beatGridCache) diff --git a/src/project-store/schema.ts b/src/project-store/schema.ts index 629c0a7..f7ae660 100644 --- a/src/project-store/schema.ts +++ b/src/project-store/schema.ts @@ -1,6 +1,6 @@ import type { AspectRatio } from '../timeline-core/aspect' import type { GlobalSettings, SlideOverrides, ThemeName } from '../timeline-core/settings' -import type { BeatGrid } from '../beat-grid/types' +import type { BeatGrid, BeatGridCache } from '../beat-grid/types' import type { LoudnessCache } from '../audio-analysis/types' import type { JamendoAttribution } from '../jamendo/types' @@ -43,7 +43,7 @@ export type SlideshowJson = { slides: SerializedSlide[] soundtrackFilename?: string themeName?: ThemeName - beatGridCache?: BeatGrid + beatGridCache?: BeatGridCache manualBeatGrid?: BeatGrid soundtrackAttribution?: JamendoAttribution } diff --git a/src/sequence-planner/planner.test.ts b/src/sequence-planner/planner.test.ts index 40b14df..c5b05db 100644 --- a/src/sequence-planner/planner.test.ts +++ b/src/sequence-planner/planner.test.ts @@ -588,3 +588,45 @@ describe('plan — beat sync energy effect (AC4)', () => { expect(avg(calmResult.entries)).toBeGreaterThan(avg(punchyResult.entries)) }) }) + +describe('plan — concatenated beat times (multi-clip)', () => { + const AUDIO_CLIPS = [ + { blobUrl: 'blob:a', durationInFrames: 60 }, + { blobUrl: 'blob:b', durationInFrames: 60 }, + ] + + it('snaps slide end to beats on the concatenated timeline using start position', () => { + const concatenatedBeatTimes = [0, 1, 2.1, 3.1] + const slides = [makeSlide('a', 'image', 50)] + const result = plan( + slides, + BEAT_SYNC_ON, + undefined, + AUDIO_CLIPS, + undefined, + concatenatedBeatTimes, + ) + expect(result.entries[0].durationInFrames).toBe(63) + expect(result.totalFrames).toBe(120) + }) + + it('position-aware snap differs from uniform grid when slide starts in clip 2', () => { + const concatenatedBeatTimes = [0, 1, 2.1, 3.1] + const slides = [ + makeSlide('a', 'image', 30), + makeSlide('b', 'image', 50), + ] + const withConcat = plan( + slides, + BEAT_SYNC_ON, + undefined, + AUDIO_CLIPS, + undefined, + concatenatedBeatTimes, + ) + expect(withConcat.entries[0].durationInFrames).toBe(30) + const secondEntry = withConcat.entries[1] + expect(secondEntry.startFrame).toBe(30) + expect(secondEntry.durationInFrames).toBe(63) + }) +}) diff --git a/src/sequence-planner/planner.ts b/src/sequence-planner/planner.ts index 0c65c26..d79eb74 100644 --- a/src/sequence-planner/planner.ts +++ b/src/sequence-planner/planner.ts @@ -4,6 +4,7 @@ import { isTitleSlide } from '../timeline-core/types' import type { Slide } from '../timeline-core/types' import type { BeatGrid } from '../beat-grid/types' import { nudge } from '../beat-grid/nudge' +import { nudgeSlideEndFrame } from '../beat-grid/nudge-position' import { buildDuckingEnvelope, resolveVideoVolume } from './ducking' import type { AudioSegment, @@ -102,6 +103,7 @@ export function plan( mediaMetadata?: Map, audioClips?: AudioClipInput[], beatGrid?: BeatGrid, + concatenatedBeatTimesSecs?: number[], ): RenderPlan { if (slides.length === 0) { const audioTotal = totalAudioFrames(audioClips) @@ -117,19 +119,30 @@ export function plan( return resolve(settings, slide.overrides) } - const beatSyncActive = !!beatGrid && settings.beatSync !== false + const beatSyncActive = settings.beatSync !== false + && (beatGrid !== undefined || (concatenatedBeatTimesSecs?.length ?? 0) > 0) - function getDuration(slide: Slide): number { + function getRawDuration(slide: Slide): number { if (isTitleSlide(slide)) return slide.durationInFrames const meta = mediaMetadata?.get(slide.filename)?.durationInFrames if (meta !== undefined) return meta - // For images, honour a per-slide imageDurationSecs override. let raw = slide.durationInFrames if (slide.type === 'image' && slide.overrides?.imageDurationSecs !== undefined) { raw = Math.round(resolved(slide).imageDurationSecs * FPS) } - if (beatSyncActive && slide.type !== 'video') { - return nudge(raw, beatGrid!, resolved(slide).energy ?? 'medium', FPS) + return raw + } + + function getDuration(slide: Slide, startFrame: number): number { + const raw = getRawDuration(slide) + if (beatSyncActive && !isTitleSlide(slide) && slide.type !== 'video') { + const energy = resolved(slide).energy ?? 'medium' + if (concatenatedBeatTimesSecs?.length) { + return nudgeSlideEndFrame(startFrame, raw, concatenatedBeatTimesSecs, energy, FPS) + } + if (beatGrid) { + return nudge(raw, beatGrid, energy, FPS) + } } return raw } @@ -143,7 +156,9 @@ export function plan( return TRANSITION_FRAMES[resolved(slide).transitionType] } - const slideDurations = slides.map((slide) => getDuration(slide)) + const slideDurations = slides.map((slide) => ( + concatenatedBeatTimesSecs?.length ? getRawDuration(slide) : getDuration(slide, 0) + )) const effectiveTrans = slides.map((slide, index) => { if (index === 0) return 0 const transitionDur = getTransitionDur(slide) @@ -169,7 +184,7 @@ export function plan( if (cursor >= totalFrames) break const slide = slides[index] - const fullDuration = slideDurations[index] + const fullDuration = getDuration(slide, cursor) const remaining = totalFrames - cursor const durationInFrames = Math.min(fullDuration, remaining) const slideResolved = resolved(slide) From 5e60d81a4643dd3fd7d2fe84c197cbd02ab91ef4 Mon Sep 17 00:00:00 2001 From: cbaucom Date: Mon, 22 Jun 2026 11:00:27 -0400 Subject: [PATCH 2/5] Speed up folder load and audio analysis. Single-pass clip analysis, parallel media enumeration, cap beat/loudness math to 30s, batch cache updates, and defer expensive replanning while beat grid analysis runs. Co-authored-by: Cursor --- scripts/playwright-folder-load-smoke.mjs | 149 +++++++++++++++++++++++ src/beat-grid/detection.ts | 9 +- src/editor-shell/App.tsx | 28 +++-- src/editor-shell/useAudioClipAnalysis.ts | 133 ++++++++++++++++++++ src/editor-shell/useBeatGrid.ts | 73 +---------- src/editor-shell/useProject.ts | 15 ++- src/project-store/audio-loader.ts | 17 +-- src/project-store/media-loader.ts | 15 +-- 8 files changed, 342 insertions(+), 97 deletions(-) create mode 100644 scripts/playwright-folder-load-smoke.mjs create mode 100644 src/editor-shell/useAudioClipAnalysis.ts diff --git a/scripts/playwright-folder-load-smoke.mjs b/scripts/playwright-folder-load-smoke.mjs new file mode 100644 index 0000000..7e459c0 --- /dev/null +++ b/scripts/playwright-folder-load-smoke.mjs @@ -0,0 +1,149 @@ +// Smoke: folder open + audio playlist load timing. Run: +// pnpm dev (or pnpm build && pnpm preview) +// node scripts/playwright-folder-load-smoke.mjs +import fs from 'node:fs' +import path from 'node:path' +import { spawnSync } from 'node:child_process' + +const APP_URL = process.env.APP_URL ?? 'http://localhost:5173/' +const MAX_LOAD_MS = Number(process.env.MAX_LOAD_MS ?? 8000) +const MAX_ANALYSIS_MS = Number(process.env.MAX_ANALYSIS_MS ?? 15000) + +function run(args) { + const result = spawnSync('npx', ['playwright-cli', ...args], { + cwd: process.cwd(), + encoding: 'utf8', + maxBuffer: 20 * 1024 * 1024, + }) + if (result.stdout) process.stdout.write(result.stdout) + if (result.stderr) process.stderr.write(result.stderr) + if (result.error) console.error(result.error) + return result.status ?? 1 +} + +const demo = path.resolve('test-fixtures/demo') +const names = ['photo.jpg', 'photo2.jpg', 'theme.wav'] +const payload = Object.fromEntries( + names.map((name) => [name, fs.readFileSync(path.join(demo, name)).toString('base64')]), +) +payload['slideshow.json'] = Buffer.from(JSON.stringify({ + audioClips: [{ filename: 'theme.wav' }], + globalSettings: { + fitMode: 'cover', + imageDurationSecs: 3, + kenBurns: true, + transitionType: 'crossfade', + }, + schemaVersion: 1, + slides: [ + { durationInFrames: 90, excluded: false, filename: 'photo.jpg', id: 'photo-a', type: 'image' }, + { durationInFrames: 90, excluded: false, filename: 'photo2.jpg', id: 'photo-b', type: 'image' }, + ], +}, null, 2)).toString('base64') + +const code = `async page => { + const payload = ${JSON.stringify(payload)}; + const maxLoadMs = ${MAX_LOAD_MS}; + const maxAnalysisMs = ${MAX_ANALYSIS_MS}; + + await page.goto(${JSON.stringify(APP_URL)}); + + await page.evaluate((files) => { + function decodeBase64(base64) { + const binary = atob(base64); + const bytes = new Uint8Array(binary.length); + for (let index = 0; index < binary.length; index++) bytes[index] = binary.charCodeAt(index); + return bytes; + } + class MockFileHandle { + constructor(name, base64) { + this.kind = 'file'; + this.name = name; + this._base64 = base64; + } + async getFile() { + const bytes = decodeBase64(this._base64); + const type = nameToType(this.name); + return new File([bytes], this.name, { type, lastModified: 1781145972852 }); + } + async createWritable() { + const handle = this; + return { + write: async (data) => { + const buffer = data instanceof Blob ? await data.arrayBuffer() : data; + handle._base64 = btoa(String.fromCharCode(...new Uint8Array(buffer))); + }, + close: async () => {}, + }; + } + } + function nameToType(name) { + if (name.endsWith('.jpg')) return 'image/jpeg'; + if (name.endsWith('.wav')) return 'audio/wav'; + if (name.endsWith('.json')) return 'application/json'; + return 'application/octet-stream'; + } + class MockDirHandle { + constructor(files) { + this.name = 'demo'; + this._files = files; + } + async getFileHandle(name, options) { + const handle = this._files.get(name); + if (handle) return handle; + if (options?.create) { + const created = new MockFileHandle(name, ''); + this._files.set(name, created); + return created; + } + throw new DOMException('NotFoundError'); + } + async *values() { + for (const handle of this._files.values()) yield handle; + } + } + const handles = new Map( + Object.entries(files).map(([name, base64]) => [name, new MockFileHandle(name, base64)]), + ); + window.showDirectoryPicker = async () => new MockDirHandle(handles); + }, payload); + + const loadStart = Date.now(); + await page.getByRole('button', { name: 'Open Folder' }).click(); + await page.getByText('photo2.jpg').first().waitFor({ timeout: maxLoadMs }); + const loadMs = Date.now() - loadStart; + + const loadingBanner = page.getByText('Loading media…'); + if (await loadingBanner.isVisible()) { + throw new Error('Loading media banner still visible after filmstrip appeared'); + } + + const player = page.locator('.__remotion-player'); + await player.waitFor({ timeout: 5000 }); + const playerMs = Date.now() - loadStart; + + const analysisStart = Date.now(); + const analyzing = page.getByText('Analyzing soundtrack…'); + if (await analyzing.isVisible().catch(() => false)) { + await analyzing.waitFor({ state: 'hidden', timeout: maxAnalysisMs }); + } + const analysisMs = Date.now() - analysisStart; + + const errors = await page.evaluate(() => { + return window.__playwrightErrors ?? []; + }); + + console.log('TIMING loadMs=' + loadMs + ' playerMs=' + playerMs + ' analysisMs=' + analysisMs); + if (loadMs > maxLoadMs) { + throw new Error('folder load exceeded ' + maxLoadMs + 'ms: ' + loadMs); + } + console.log('PASS folder load smoke'); +}` + +let status = run(['open', APP_URL, '--browser=chrome']) +if (status !== 0) process.exit(status) + +status = run(['run-code', code]) +run(['close']) +if (status !== 0) process.exit(status) +console.log('PASS folder load smoke complete') diff --git a/src/beat-grid/detection.ts b/src/beat-grid/detection.ts index fe4fe84..a24417c 100644 --- a/src/beat-grid/detection.ts +++ b/src/beat-grid/detection.ts @@ -1,6 +1,12 @@ import type { BeatGrid } from './types' const HOP_SIZE = 512 +const MAX_ANALYSIS_SECS = 30 + +function analysisSamples(samples: Float32Array, sampleRate: number): Float32Array { + const maxSamples = sampleRate * MAX_ANALYSIS_SECS + return samples.length > maxSamples ? samples.subarray(0, maxSamples) : samples +} /** Compute RMS energy per hop. */ function computeEnergy(samples: Float32Array): Float32Array { @@ -80,7 +86,8 @@ function findBeatPhase(onset: Float32Array, periodLag: number): number { * Pure math — no browser APIs. */ export function detectBeatGrid(samples: Float32Array, sampleRate: number): BeatGrid { - const energy = computeEnergy(samples) + const clipped = analysisSamples(samples, sampleRate) + const energy = computeEnergy(clipped) const onset = computeOnsetStrength(energy) // Lag range corresponding to [60, 200] BPM diff --git a/src/editor-shell/App.tsx b/src/editor-shell/App.tsx index 9f8b940..c82e32d 100644 --- a/src/editor-shell/App.tsx +++ b/src/editor-shell/App.tsx @@ -24,8 +24,8 @@ import { StoryboardFilmstrip } from './StoryboardFilmstrip' import { SlideSettingsDialog } from './SlideSettingsDialog' import { TitleSlideDialog } from './TitleSlideDialog' import { useProject } from './useProject' +import { useAudioClipAnalysis } from './useAudioClipAnalysis' import { useBeatGrid } from './useBeatGrid' -import { useLoudness } from './useLoudness' import { resolveEffectiveGainDb } from '../audio-analysis' export function App() { @@ -51,6 +51,7 @@ export function App() { themeName, setThemeName, updateAudioClips, + updateBeatGridCacheEntry, updateBeatGridPersist, updateLoudnessCache, loading, @@ -61,17 +62,22 @@ export function App() { recentProjects, } = project - const beatGrid = useBeatGrid({ + const { pendingBeatFilenames } = useAudioClipAnalysis({ audioClips, audioTracks, - onPersistChange: updateBeatGridPersist, - persisted: { beatGridCache, manualBeatGrid }, + beatGridCache, + loudnessCache, + manualBeatGrid, + onBeatGridCacheChange: updateBeatGridCacheEntry, + onLoudnessCacheChange: updateLoudnessCache, }) - useLoudness({ + const beatGrid = useBeatGrid({ + audioClips, audioTracks, - loudnessCache, - onPersistChange: updateLoudnessCache, + onPersistChange: updateBeatGridPersist, + pendingBeatFilenames, + persisted: { beatGridCache, manualBeatGrid }, }) const handleReorder = useCallback((fromIndex: number, toIndex: number) => { @@ -124,6 +130,10 @@ export function App() { [audioClips, audioTracks, loudnessCache], ) + const planBeatTimes = beatGrid.analysisStatus === 'analyzing' + ? undefined + : beatGrid.concatenatedBeatTimes + const renderPlan = useMemo( () => plan( filterIncluded(slides), @@ -131,9 +141,9 @@ export function App() { undefined, planAudioClips.length > 0 ? planAudioClips : undefined, beatGrid.effectiveBeatGrid, - beatGrid.concatenatedBeatTimes, + planBeatTimes, ), - [beatGrid.concatenatedBeatTimes, beatGrid.effectiveBeatGrid, globalSettings, planAudioClips, slides], + [beatGrid.effectiveBeatGrid, globalSettings, planAudioClips, planBeatTimes, slides], ) const totalFrames = renderPlan.totalFrames > 0 ? renderPlan.totalFrames : FPS const canvas = dimensionsForAspectRatio(aspectRatio) diff --git a/src/editor-shell/useAudioClipAnalysis.ts b/src/editor-shell/useAudioClipAnalysis.ts new file mode 100644 index 0000000..01190e6 --- /dev/null +++ b/src/editor-shell/useAudioClipAnalysis.ts @@ -0,0 +1,133 @@ +import { useEffect, useMemo } from 'react' +import type { AudioClip } from '../timeline-core/types' +import type { AudioTrack } from '../project-store' +import { decodeMono, detectBeatGrid } from '../beat-grid' +import type { BeatGrid, BeatGridCache } from '../beat-grid/types' +import { recommendedGainDb } from '../audio-analysis' +import { isLoudnessCacheEntryValid } from '../audio-analysis/gain' +import type { LoudnessCache } from '../audio-analysis/types' + +type Options = { + audioClips: AudioClip[] + audioTracks: AudioTrack[] + beatGridCache: BeatGridCache | undefined + loudnessCache: LoudnessCache | undefined + manualBeatGrid: BeatGrid | undefined + onBeatGridCacheChange: (entry: BeatGridCache) => void + onLoudnessCacheChange: (entry: LoudnessCache) => void +} + +function tracksForClips(audioClips: AudioClip[], audioTracks: AudioTrack[]): AudioTrack[] { + const filenames = new Set(audioClips.map((clip) => clip.filename)) + return audioTracks.filter((track) => filenames.has(track.filename)) +} + +export function useAudioClipAnalysis({ + audioClips, + audioTracks, + beatGridCache, + loudnessCache, + manualBeatGrid, + onBeatGridCacheChange, + onLoudnessCacheChange, +}: Options) { + const playlistTracks = useMemo( + () => tracksForClips(audioClips, audioTracks), + [audioClips, audioTracks], + ) + + const pendingBeatFilenames = useMemo( + () => audioClips + .map((clip) => clip.filename) + .filter((filename) => !manualBeatGrid && !beatGridCache?.[filename]), + [audioClips, beatGridCache, manualBeatGrid], + ) + + const pendingAnalysisFilenames = useMemo(() => { + return playlistTracks + .map((track) => track.filename) + .filter((filename) => { + const track = playlistTracks.find((entry) => entry.filename === filename) + if (!track) return false + const needsBeat = !manualBeatGrid && !beatGridCache?.[filename] + const needsLoudness = !isLoudnessCacheEntryValid( + loudnessCache?.[filename], + track.byteLength, + ) + return needsBeat || needsLoudness + }) + }, [beatGridCache, loudnessCache, manualBeatGrid, playlistTracks]) + + useEffect(() => { + if (pendingAnalysisFilenames.length === 0) return + + let cancelled = false + + async function analyzePending() { + const beatUpdates: BeatGridCache = {} + const loudnessUpdates: LoudnessCache = {} + + for (const filename of pendingAnalysisFilenames) { + if (cancelled) return + + const track = playlistTracks.find((entry) => entry.filename === filename) + if (!track) continue + + const needsBeat = !manualBeatGrid && !beatGridCache?.[filename] + const needsLoudness = !isLoudnessCacheEntryValid( + loudnessCache?.[filename], + track.byteLength, + ) + if (!needsBeat && !needsLoudness) continue + + try { + const response = await fetch(track.blobUrl) + const buffer = await response.arrayBuffer() + const { sampleRate, samples } = await decodeMono(buffer) + if (cancelled) return + + const maxSamples = sampleRate * 30 + const analysisSamples = samples.length > maxSamples + ? samples.subarray(0, maxSamples) + : samples + + if (needsBeat) { + beatUpdates[filename] = detectBeatGrid(samples, sampleRate) + } + if (needsLoudness) { + loudnessUpdates[filename] = { + byteLength: track.byteLength, + offsetDb: recommendedGainDb(analysisSamples), + } + } + } catch { + if (cancelled) return + } + } + + if (cancelled) return + if (Object.keys(beatUpdates).length > 0) { + onBeatGridCacheChange(beatUpdates) + } + if (Object.keys(loudnessUpdates).length > 0) { + onLoudnessCacheChange(loudnessUpdates) + } + } + + const deferId = window.setTimeout(() => { void analyzePending() }, 0) + return () => { + cancelled = true + window.clearTimeout(deferId) + } + // pendingAnalysisFilenames already reflects cache state; omit caches to avoid cancel/restart per file + // eslint-disable-next-line react-hooks/exhaustive-deps -- intentional + }, [ + manualBeatGrid, + onBeatGridCacheChange, + onLoudnessCacheChange, + pendingAnalysisFilenames, + playlistTracks, + ]) + + return { pendingBeatFilenames } +} diff --git a/src/editor-shell/useBeatGrid.ts b/src/editor-shell/useBeatGrid.ts index a8b612f..ee5e99e 100644 --- a/src/editor-shell/useBeatGrid.ts +++ b/src/editor-shell/useBeatGrid.ts @@ -1,9 +1,7 @@ -import { useCallback, useEffect, useMemo, useState } from 'react' +import { useCallback, useMemo } from 'react' import type { AudioClip } from '../timeline-core/types' import type { AudioTrack } from '../project-store' import { - decodeMono, - detectBeatGrid, manualBeatGridFromBpm, resolveConcatenatedBeatTimes, resolveEffectiveBeatGrid, @@ -25,6 +23,7 @@ type Options = { audioTracks: AudioTrack[] onPersistChange: (update: PersistedBeatGrid) => void persisted: PersistedBeatGrid + pendingBeatFilenames: string[] } export function useBeatGrid({ @@ -32,9 +31,8 @@ export function useBeatGrid({ audioTracks, onPersistChange, persisted, + pendingBeatFilenames, }: Options) { - const [analysisFailedForFilename, setAnalysisFailedForFilename] = useState(null) - const clipTimings = useMemo( () => audioClips.flatMap((clip) => { const track = audioTracks.find((entry) => entry.filename === clip.filename) @@ -64,33 +62,15 @@ export function useBeatGrid({ [clipTimings, persisted.beatGridCache, persisted.manualBeatGrid], ) - const pendingFilenames = useMemo( - () => audioClips - .map((clip) => clip.filename) - .filter((filename) => ( - !persisted.manualBeatGrid && !persisted.beatGridCache?.[filename] - )), - [audioClips, persisted.beatGridCache, persisted.manualBeatGrid], - ) - const analysisStatus = useMemo((): BeatGridAnalysisStatus => { if (audioClips.length === 0) return 'idle' if (persisted.manualBeatGrid) return 'ready' - if (pendingFilenames.length === 0) return 'ready' - if (analysisFailedForFilename && pendingFilenames.includes(analysisFailedForFilename)) { - return 'error' - } + if (pendingBeatFilenames.length === 0) return 'ready' return 'analyzing' - }, [ - analysisFailedForFilename, - audioClips.length, - pendingFilenames, - persisted.manualBeatGrid, - ]) + }, [audioClips.length, pendingBeatFilenames.length, persisted.manualBeatGrid]) const setManualBeatGrid = useCallback((grid: BeatGrid | undefined) => { onPersistChange({ manualBeatGrid: grid }) - setAnalysisFailedForFilename(null) }, [onPersistChange]) const clearManualBeatGrid = useCallback(() => { @@ -105,49 +85,6 @@ export function useBeatGrid({ setManualBeatGrid(tapToBpm(tapTimestampsMs)) }, [setManualBeatGrid]) - useEffect(() => { - if (persisted.manualBeatGrid || pendingFilenames.length === 0) { - return - } - - let cancelled = false - - async function analyzePending() { - const nextCache: BeatGridCache = { ...persisted.beatGridCache } - - for (const filename of pendingFilenames) { - const track = audioTracks.find((entry) => entry.filename === filename) - if (!track) continue - - try { - const response = await fetch(track.blobUrl) - const buffer = await response.arrayBuffer() - const { sampleRate, samples } = await decodeMono(buffer) - const grid = detectBeatGrid(samples, sampleRate) - if (cancelled) return - nextCache[filename] = grid - } catch { - if (cancelled) return - setAnalysisFailedForFilename(filename) - return - } - } - - if (!cancelled) { - onPersistChange({ beatGridCache: nextCache }) - } - } - - void analyzePending() - return () => { cancelled = true } - }, [ - audioTracks, - onPersistChange, - pendingFilenames, - persisted.beatGridCache, - persisted.manualBeatGrid, - ]) - return { analysisStatus, applyManualBpm, diff --git a/src/editor-shell/useProject.ts b/src/editor-shell/useProject.ts index 2cb29bc..cda2b1f 100644 --- a/src/editor-shell/useProject.ts +++ b/src/editor-shell/useProject.ts @@ -121,8 +121,10 @@ export function useProject({ onFolderLoaded }: Options = {}) { setLoading(true) setError(null) try { - const enumerated = await enumerateFolder(handle) - const nextAudioTracks = await enumerateAudioTracks(handle) + const [enumerated, nextAudioTracks] = await Promise.all([ + enumerateFolder(handle), + enumerateAudioTracks(handle), + ]) const restoredSettings = savedData?.globalSettings ?? DEFAULT_GLOBAL_SETTINGS const restoredThemeName = savedData?.themeName ?? null const restoredClips = filterValidAudioClips( @@ -256,8 +258,12 @@ export function useProject({ onFolderLoaded }: Options = {}) { setAudioClips(clips) }, []) - const updateLoudnessCache = useCallback((cache: LoudnessCache) => { - setLoudnessCache(cache) + const updateLoudnessCache = useCallback((entry: LoudnessCache) => { + setLoudnessCache((previous) => ({ ...previous, ...entry })) + }, []) + + const updateBeatGridCacheEntry = useCallback((entry: BeatGridCache) => { + setBeatGridCache((previous) => ({ ...previous, ...entry })) }, []) const updateBeatGridPersist = useCallback((update: { @@ -323,6 +329,7 @@ export function useProject({ onFolderLoaded }: Options = {}) { themeName, setThemeName, updateAudioClips, + updateBeatGridCacheEntry, updateBeatGridPersist, updateLoudnessCache, loading, diff --git a/src/project-store/audio-loader.ts b/src/project-store/audio-loader.ts index 249e0e6..91cfc36 100644 --- a/src/project-store/audio-loader.ts +++ b/src/project-store/audio-loader.ts @@ -41,14 +41,15 @@ export async function enumerateAudioTracks( const createdUrls: string[] = [] try { - const tracks: AudioTrack[] = [] - for (const filename of sorted) { - const file = filesByName.get(filename)! - const blobUrl = URL.createObjectURL(file) - createdUrls.push(blobUrl) - const durationInFrames = await getAudioDurationFrames(file) - tracks.push({ blobUrl, byteLength: file.size, durationInFrames, filename }) - } + const tracks = await Promise.all( + sorted.map(async (filename) => { + const file = filesByName.get(filename)! + const blobUrl = URL.createObjectURL(file) + createdUrls.push(blobUrl) + const durationInFrames = await getAudioDurationFrames(file) + return { blobUrl, byteLength: file.size, durationInFrames, filename } + }), + ) return tracks } catch (error) { createdUrls.forEach((url) => URL.revokeObjectURL(url)) diff --git a/src/project-store/media-loader.ts b/src/project-store/media-loader.ts index ae877d7..c77872a 100644 --- a/src/project-store/media-loader.ts +++ b/src/project-store/media-loader.ts @@ -61,13 +61,14 @@ export async function enumerateFolder( const createdUrls: string[] = [] try { - const slides: MediaSlide[] = [] - for (const filename of sorted) { - const file = filesByName.get(filename)! - const slide = await createMediaSlideFromFile(file) - createdUrls.push(slide.blobUrl) - slides.push(slide) - } + const slides = await Promise.all( + sorted.map(async (filename) => { + const file = filesByName.get(filename)! + const slide = await createMediaSlideFromFile(file) + createdUrls.push(slide.blobUrl) + return slide + }), + ) return slides } catch (error) { createdUrls.forEach((url) => URL.revokeObjectURL(url)) From 0958387b76dad1d70e83afffce1ba85c1543c2f9 Mon Sep 17 00:00:00 2001 From: cbaucom Date: Mon, 22 Jun 2026 11:05:24 -0400 Subject: [PATCH 3/5] Fix theme switch freeze on long cut timelines. Use absolute Sequence placement for cut-only plans instead of TransitionSeries, binary-search beat snap, and startTransition when applying themes. Co-authored-by: Cursor --- src/beat-grid/nudge-position.test.ts | 6 +++ src/beat-grid/nudge-position.ts | 28 ++++++++-- src/composition/SlideshowComposition.tsx | 69 +++++++++++++++++------- src/editor-shell/App.tsx | 10 ++-- 4 files changed, 87 insertions(+), 26 deletions(-) diff --git a/src/beat-grid/nudge-position.test.ts b/src/beat-grid/nudge-position.test.ts index e8a1eaa..97189f1 100644 --- a/src/beat-grid/nudge-position.test.ts +++ b/src/beat-grid/nudge-position.test.ts @@ -14,4 +14,10 @@ describe('nudgeSlideEndFrame', () => { const duration = nudgeSlideEndFrame(60, 40, BEAT_TIMES, 'medium', FPS) expect(60 + duration).toBe(105) }) + + it('matches linear scan on a long concatenated beat grid', () => { + const longBeatTimes = Array.from({ length: 1200 }, (_, index) => index * 0.5) + const duration = nudgeSlideEndFrame(900, 50, longBeatTimes, 'medium', FPS) + expect(900 + duration).toBe(945) + }) }) diff --git a/src/beat-grid/nudge-position.ts b/src/beat-grid/nudge-position.ts index 92ea34b..1daca45 100644 --- a/src/beat-grid/nudge-position.ts +++ b/src/beat-grid/nudge-position.ts @@ -6,6 +6,20 @@ const ENERGY_MULTIPLIER: Record = { punchy: 0.67, } +function lowerBoundBeatIndex(beatTimesSecs: number[], secs: number): number { + let low = 0 + let high = beatTimesSecs.length + while (low < high) { + const mid = (low + high) >> 1 + if (beatTimesSecs[mid] < secs) { + low = mid + 1 + } else { + high = mid + } + } + return low +} + export function nudgeSlideEndFrame( startFrame: number, targetDurationFrames: number, @@ -19,11 +33,19 @@ export function nudgeSlideEndFrame( const startSecs = startFrame / fps const targetEndSecs = startSecs + scaledTargetFrames / fps - let nearestBeatSecs = beatTimesSecs[0] + const startBeatIndex = lowerBoundBeatIndex(beatTimesSecs, startSecs) + if (startBeatIndex >= beatTimesSecs.length) { + const endFrame = Math.round(beatTimesSecs[beatTimesSecs.length - 1] * fps) + return Math.max(1, endFrame - startFrame) + } + + const nearestBeatIndex = lowerBoundBeatIndex(beatTimesSecs, targetEndSecs) + let nearestBeatSecs = beatTimesSecs[startBeatIndex] let minDistance = Math.abs(nearestBeatSecs - targetEndSecs) - for (const beatSecs of beatTimesSecs) { - if (beatSecs < startSecs) continue + for (const candidateIndex of [nearestBeatIndex - 1, nearestBeatIndex, nearestBeatIndex + 1]) { + if (candidateIndex < startBeatIndex || candidateIndex >= beatTimesSecs.length) continue + const beatSecs = beatTimesSecs[candidateIndex] const distance = Math.abs(beatSecs - targetEndSecs) if (distance < minDistance) { minDistance = distance diff --git a/src/composition/SlideshowComposition.tsx b/src/composition/SlideshowComposition.tsx index 3ee3a33..0dc7f47 100644 --- a/src/composition/SlideshowComposition.tsx +++ b/src/composition/SlideshowComposition.tsx @@ -51,6 +51,51 @@ function getPresentation(type: TransitionType): TransitionPresentation { } } +function planUsesTimedTransitions(entries: RenderPlanEntry[]): boolean { + return entries.some( + (entry) => entry.transitionIn !== undefined && entry.transitionIn.durationInFrames > 0, + ) +} + +function SlideEntryView({ entry }: { entry: RenderPlanEntry }) { + return isTitleSlide(entry.slide) + ? + : +} + +function AbsoluteTimeline({ entries }: { entries: RenderPlanEntry[] }) { + return entries.map((entry) => ( + + + + )) +} + +function TransitionSeriesTimeline({ entries }: { entries: RenderPlanEntry[] }) { + return ( + + {entries.map((entry) => ( + + {entry.transitionIn && entry.transitionIn.durationInFrames > 0 ? ( + + ) : null} + + + + + ))} + + ) +} + export function SlideshowComposition({ plan }: SlideshowProps) { if (plan.entries.length === 0) { return ( @@ -65,25 +110,11 @@ export function SlideshowComposition({ plan }: SlideshowProps) { {plan.audioSegments && plan.duckingEnvelope ? ( ) : null} - - {plan.entries.map((entry) => ( - - {entry.transitionIn && entry.transitionIn.durationInFrames > 0 && ( - - )} - - {isTitleSlide(entry.slide) ? ( - - ) : ( - - )} - - - ))} - + {planUsesTimedTransitions(plan.entries) ? ( + + ) : ( + + )} ) } diff --git a/src/editor-shell/App.tsx b/src/editor-shell/App.tsx index c82e32d..ecf7940 100644 --- a/src/editor-shell/App.tsx +++ b/src/editor-shell/App.tsx @@ -1,4 +1,4 @@ -import { useCallback, useMemo, useRef, useState } from 'react' +import { useCallback, useMemo, useRef, useState, startTransition } from 'react' import type { PlayerRef } from '@remotion/player' import { Button } from '@/components/ui/button' import type { Slide, TitleSlide } from '../timeline-core/types' @@ -96,9 +96,11 @@ export function App() { const handleThemeChange = useCallback((name: ThemeName) => { const themeSettings = applyTheme(name) - setGlobalSettings(themeSettings) - setThemeName(name) - setSlides(prev => applyImageDuration(prev, themeSettings.imageDurationSecs)) + startTransition(() => { + setThemeName(name) + setGlobalSettings((previous) => ({ ...previous, ...themeSettings })) + setSlides((previous) => applyImageDuration(previous, themeSettings.imageDurationSecs)) + }) }, [setGlobalSettings, setSlides, setThemeName]) const handleSlideOverride = useCallback((id: string, overrides: SlideOverrides | undefined) => { From ba86d0d9ec523b51af16a30045d5bff37383d52b Mon Sep 17 00:00:00 2001 From: cbaucom Date: Mon, 22 Jun 2026 11:07:52 -0400 Subject: [PATCH 4/5] Keep theme toggles responsive on long timelines. Drop TransitionSeries for all previews, defer replanning, skip mass slide updates on theme change, and disable premount on large plans. Co-authored-by: Cursor --- src/composition/SlideshowComposition.tsx | 150 ++++++++++------------- src/editor-shell/App.tsx | 21 +++- src/sequence-planner/planner.test.ts | 13 +- src/sequence-planner/planner.ts | 7 +- 4 files changed, 97 insertions(+), 94 deletions(-) diff --git a/src/composition/SlideshowComposition.tsx b/src/composition/SlideshowComposition.tsx index 0dc7f47..66509fc 100644 --- a/src/composition/SlideshowComposition.tsx +++ b/src/composition/SlideshowComposition.tsx @@ -1,12 +1,8 @@ import React, { useCallback } from 'react' import { AbsoluteFill, Img, interpolate, Sequence, useCurrentFrame } from 'remotion' import { Audio, Video } from '@remotion/media' -import { TransitionSeries, linearTiming } from '@remotion/transitions' -import { fade } from '@remotion/transitions/fade' -import type { TransitionPresentation, TransitionPresentationComponentProps } from '@remotion/transitions' import { loadFont } from '@remotion/google-fonts/Inter' -import type { AudioSegment, RenderPlanEntry, RenderPlan } from '../sequence-planner/types' -import type { TransitionType } from '../timeline-core/settings' +import type { AudioSegment, RenderPlanEntry, RenderPlan, TransitionSpec } from '../sequence-planner/types' import { isTitleSlide } from '../timeline-core/types' import type { MediaSlide, TitleSlide } from '../timeline-core/types' import { dbToLinear, volumeAtFrame } from './soundtrackVolume' @@ -16,86 +12,83 @@ type TitleRenderPlanEntry = Omit & { slide: TitleSlide const { fontFamily } = loadFont('normal', { weights: ['400', '700'], subsets: ['latin'] }) +const LARGE_TIMELINE_ENTRY_COUNT = 60 + export type SlideshowProps = { plan: RenderPlan } -// Custom dip-to-black presentation: both scenes fade through solid black. -type DipToBlackProps = Record +function transitionOpacity(frame: number, transitionIn: TransitionSpec | undefined): number { + if (!transitionIn || transitionIn.durationInFrames <= 0) return 1 -const DipToBlackComponent: React.FC> = ({ + const duration = transitionIn.durationInFrames + if (transitionIn.type === 'dip-to-black') { + const midpoint = duration / 2 + if (frame < midpoint) return 0 + return interpolate(frame, [midpoint, duration], [0, 1], { + extrapolateLeft: 'clamp', + extrapolateRight: 'clamp', + }) + } + + return interpolate(frame, [0, duration], [0, 1], { + extrapolateLeft: 'clamp', + extrapolateRight: 'clamp', + }) +} + +function TransitionLayer({ children, - presentationProgress, - presentationDirection, -}) => { - const opacity = - presentationDirection === 'exiting' ? presentationProgress : 1 - presentationProgress + transitionIn, +}: { + children: React.ReactNode + transitionIn: TransitionSpec | undefined +}) { + const frame = useCurrentFrame() + const opacity = transitionOpacity(frame, transitionIn) + const showDipBlack = transitionIn?.type === 'dip-to-black' + && transitionIn.durationInFrames > 0 + && frame < transitionIn.durationInFrames + return ( - {children} - + {showDipBlack ? : null} + {children} ) } -function dipToBlack(): TransitionPresentation { - return { component: DipToBlackComponent, props: {} } -} +function SlideEntryView({ entry }: { entry: RenderPlanEntry }) { + const content = isTitleSlide(entry.slide) + ? + : -// eslint-disable-next-line @typescript-eslint/no-explicit-any -function getPresentation(type: TransitionType): TransitionPresentation { - switch (type) { - case 'crossfade': return fade() - case 'dip-to-black': return dipToBlack() - case 'cut': return fade() // unreachable: cut entries have durationInFrames=0 + if (!entry.transitionIn || entry.transitionIn.durationInFrames <= 0) { + return content } -} -function planUsesTimedTransitions(entries: RenderPlanEntry[]): boolean { - return entries.some( - (entry) => entry.transitionIn !== undefined && entry.transitionIn.durationInFrames > 0, + return ( + + {content} + ) } -function SlideEntryView({ entry }: { entry: RenderPlanEntry }) { - return isTitleSlide(entry.slide) - ? - : -} - function AbsoluteTimeline({ entries }: { entries: RenderPlanEntry[] }) { + const premountFor = entries.length > LARGE_TIMELINE_ENTRY_COUNT ? 0 : 30 + return entries.map((entry) => ( )) } -function TransitionSeriesTimeline({ entries }: { entries: RenderPlanEntry[] }) { - return ( - - {entries.map((entry) => ( - - {entry.transitionIn && entry.transitionIn.durationInFrames > 0 ? ( - - ) : null} - - - - - ))} - - ) -} - export function SlideshowComposition({ plan }: SlideshowProps) { if (plan.entries.length === 0) { return ( @@ -110,11 +103,7 @@ export function SlideshowComposition({ plan }: SlideshowProps) { {plan.audioSegments && plan.duckingEnvelope ? ( ) : null} - {planUsesTimedTransitions(plan.entries) ? ( - - ) : ( - - )} + ) } @@ -165,43 +154,42 @@ function TitleSlideView({ entry }: { entry: TitleRenderPlanEntry }) {
{slide.heading}
- {slide.subtext && ( + {slide.subtext ? (
{slide.subtext}
- )} + ) : null}
) } function MediaSlideView({ entry }: { entry: MediaRenderPlanEntry }) { - const { fitMode, kenBurns, durationInFrames, slide, videoVolume } = entry + const { durationInFrames, fitMode, kenBurns, slide, videoVolume } = entry const frame = useCurrentFrame() - // Compute Ken Burns transform (images only; null for videos). let kenBurnsStyle: React.CSSProperties = {} if (kenBurns) { const lastFrame = Math.max(1, durationInFrames - 1) @@ -216,7 +204,6 @@ function MediaSlideView({ entry }: { entry: MediaRenderPlanEntry }) { } if (slide.type === 'video') { - // Videos are always letterboxed (contain), never cropped. return ( @@ -263,23 +248,22 @@ function MediaSlideView({ entry }: { entry: MediaRenderPlanEntry }) { ) } - // Default: cover (crop-to-fill) with Ken Burns. return ( diff --git a/src/editor-shell/App.tsx b/src/editor-shell/App.tsx index ecf7940..b9176c4 100644 --- a/src/editor-shell/App.tsx +++ b/src/editor-shell/App.tsx @@ -1,4 +1,4 @@ -import { useCallback, useMemo, useRef, useState, startTransition } from 'react' +import { useCallback, useDeferredValue, useMemo, useRef, useState, startTransition } from 'react' import type { PlayerRef } from '@remotion/player' import { Button } from '@/components/ui/button' import type { Slide, TitleSlide } from '../timeline-core/types' @@ -99,9 +99,8 @@ export function App() { startTransition(() => { setThemeName(name) setGlobalSettings((previous) => ({ ...previous, ...themeSettings })) - setSlides((previous) => applyImageDuration(previous, themeSettings.imageDurationSecs)) }) - }, [setGlobalSettings, setSlides, setThemeName]) + }, [setGlobalSettings, setThemeName]) const handleSlideOverride = useCallback((id: string, overrides: SlideOverrides | undefined) => { setSlides(prev => prev.map(s => s.id === id ? { ...s, overrides } : s)) @@ -136,16 +135,20 @@ export function App() { ? undefined : beatGrid.concatenatedBeatTimes + const deferredSlides = useDeferredValue(slides) + const deferredGlobalSettings = useDeferredValue(globalSettings) + const isReplanning = deferredSlides !== slides || deferredGlobalSettings !== globalSettings + const renderPlan = useMemo( () => plan( - filterIncluded(slides), - globalSettings, + filterIncluded(deferredSlides), + deferredGlobalSettings, undefined, planAudioClips.length > 0 ? planAudioClips : undefined, beatGrid.effectiveBeatGrid, planBeatTimes, ), - [beatGrid.effectiveBeatGrid, globalSettings, planAudioClips, planBeatTimes, slides], + [beatGrid.effectiveBeatGrid, deferredGlobalSettings, deferredSlides, planAudioClips, planBeatTimes], ) const totalFrames = renderPlan.totalFrames > 0 ? renderPlan.totalFrames : FPS const canvas = dimensionsForAspectRatio(aspectRatio) @@ -208,6 +211,12 @@ export function App() { )} + {isReplanning ? ( +
+ Updating preview… +
+ ) : null} +
{folderOpen ? ( Date: Mon, 22 Jun 2026 12:23:56 -0400 Subject: [PATCH 5/5] Fix plan() infinite loop when beat-synced slides are shorter than crossfade overlap. Cap transition overlap to actual slide duration and render only active composition entries so theme toggles stay responsive on long beat-synced timelines. Co-authored-by: Cursor --- scripts/playwright-theme-toggle-smoke.mjs | 154 ++++++++++++++++++++++ src/composition/SlideshowComposition.tsx | 29 ++-- src/sequence-planner/planner.perf.test.ts | 39 ++++++ src/sequence-planner/planner.test.ts | 16 +++ src/sequence-planner/planner.ts | 6 +- 5 files changed, 235 insertions(+), 9 deletions(-) create mode 100644 scripts/playwright-theme-toggle-smoke.mjs create mode 100644 src/sequence-planner/planner.perf.test.ts diff --git a/scripts/playwright-theme-toggle-smoke.mjs b/scripts/playwright-theme-toggle-smoke.mjs new file mode 100644 index 0000000..d646cb0 --- /dev/null +++ b/scripts/playwright-theme-toggle-smoke.mjs @@ -0,0 +1,154 @@ +// Smoke: theme toggle with heavy slideshow. Run: pnpm dev && node scripts/playwright-theme-toggle-smoke.mjs +import fs from 'node:fs' +import path from 'node:path' +import { spawnSync } from 'node:child_process' + +const APP_URL = process.env.APP_URL ?? 'http://localhost:5173/' +const MAX_TOGGLE_MS = Number(process.env.MAX_TOGGLE_MS ?? 3000) + +function run(args) { + const result = spawnSync('npx', ['playwright-cli', ...args], { + cwd: process.cwd(), + encoding: 'utf8', + maxBuffer: 20 * 1024 * 1024, + }) + if (result.stdout) process.stdout.write(result.stdout) + if (result.stderr) process.stderr.write(result.stderr) + return result.status ?? 1 +} + +const demo = path.resolve('test-fixtures/demo') +const photo = fs.readFileSync(path.join(demo, 'photo.jpg')).toString('base64') +const photo2 = fs.readFileSync(path.join(demo, 'photo2.jpg')).toString('base64') +const theme = fs.readFileSync(path.join(demo, 'theme.wav')).toString('base64') +const slides = Array.from({ length: 80 }, (_, index) => ({ + durationInFrames: 90, + excluded: false, + filename: index % 2 === 0 ? 'photo.jpg' : 'photo2.jpg', + id: `slide-${index}`, + type: 'image', +})) +const slideshow = { + audioClips: [ + { filename: 'theme.wav' }, + { filename: 'theme.wav' }, + { filename: 'theme.wav' }, + ], + globalSettings: { + beatSync: true, + fitMode: 'cover', + imageDurationSecs: 4, + kenBurns: true, + transitionType: 'crossfade', + }, + schemaVersion: 1, + slides, +} +const payload = { + 'photo.jpg': photo, + 'photo2.jpg': photo2, + 'theme.wav': theme, + 'slideshow.json': Buffer.from(JSON.stringify(slideshow, null, 2)).toString('base64'), +} + +const code = `async page => { + const payload = ${JSON.stringify(payload)}; + const maxToggleMs = ${MAX_TOGGLE_MS}; + + await page.goto(${JSON.stringify(APP_URL)}); + await page.setViewportSize({ width: 1280, height: 800 }); + + await page.evaluate((files) => { + function decodeBase64(base64) { + const binary = atob(base64); + const bytes = new Uint8Array(binary.length); + for (let index = 0; index < binary.length; index++) bytes[index] = binary.charCodeAt(index); + return bytes; + } + class MockFileHandle { + constructor(name, base64) { + this.kind = 'file'; + this.name = name; + this._base64 = base64; + } + async getFile() { + const bytes = decodeBase64(this._base64); + const type = this.name.endsWith('.jpg') ? 'image/jpeg' : this.name.endsWith('.wav') ? 'audio/wav' : 'application/json'; + return new File([bytes], this.name, { type, lastModified: 1781145972852 }); + } + async createWritable() { + const handle = this; + return { + write: async (data) => { + const buffer = data instanceof Blob ? await data.arrayBuffer() : data; + handle._base64 = btoa(String.fromCharCode(...new Uint8Array(buffer))); + }, + close: async () => {}, + }; + } + } + class MockDirHandle { + constructor(files) { + this.name = 'demo'; + this._files = files; + } + async getFileHandle(name, options) { + const handle = this._files.get(name); + if (handle) return handle; + if (options?.create) { + const created = new MockFileHandle(name, ''); + this._files.set(name, created); + return created; + } + throw new DOMException('NotFoundError'); + } + async *values() { + for (const handle of this._files.values()) yield handle; + } + } + const handles = new Map( + Object.entries(files).map(([name, base64]) => [name, new MockFileHandle(name, base64)]), + ); + window.showDirectoryPicker = async () => new MockDirHandle(handles); + }, payload); + + await page.getByRole('button', { name: 'Open Folder' }).click(); + await page.getByText('photo2.jpg').first().waitFor({ timeout: 15000 }); + + const analyzing = page.getByText('Analyzing soundtrack'); + if (await analyzing.first().isVisible().catch(() => false)) { + await analyzing.first().waitFor({ state: 'hidden', timeout: 30000 }); + } + + await page.getByRole('radio', { name: 'Classic', exact: true }).waitFor({ timeout: 10000 }); + + async function toggleTheme(name) { + const start = Date.now(); + await page.getByRole('radio', { name, exact: true }).click(); + const banner = page.getByText('Updating preview'); + if (await banner.isVisible().catch(() => false)) { + await banner.waitFor({ state: 'hidden', timeout: maxToggleMs }); + } + const ms = Date.now() - start; + console.log('toggle ' + name + ' ms=' + ms); + if (ms > maxToggleMs) { + throw new Error('toggle ' + name + ' took ' + ms + 'ms'); + } + } + + await toggleTheme('Energetic'); + await toggleTheme('Classic'); + await toggleTheme('Energetic'); + console.log('PASS theme toggle smoke'); +}` + +let status = run(['open', APP_URL, '--browser=chrome']) +if (status !== 0) process.exit(status) + +status = run(['run-code', code]) +run(['close']) +if (status !== 0) { + console.error('FAIL theme toggle smoke (exit ' + status + ')') + process.exit(status) +} +console.log('PASS theme toggle smoke complete') diff --git a/src/composition/SlideshowComposition.tsx b/src/composition/SlideshowComposition.tsx index 66509fc..681ab99 100644 --- a/src/composition/SlideshowComposition.tsx +++ b/src/composition/SlideshowComposition.tsx @@ -12,8 +12,6 @@ type TitleRenderPlanEntry = Omit & { slide: TitleSlide const { fontFamily } = loadFont('normal', { weights: ['400', '700'], subsets: ['latin'] }) -const LARGE_TIMELINE_ENTRY_COUNT = 60 - export type SlideshowProps = { plan: RenderPlan } @@ -74,21 +72,36 @@ function SlideEntryView({ entry }: { entry: RenderPlanEntry }) { ) } -function AbsoluteTimeline({ entries }: { entries: RenderPlanEntry[] }) { - const premountFor = entries.length > LARGE_TIMELINE_ENTRY_COUNT ? 0 : 30 +function findActiveEntries(entries: RenderPlanEntry[], frame: number): RenderPlanEntry[] { + const active: RenderPlanEntry[] = [] + for (const entry of entries) { + const endFrame = entry.startFrame + entry.durationInFrames + if (frame >= entry.startFrame && frame < endFrame) { + active.push(entry) + } + } + return active +} - return entries.map((entry) => ( +function ActiveTimeline({ entries, frame }: { entries: RenderPlanEntry[]; frame: number }) { + const activeEntries = findActiveEntries(entries, frame) + return activeEntries.map((entry) => ( )) } +function FrameTimeline({ entries }: { entries: RenderPlanEntry[] }) { + const frame = useCurrentFrame() + return +} + export function SlideshowComposition({ plan }: SlideshowProps) { if (plan.entries.length === 0) { return ( @@ -103,7 +116,7 @@ export function SlideshowComposition({ plan }: SlideshowProps) { {plan.audioSegments && plan.duckingEnvelope ? ( ) : null} - + ) } diff --git a/src/sequence-planner/planner.perf.test.ts b/src/sequence-planner/planner.perf.test.ts new file mode 100644 index 0000000..5c93c42 --- /dev/null +++ b/src/sequence-planner/planner.perf.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, it } from 'vitest' +import { applyTheme } from '../timeline-core/settings' +import type { MediaSlide } from '../timeline-core/types' +import { plan } from './planner' + +function makeSlide(index: number): MediaSlide { + return { + blobUrl: `blob:${index}`, + durationInFrames: 90, + excluded: false, + filename: `photo-${index}.jpg`, + id: `slide-${index}`, + type: 'image', + } +} + +const LONG_AUDIO = [{ blobUrl: 'blob:audio', durationInFrames: 18000, gainDb: 0 }] +const CONCATENATED_BEATS = Array.from({ length: 1200 }, (_, index) => index * 0.5) + +describe('plan — performance guard', () => { + it('plans a long looped timeline within 500ms', () => { + const slides = Array.from({ length: 80 }, (_, index) => makeSlide(index)) + const classic = applyTheme('classic') + const energetic = applyTheme('energetic') + + const classicStart = performance.now() + const classicPlan = plan(slides, classic, undefined, LONG_AUDIO, undefined, CONCATENATED_BEATS) + const classicMs = performance.now() - classicStart + + const energeticStart = performance.now() + const energeticPlan = plan(slides, energetic, undefined, LONG_AUDIO, undefined, CONCATENATED_BEATS) + const energeticMs = performance.now() - energeticStart + + expect(classicPlan.entries.length).toBeGreaterThan(100) + expect(energeticPlan.entries.length).toBeGreaterThan(classicPlan.entries.length) + expect(classicMs).toBeLessThan(500) + expect(energeticMs).toBeLessThan(500) + }) +}) diff --git a/src/sequence-planner/planner.test.ts b/src/sequence-planner/planner.test.ts index 2941661..0193dcb 100644 --- a/src/sequence-planner/planner.test.ts +++ b/src/sequence-planner/planner.test.ts @@ -640,4 +640,20 @@ describe('plan — concatenated beat times (multi-clip)', () => { expect(secondEntry.startFrame).toBe(30) expect(secondEntry.durationInFrames).toBe(63) }) + + it('terminates when beat-snapped durations are shorter than crossfade overlap', () => { + const concatenatedBeatTimes = Array.from({ length: 1200 }, (_, index) => index * 0.5) + const slides = Array.from({ length: 20 }, (_, index) => makeSlide(`slide-${index}`, 'image', 90)) + const classic = { ...BEAT_SYNC_ON, transitionType: 'crossfade' as const } + const result = plan( + slides, + classic, + undefined, + [{ blobUrl: 'blob:audio', durationInFrames: 18000 }], + undefined, + concatenatedBeatTimes, + ) + expect(result.entries.length).toBeLessThan(5000) + expect(result.totalFrames).toBe(18000) + }) }) diff --git a/src/sequence-planner/planner.ts b/src/sequence-planner/planner.ts index 949031b..fc7c6ba 100644 --- a/src/sequence-planner/planner.ts +++ b/src/sequence-planner/planner.ts @@ -226,7 +226,11 @@ export function plan( const hasNextInPass = index < slides.length - 1 const isFullSlide = durationInFrames === fullDuration if (hasNextInPass && isFullSlide && cursor + durationInFrames < totalFrames) { - cursor += durationInFrames - effectiveTrans[index + 1] + const nextTransitionOverlap = Math.min( + effectiveTrans[index + 1], + Math.floor(durationInFrames / 2), + ) + cursor += Math.max(1, durationInFrames - nextTransitionOverlap) } else { cursor += durationInFrames }