From 4f09cfc3fa3f77b0bc1348229c0d41ae5c687937 Mon Sep 17 00:00:00 2001 From: dazzatronus Date: Mon, 13 Jul 2026 14:55:15 +1000 Subject: [PATCH] feat: support animated GIF image assets --- THIRD_PARTY_NOTICES | 47 ++ package.json | 3 +- src/components/canvas/players/audio-player.ts | 118 +++-- .../canvas/players/caption-player.ts | 93 +++- src/components/canvas/players/image-player.ts | 142 ++++- .../canvas/players/image-to-video-player.ts | 14 +- src/components/canvas/players/luma-player.ts | 76 ++- src/components/canvas/players/player.ts | 56 +- .../canvas/players/rich-caption-player.ts | 488 +++++++++--------- src/components/canvas/players/svg-player.ts | 1 + src/components/canvas/players/video-player.ts | 128 ++++- .../timeline/media-thumbnail-renderer.ts | 21 +- src/components/timeline/timeline.ts | 22 +- src/core/commands/add-clip-command.ts | 6 +- src/core/commands/types.ts | 2 +- src/core/debug/state-assertions.ts | 2 +- src/core/edit-session.ts | 116 +++-- src/core/loaders/asset-loader.ts | 143 ++++- src/core/loaders/gif-image-load-error.ts | 6 + src/core/loaders/gif-image-source.ts | 441 ++++++++++++++++ src/core/loaders/gif-url.ts | 24 + src/core/player-reconciler.ts | 69 ++- src/core/resolver.ts | 37 +- src/core/timing-manager.ts | 49 +- src/core/timing/resolver.ts | 88 ++-- src/core/timing/types.ts | 27 +- test-package.js | 21 + tests/asset-loader.test.ts | 12 + tests/edit-clip-operations.test.ts | 3 + tests/edit-load.test.ts | 3 + tests/edit-merge-fields.test.ts | 3 + tests/edit-timing.test.ts | 20 +- tests/gif-asset-loader.test.ts | 143 +++++ tests/gif-image-source.test.ts | 253 +++++++++ tests/gif-url.test.ts | 29 ++ tests/media-auto-timing.test.ts | 81 +++ tests/media-player-fallback.test.ts | 325 +++++++++++- tests/media-thumbnail-renderer.test.ts | 23 + tests/player-reconciler-loads.test.ts | 127 +++++ tests/player-sync.test.ts | 114 +++- tests/resolver.test.ts | 49 +- tests/rich-caption-player.test.ts | 47 +- vite.shared.ts | 1 + 43 files changed, 2923 insertions(+), 550 deletions(-) create mode 100644 THIRD_PARTY_NOTICES create mode 100644 src/core/loaders/gif-image-load-error.ts create mode 100644 src/core/loaders/gif-image-source.ts create mode 100644 src/core/loaders/gif-url.ts create mode 100644 tests/gif-asset-loader.test.ts create mode 100644 tests/gif-image-source.test.ts create mode 100644 tests/gif-url.test.ts create mode 100644 tests/media-auto-timing.test.ts create mode 100644 tests/player-reconciler-loads.test.ts diff --git a/THIRD_PARTY_NOTICES b/THIRD_PARTY_NOTICES new file mode 100644 index 00000000..9327c9e2 --- /dev/null +++ b/THIRD_PARTY_NOTICES @@ -0,0 +1,47 @@ +This product includes portions of PixiJS and gifuct-js. + +PixiJS +Copyright (c) 2013-2023 Mathew Groves, Chad Engler + +The MIT License + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +gifuct-js +Copyright (c) 2015 Matt Way + +The MIT License (MIT) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/package.json b/package.json index a6ba9ae2..340cd1b4 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,8 @@ "dist/shotstack-studio.es.js", "dist/internal.umd.js", "dist/internal.es.js", - "dist/**/*.d.ts" + "dist/**/*.d.ts", + "THIRD_PARTY_NOTICES" ], "keywords": [ "shotstack", diff --git a/src/components/canvas/players/audio-player.ts b/src/components/canvas/players/audio-player.ts index 56ebeefa..4abd6ab6 100644 --- a/src/components/canvas/players/audio-player.ts +++ b/src/components/canvas/players/audio-player.ts @@ -1,5 +1,6 @@ import { KeyframeBuilder } from "@animations/keyframe-builder"; import type { Edit } from "@core/edit-session"; +import { sec } from "@core/timing/types"; import { type Size } from "@layouts/geometry"; import { AudioLoadParser } from "@loaders/audio-load-parser"; import { type AudioAsset, type ResolvedClip, type Keyframe } from "@schemas"; @@ -10,6 +11,7 @@ import { Player, PlayerType } from "./player"; export class AudioPlayer extends Player { private audioResource: howler.Howl | null; + private loadedResourceIdentifier: string | null; private isPlaying: boolean; private volumeKeyframeBuilder!: KeyframeBuilder; @@ -20,38 +22,54 @@ export class AudioPlayer extends Player { super(edit, clipConfiguration, PlayerType.Audio); this.audioResource = null; + this.loadedResourceIdentifier = null; this.isPlaying = false; this.syncTimer = 0; } public override async load(): Promise { - await super.load(); + const mediaTimingRevision = this.beginMediaTimingLoad(); + try { + await super.load(); + if (!this.isMediaTimingLoadCurrent(mediaTimingRevision)) return; - const audioClipConfiguration = this.clipConfiguration.asset as AudioAsset; + const audioClipConfiguration = this.clipConfiguration.asset as AudioAsset; - const identifier = audioClipConfiguration.src; - if (!identifier) { - // Prompt-bearing assets route to pending placeholder players — reaching here without a src is invalid data - throw new Error("Audio asset has no src to load."); - } - const loadOptions: pixi.UnresolvedAsset = { src: identifier, parser: AudioLoadParser.Name }; - const audioResource = await this.edit.assetLoader.load(identifier, loadOptions); + const identifier = audioClipConfiguration.src; + if (!identifier) { + // Prompt-bearing assets route to pending placeholder players — reaching here without a src is invalid data + throw new Error("Audio asset has no src to load."); + } + const loadOptions: pixi.UnresolvedAsset = { src: identifier, parser: AudioLoadParser.Name }; + const audioResource = await this.edit.assetLoader.load(identifier, loadOptions); + if (!this.isMediaTimingLoadCurrent(mediaTimingRevision)) { + if (audioResource) this.edit.assetLoader.release(identifier); + return; + } - const isValidAudioSource = audioResource instanceof howler.Howl; - if (!isValidAudioSource) { - throw new Error(`Invalid audio source '${audioClipConfiguration.src}'.`); - } + const isValidAudioSource = audioResource instanceof howler.Howl; + if (!isValidAudioSource) { + if (audioResource) this.edit.assetLoader.release(identifier); + throw new Error(`Invalid audio source '${audioClipConfiguration.src}'.`); + } - this.audioResource = audioResource; + this.audioResource = audioResource; + this.loadedResourceIdentifier = identifier; + this.completeMediaTimingLoad(mediaTimingRevision, sec(audioResource.duration())); - // Create volume keyframes after timing is resolved (not in constructor) - const baseVolume = typeof audioClipConfiguration.volume === "number" ? audioClipConfiguration.volume : 1; - this.volumeKeyframeBuilder = new KeyframeBuilder(this.createVolumeKeyframes(audioClipConfiguration, baseVolume), this.getLength(), baseVolume); + // Create volume keyframes after timing is resolved (not in constructor) + const baseVolume = typeof audioClipConfiguration.volume === "number" ? audioClipConfiguration.volume : 1; + this.volumeKeyframeBuilder = new KeyframeBuilder(this.createVolumeKeyframes(audioClipConfiguration, baseVolume), this.getLength(), baseVolume); - // Set initial volume immediately so the Howl never sits at the default of 1.0 - this.audioResource.volume(this.getVolume()); + // Set initial volume immediately so the Howl never sits at the default of 1.0 + this.audioResource.volume(this.getVolume()); - this.configureKeyframes(); + this.configureKeyframes(); + } catch (error) { + if (!this.isMediaTimingLoadCurrent(mediaTimingRevision)) return; + this.completeMediaTimingLoad(mediaTimingRevision, null); + throw error; + } } public override update(deltaTime: number, elapsed: number): void { @@ -112,34 +130,58 @@ export class AudioPlayer extends Player { this.audioResource.unload(); } this.audioResource = null; + this.loadedResourceIdentifier = null; super.dispose(); } /** Reload the audio asset when asset.src changes (e.g., merge field update or loadEdit) */ public override async reloadAsset(): Promise { - if (this.audioResource) { - this.audioResource.stop(); - this.audioResource.unload(); - } - this.audioResource = null; - this.isPlaying = false; - this.syncTimer = 0; + const mediaTimingRevision = this.beginMediaTimingLoad(); + try { + this.releaseLoadedResource(); + if (this.audioResource) { + this.audioResource.stop(); + this.audioResource.unload(); + } + this.audioResource = null; + this.isPlaying = false; + this.syncTimer = 0; - const audioAsset = this.clipConfiguration.asset as AudioAsset; - const { src } = audioAsset; - if (!src) { - throw new Error("Audio asset has no src to load."); - } - const loadOptions: pixi.UnresolvedAsset = { src, parser: AudioLoadParser.Name }; - const audioResource = await this.edit.assetLoader.load(src, loadOptions); + const audioAsset = this.clipConfiguration.asset as AudioAsset; + const { src } = audioAsset; + if (!src) throw new Error("Audio source is required."); + const loadOptions: pixi.UnresolvedAsset = { src, parser: AudioLoadParser.Name }; + const audioResource = await this.edit.assetLoader.load(src, loadOptions); + if (!this.isMediaTimingLoadCurrent(mediaTimingRevision)) { + if (audioResource) this.edit.assetLoader.release(src); + return; + } + + if (!(audioResource instanceof howler.Howl)) { + if (audioResource) this.edit.assetLoader.release(src); + throw new Error(`Invalid audio source '${src}'.`); + } - if (!(audioResource instanceof howler.Howl)) { - throw new Error(`Invalid audio source '${audioAsset.src}'.`); + this.audioResource = audioResource; + this.loadedResourceIdentifier = src; + this.completeMediaTimingLoad(mediaTimingRevision, sec(audioResource.duration())); + this.audioResource.volume(this.getVolume()); + } catch (error) { + if (!this.isMediaTimingLoadCurrent(mediaTimingRevision)) return; + this.completeMediaTimingLoad(mediaTimingRevision, null); + throw error; } + } + + public override getLoadedResourceIdentifier(): string | null { + return this.loadedResourceIdentifier; + } - this.audioResource = audioResource; - this.audioResource.volume(this.getVolume()); + private releaseLoadedResource(): void { + if (!this.loadedResourceIdentifier) return; + this.edit.assetLoader.release(this.loadedResourceIdentifier); + this.loadedResourceIdentifier = null; } public override reconfigureAfterRestore(): void { diff --git a/src/components/canvas/players/caption-player.ts b/src/components/canvas/players/caption-player.ts index f8908d7e..39d00582 100644 --- a/src/components/canvas/players/caption-player.ts +++ b/src/components/canvas/players/caption-player.ts @@ -2,7 +2,7 @@ import { Player, PlayerType } from "@canvas/players/player"; import { type Cue, findActiveCue } from "@core/captions"; import type { Edit } from "@core/edit-session"; import { parseFontFamily, resolveFontPath } from "@core/fonts/font-config"; -import { isAliasReference } from "@core/timing/types"; +import { isAliasReference, sec, type Seconds } from "@core/timing/types"; import { type Size, type Vector } from "@layouts/geometry"; import { SubtitleLoadParser, type SubtitleAsset } from "@loaders/subtitle-load-parser"; import { type ExtendedCaptionAsset, type ResolvedClip } from "@schemas"; @@ -12,6 +12,7 @@ import * as pixi from "pixi.js"; const PLACEHOLDER_TEXT = "Captions will appear here"; type CaptionState = { readonly kind: "loaded"; readonly cues: Cue[] } | { readonly kind: "placeholder" }; +type CaptionLoadResult = { readonly state: CaptionState; readonly retainedIdentifier: string | null }; /** * CaptionPlayer renders timed subtitle cues from SRT/VTT files. @@ -24,21 +25,18 @@ export class CaptionPlayer extends Player { private currentCue: Cue | null = null; private background: pixi.Graphics | null = null; private text: pixi.Text | null = null; + private loadedSubtitleIdentifier: string | null = null; constructor(edit: Edit, clipConfiguration: ResolvedClip) { super(edit, clipConfiguration, PlayerType.Caption); } public override async load(): Promise { + const mediaTimingRevision = this.beginMediaTimingLoad(); await super.load(); + if (!this.isMediaTimingLoadCurrent(mediaTimingRevision)) return; const captionAsset = this.clipConfiguration.asset as ExtendedCaptionAsset; - - const fontFamily = captionAsset.font?.family ?? "Open Sans"; - await this.loadFont(fontFamily); - - this.state = isAliasReference(captionAsset.src) ? { kind: "placeholder" } : await this.loadSubtitles(captionAsset.src); - this.background = new pixi.Graphics(); this.contentContainer.addChild(this.background); @@ -54,12 +52,56 @@ export class CaptionPlayer extends Player { } this.contentContainer.addChild(this.text); + this.configureKeyframes(); - if (this.state.kind === "placeholder") { - this.showPlaceholder(captionAsset); + try { + const fontFamily = captionAsset.font?.family ?? "Open Sans"; + await this.loadFont(fontFamily); + + const result = isAliasReference(captionAsset.src) + ? { state: { kind: "placeholder" as const }, retainedIdentifier: null } + : await this.loadSubtitles(captionAsset.src); + if (!this.isMediaTimingLoadCurrent(mediaTimingRevision)) { + this.releaseSubtitle(result.retainedIdentifier); + return; + } + this.replaceLoadedSubtitle(result.retainedIdentifier); + this.state = result.state; + this.completeMediaTimingLoad(mediaTimingRevision, this.getCaptionDuration(result.state)); + + if (this.state.kind === "placeholder") { + this.showPlaceholder(captionAsset); + } else { + this.updateDisplay(null, captionAsset); + } + } catch (error) { + if (!this.isMediaTimingLoadCurrent(mediaTimingRevision)) return; + this.completeMediaTimingLoad(mediaTimingRevision, null); + throw error; } + } - this.configureKeyframes(); + public override async reloadAsset(): Promise { + const mediaTimingRevision = this.beginMediaTimingLoad(); + const captionAsset = this.clipConfiguration.asset as ExtendedCaptionAsset; + const result = isAliasReference(captionAsset.src) + ? { state: { kind: "placeholder" as const }, retainedIdentifier: null } + : await this.loadSubtitles(captionAsset.src); + + if (!this.isMediaTimingLoadCurrent(mediaTimingRevision)) { + this.releaseSubtitle(result.retainedIdentifier); + return; + } + this.replaceLoadedSubtitle(result.retainedIdentifier); + this.state = result.state; + this.currentCue = null; + this.completeMediaTimingLoad(mediaTimingRevision, this.getCaptionDuration(result.state)); + + if (result.state.kind === "placeholder") { + this.showPlaceholder(captionAsset); + } else { + this.updateDisplay(null, captionAsset); + } } public override update(deltaTime: number, elapsed: number): void { @@ -94,6 +136,10 @@ export class CaptionPlayer extends Player { this.currentCue = null; } + public override getLoadedResourceIdentifier(): string | null { + return this.loadedSubtitleIdentifier; + } + public override getSize(): Size { const captionAsset = this.clipConfiguration.asset as ExtendedCaptionAsset; @@ -112,7 +158,7 @@ export class CaptionPlayer extends Player { return { x: scale, y: scale }; } - private async loadSubtitles(src: string): Promise { + private async loadSubtitles(src: string): Promise { try { const loadOptions: pixi.UnresolvedAsset = { src, @@ -121,17 +167,36 @@ export class CaptionPlayer extends Player { const subtitle = await this.edit.assetLoader.load(src, loadOptions); if (subtitle) { - return { kind: "loaded", cues: subtitle.cues }; + return { state: { kind: "loaded", cues: subtitle.cues }, retainedIdentifier: src }; } console.warn("Failed to load subtitles"); - return { kind: "placeholder" }; + return { state: { kind: "placeholder" }, retainedIdentifier: null }; } catch (error) { console.warn("Failed to load subtitles:", error); - return { kind: "placeholder" }; + return { state: { kind: "placeholder" }, retainedIdentifier: null }; } } + private getCaptionDuration(state: CaptionState): Seconds | null { + if (state.kind !== "loaded" || state.cues.length === 0) return null; + let maxEnd = Number.NEGATIVE_INFINITY; + for (const cue of state.cues) { + if (Number.isFinite(cue.end)) maxEnd = Math.max(maxEnd, cue.end); + } + return Number.isFinite(maxEnd) ? sec(maxEnd) : null; + } + + private replaceLoadedSubtitle(identifier: string | null): void { + const previousIdentifier = this.loadedSubtitleIdentifier; + this.loadedSubtitleIdentifier = identifier; + this.releaseSubtitle(previousIdentifier); + } + + private releaseSubtitle(identifier: string | null): void { + if (identifier) this.edit.assetLoader.release(identifier); + } + private showPlaceholder(captionAsset: ExtendedCaptionAsset): void { const placeholderCue: Cue = { start: 0, end: Infinity, text: PLACEHOLDER_TEXT }; this.updateDisplay(placeholderCue, captionAsset); diff --git a/src/components/canvas/players/image-player.ts b/src/components/canvas/players/image-player.ts index db47746d..00da0d7e 100644 --- a/src/components/canvas/players/image-player.ts +++ b/src/components/canvas/players/image-player.ts @@ -1,4 +1,8 @@ import type { Edit } from "@core/edit-session"; +import { GifImageLoadError } from "@core/loaders/gif-image-load-error"; +import { getAnimatedGifDurationMs, GifImageSource } from "@core/loaders/gif-image-source"; +import { appendCorsQuery } from "@core/loaders/gif-url"; +import { sec } from "@core/timing/types"; import { type Size } from "@layouts/geometry"; import { type ResolvedClip, type ImageAsset } from "@schemas"; import * as pixi from "pixi.js"; @@ -7,9 +11,14 @@ import { createPlaceholderGraphic } from "./placeholder-graphic"; import { Player, PlayerType } from "./player"; export class ImagePlayer extends Player { - private texture: pixi.Texture | null; + private texture: pixi.Texture | null; private sprite: pixi.Sprite | null; private placeholder: pixi.Graphics | null; + private gifSource: GifImageSource | null; + private gifFrameTextures: pixi.Texture[]; + private ownedTextureWrappers: pixi.Texture[]; + private currentGifFrame: number; + private loadedAssetIdentifier: string | null; constructor(edit: Edit, clipConfiguration: ResolvedClip) { super(edit, clipConfiguration, PlayerType.Image); @@ -17,15 +26,27 @@ export class ImagePlayer extends Player { this.texture = null; this.sprite = null; this.placeholder = null; + this.gifSource = null; + this.gifFrameTextures = []; + this.ownedTextureWrappers = []; + this.currentGifFrame = -1; + this.loadedAssetIdentifier = null; } public override async load(): Promise { + const mediaTimingRevision = this.beginMediaTimingLoad(); await super.load(); + if (!this.isMediaTimingLoadCurrent(mediaTimingRevision)) return; try { - await this.loadTexture(); + if (!(await this.loadTexture(mediaTimingRevision))) return; + const duration = getAnimatedGifDurationMs(this.gifSource); + this.completeMediaTimingLoad(mediaTimingRevision, duration === null ? null : sec(duration / 1000)); this.configureKeyframes(); - } catch { + } catch (error) { + if (!this.isMediaTimingLoadCurrent(mediaTimingRevision)) return; + this.completeMediaTimingLoad(mediaTimingRevision, null); this.createFallbackGraphic(); + if (error instanceof GifImageLoadError) throw error; } } @@ -40,11 +61,13 @@ export class ImagePlayer extends Player { public override update(deltaTime: number, elapsed: number): void { super.update(deltaTime, elapsed); + this.updateGifFrame(); } public override dispose(): void { this.disposeTexture(); this.clearPlaceholder(); + this.loadedAssetIdentifier = null; super.dispose(); } @@ -71,34 +94,80 @@ export class ImagePlayer extends Player { return this.placeholder ? this.getDisplaySize() : { width: 0, height: 0 }; } + public override getLoadedResourceIdentifier(): string | null { + return this.loadedAssetIdentifier; + } + + public override async prepareStaticRender(): Promise { + this.updateGifFrame(); + } + /** Reload the image asset when asset.src changes (e.g., merge field update) */ public override async reloadAsset(): Promise { + const mediaTimingRevision = this.beginMediaTimingLoad(); this.disposeTexture(); this.clearPlaceholder(); + this.releaseLoadedAsset(); try { - await this.loadTexture(); - } catch { + if (!(await this.loadTexture(mediaTimingRevision))) return; + const duration = getAnimatedGifDurationMs(this.gifSource); + this.completeMediaTimingLoad(mediaTimingRevision, duration === null ? null : sec(duration / 1000)); + } catch (error) { + if (!this.isMediaTimingLoadCurrent(mediaTimingRevision)) return; + this.completeMediaTimingLoad(mediaTimingRevision, null); this.createFallbackGraphic(); + if (error instanceof GifImageLoadError) throw error; } } - private async loadTexture(): Promise { + private async loadTexture(mediaTimingRevision: number): Promise { const imageAsset = this.clipConfiguration.asset as ImageAsset; const { src } = imageAsset; if (!src) { // Prompt-bearing assets route to pending placeholder players — reaching here without a src is invalid data throw new Error("Image asset has no src to load."); } + const requestUrl = appendCorsQuery(src); + let isGif: boolean; + try { + isGif = await this.edit.assetLoader.isGif(src, requestUrl); + } catch (error) { + throw new GifImageLoadError(`Unable to validate GIF image source '${src}'.`, { cause: error }); + } + if (!this.isMediaTimingLoadCurrent(mediaTimingRevision)) return false; + if (!isGif) this.completeMediaTimingLoad(mediaTimingRevision, null); - const corsUrl = `${src}${src.includes("?") ? "&" : "?"}x-cors=1`; - const loadOptions: pixi.UnresolvedAsset = { src: corsUrl, crossorigin: "anonymous", data: {} }; - const texture = await this.edit.assetLoader.load>(corsUrl, loadOptions); + if (isGif) { + const source = await this.edit.assetLoader.loadGif(src, requestUrl); + if (!source) throw new GifImageLoadError(`Unable to decode GIF image source '${src}'.`); + if (!this.isMediaTimingLoadCurrent(mediaTimingRevision)) { + this.edit.assetLoader.release(src); + return false; + } + + try { + this.configureGif(source); + this.loadedAssetIdentifier = src; + return true; + } catch (error) { + this.disposeTexture(); + this.edit.assetLoader.release(src); + throw new GifImageLoadError(`Unable to configure GIF image source '${src}'.`, { cause: error }); + } + } + + const loadOptions: pixi.UnresolvedAsset = { alias: src, src: requestUrl, crossorigin: "anonymous", data: {} }; + const texture = await this.edit.assetLoader.load>(src, loadOptions); + if (!this.isMediaTimingLoadCurrent(mediaTimingRevision)) { + if (texture) this.edit.assetLoader.release(src); + return false; + } if (!(texture?.source instanceof pixi.ImageSource)) { if (texture) { texture.destroy(true); - await this.edit.assetLoader.rejectAsset(corsUrl); + await this.edit.assetLoader.rejectAsset(src); } throw new Error(`Invalid image source '${src}'.`); } @@ -107,10 +176,42 @@ export class ImagePlayer extends Player { this.texture = this.createCroppedTexture(texture); this.sprite = new pixi.Sprite(this.texture); this.contentContainer.addChild(this.sprite); + this.loadedAssetIdentifier = src; + + if (this.clipConfiguration.width && this.clipConfiguration.height) { + this.applyFixedDimensions(); + } + return true; + } + + private configureGif(source: GifImageSource): void { + this.clearPlaceholder(); + this.gifSource = source; + this.gifFrameTextures = source.frames.map(frame => this.createCroppedTexture(frame.texture)); + this.texture = this.gifFrameTextures[0] ?? null; + if (!this.texture) throw new Error("GIF contains no renderable frames."); + + this.sprite = new pixi.Sprite(this.texture); + this.contentContainer.addChild(this.sprite); + this.currentGifFrame = 0; if (this.clipConfiguration.width && this.clipConfiguration.height) { this.applyFixedDimensions(); } + this.updateGifFrame(); + } + + private updateGifFrame(): void { + if (!this.gifSource || !this.sprite || !this.isActive()) return; + + const frameIndex = this.gifSource.frameIndexAt(this.getPlaybackTime() * 1000); + if (frameIndex === this.currentGifFrame) return; + + const frameTexture = this.gifFrameTextures[frameIndex]; + if (!frameTexture) return; + this.sprite.texture = frameTexture; + this.texture = frameTexture; + this.currentGifFrame = frameIndex; } private disposeTexture(): void { @@ -119,9 +220,20 @@ export class ImagePlayer extends Player { this.sprite.destroy(); this.sprite = null; } - // DON'T destroy the texture - it's managed by Assets - // The unloadClipAssets() method handles proper cleanup via Assets.unload() + for (const texture of this.ownedTextureWrappers) { + texture.destroy(false); + } + this.ownedTextureWrappers = []; this.texture = null; + this.gifFrameTextures = []; + this.gifSource = null; + this.currentGifFrame = -1; + } + + private releaseLoadedAsset(): void { + if (!this.loadedAssetIdentifier) return; + this.edit.assetLoader.release(this.loadedAssetIdentifier); + this.loadedAssetIdentifier = null; } private clearPlaceholder(): void { @@ -136,7 +248,7 @@ export class ImagePlayer extends Player { return true; } - private createCroppedTexture(texture: pixi.Texture): pixi.Texture { + private createCroppedTexture(texture: pixi.Texture): pixi.Texture { const imageAsset = this.clipConfiguration.asset as ImageAsset; if (!imageAsset.crop) { @@ -157,6 +269,8 @@ export class ImagePlayer extends Player { const height = originalHeight - top - bottom; const crop = new pixi.Rectangle(x, y, width, height); - return new pixi.Texture({ source: texture.source, frame: crop }); + const croppedTexture = new pixi.Texture({ source: texture.source, frame: crop }); + this.ownedTextureWrappers.push(croppedTexture); + return croppedTexture; } } diff --git a/src/components/canvas/players/image-to-video-player.ts b/src/components/canvas/players/image-to-video-player.ts index d377f13f..58aac4a0 100644 --- a/src/components/canvas/players/image-to-video-player.ts +++ b/src/components/canvas/players/image-to-video-player.ts @@ -1,4 +1,5 @@ import type { Edit } from "@core/edit-session"; +import { appendCorsQuery } from "@core/loaders/gif-url"; import { computeAiAssetNumber, isAiAsset } from "@core/shared/ai-asset-utils"; import { type Size } from "@layouts/geometry"; import { type ResolvedClip } from "@schemas"; @@ -13,6 +14,7 @@ export class ImageToVideoPlayer extends Player { private texture: pixi.Texture | null = null; private placeholder: pixi.Graphics | null = null; private aiOverlay: AiPendingOverlay | null = null; + private loadedResourceIdentifier: string | null = null; constructor(edit: Edit, clipConfiguration: ResolvedClip) { super(edit, clipConfiguration, PlayerType.ImageToVideo); @@ -102,6 +104,7 @@ export class ImageToVideoPlayer extends Player { this.sprite = null; } this.texture = null; + this.loadedResourceIdentifier = null; this.placeholder?.destroy(); this.placeholder = null; @@ -112,9 +115,13 @@ export class ImageToVideoPlayer extends Player { super.dispose(); } + public override getLoadedResourceIdentifier(): string | null { + return this.loadedResourceIdentifier; + } + private async tryLoadTexture(src: string): Promise { try { - const corsUrl = `${src}${src.includes("?") ? "&" : "?"}x-cors=1`; + const corsUrl = appendCorsQuery(src); const loadOptions: pixi.UnresolvedAsset = { src: corsUrl, crossorigin: "anonymous", data: {} }; const texture = await this.edit.assetLoader.load>(corsUrl, loadOptions); @@ -125,8 +132,13 @@ export class ImageToVideoPlayer extends Player { } return false; } + if (this.contentContainer.destroyed) { + this.edit.assetLoader.release(corsUrl); + return false; + } this.texture = texture; + this.loadedResourceIdentifier = corsUrl; this.sprite = new pixi.Sprite(this.texture); this.contentContainer.addChild(this.sprite); diff --git a/src/components/canvas/players/luma-player.ts b/src/components/canvas/players/luma-player.ts index 02564373..a6908f9b 100644 --- a/src/components/canvas/players/luma-player.ts +++ b/src/components/canvas/players/luma-player.ts @@ -1,4 +1,5 @@ import type { Edit } from "@core/edit-session"; +import { sec } from "@core/timing/types"; import { type Size } from "@layouts/geometry"; import { type ResolvedClip, type LumaAsset } from "@schemas"; import * as pixi from "pixi.js"; @@ -11,6 +12,7 @@ export class LumaPlayer extends Player { private texture: pixi.Texture | null; private sprite: pixi.Sprite | null; private isPlaying: boolean; + private loadedResourceIdentifier: string | null; constructor(edit: Edit, clipConfiguration: ResolvedClip) { super(edit, clipConfiguration, PlayerType.Luma); @@ -18,17 +20,46 @@ export class LumaPlayer extends Player { this.texture = null; this.sprite = null; this.isPlaying = false; + this.loadedResourceIdentifier = null; } public override async load(): Promise { - await super.load(); + const mediaTimingRevision = this.beginMediaTimingLoad(); + try { + await super.load(); + if (!this.isMediaTimingLoadCurrent(mediaTimingRevision)) return; + await this.loadLuma(mediaTimingRevision); + } catch (error) { + if (!this.isMediaTimingLoadCurrent(mediaTimingRevision)) return; + this.completeMediaTimingLoad(mediaTimingRevision, null); + throw error; + } + } - const lumaAsset = this.clipConfiguration.asset as LumaAsset; + public override async reloadAsset(): Promise { + const mediaTimingRevision = this.beginMediaTimingLoad(); + this.releaseLoadedResource(); + this.clearLumaVisual(); + try { + await this.loadLuma(mediaTimingRevision); + } catch (error) { + if (!this.isMediaTimingLoadCurrent(mediaTimingRevision)) return; + this.completeMediaTimingLoad(mediaTimingRevision, null); + throw error; + } + } + private async loadLuma(mediaTimingRevision: number): Promise { + const lumaAsset = this.clipConfiguration.asset as LumaAsset; const identifier = lumaAsset.src; const loadOptions: pixi.UnresolvedAsset = { src: identifier, data: { autoPlay: false, muted: true } }; const texture = await this.edit.assetLoader.load>(identifier, loadOptions); + if (!this.isMediaTimingLoadCurrent(mediaTimingRevision)) { + if (texture) this.edit.assetLoader.release(identifier); + return; + } + const isValidLumaSource = texture?.source instanceof pixi.ImageSource || texture?.source instanceof pixi.VideoSource; if (!isValidLumaSource) { if (texture) { @@ -38,16 +69,16 @@ export class LumaPlayer extends Player { throw new Error(`Invalid luma source '${lumaAsset.src}'.`); } - // Fix alpha channel rendering for WebM VP9 videos - // PixiJS 8's auto-detection is buggy, causing invisible rendering if (texture.source instanceof pixi.VideoSource) { texture.source.alphaMode = "no-premultiply-alpha"; } this.texture = texture; - this.sprite = new pixi.Sprite(this.texture); - + this.loadedResourceIdentifier = identifier; + this.sprite = new pixi.Sprite(texture); this.contentContainer.addChild(this.sprite); + const duration = texture.source instanceof pixi.VideoSource ? texture.source.resource.duration : null; + this.completeMediaTimingLoad(mediaTimingRevision, duration !== null && Number.isFinite(duration) ? sec(duration) : null); this.configureKeyframes(); } @@ -62,13 +93,15 @@ export class LumaPlayer extends Player { return; } + const { trim = 0 } = this.clipConfiguration.asset as LumaAsset; const playbackTime = this.getPlaybackTime(); + const sourceTime = playbackTime + trim; const shouldClipPlay = this.edit.isPlaying && this.isActive(); if (shouldClipPlay) { if (!this.isPlaying) { this.isPlaying = true; - this.texture.source.resource.currentTime = playbackTime; + this.texture.source.resource.currentTime = sourceTime; this.texture.source.resource.play().catch(console.error); } @@ -77,10 +110,10 @@ export class LumaPlayer extends Player { } const desyncThreshold = 0.1; - const shouldSync = Math.abs(this.texture.source.resource.currentTime - playbackTime) > desyncThreshold; + const shouldSync = Math.abs(this.texture.source.resource.currentTime - sourceTime) > desyncThreshold; if (shouldSync) { - this.texture.source.resource.currentTime = playbackTime; + this.texture.source.resource.currentTime = sourceTime; } } @@ -90,18 +123,33 @@ export class LumaPlayer extends Player { } if (!this.edit.isPlaying && this.isActive()) { - this.texture.source.resource.currentTime = playbackTime; + this.texture.source.resource.currentTime = sourceTime; } } public override dispose(): void { + this.clearLumaVisual(); + this.loadedResourceIdentifier = null; super.dispose(); + } - this.sprite?.destroy(); - this.sprite = null; + public override getLoadedResourceIdentifier(): string | null { + return this.loadedResourceIdentifier; + } + + private releaseLoadedResource(): void { + if (!this.loadedResourceIdentifier) return; + this.edit.assetLoader.release(this.loadedResourceIdentifier); + this.loadedResourceIdentifier = null; + } - // DON'T destroy the texture - it's managed by Assets - // The unloadClipAssets() method in Edit already calls Assets.unload() + private clearLumaVisual(): void { + if (this.sprite) { + this.contentContainer.removeChild(this.sprite); + this.sprite.destroy(); + this.sprite = null; + } + // The texture is shared and released through AssetLoader. this.texture = null; } diff --git a/src/components/canvas/players/player.ts b/src/components/canvas/players/player.ts index f9032722..5dc560a0 100644 --- a/src/components/canvas/players/player.ts +++ b/src/components/canvas/players/player.ts @@ -6,8 +6,10 @@ import { WipeFilter } from "@animations/wipe-filter"; import { type Edit } from "@core/edit-session"; import { InternalEvent } from "@core/events/edit-events"; import { calculateContainerScale, calculateFitScale, calculateSpriteTransform, type FitMode } from "@core/layout/fit-system"; +import { getAssetTimingIdentity, mediaTimingMatchesAsset } from "@core/timing/resolver"; import { type AliasReference, + type MediaTimingState, type ResolvedTiming, type Seconds, type TimingIntent, @@ -79,6 +81,8 @@ export abstract class Player extends Entity { public clipConfiguration: ResolvedClip; private resolvedTiming: ResolvedTiming; + private mediaTimingState: MediaTimingState; + private mediaTimingRevision = 0; private offsetXKeyframeBuilder?: ComposedKeyframeBuilder; private offsetYKeyframeBuilder?: ComposedKeyframeBuilder; @@ -109,6 +113,11 @@ export abstract class Player extends Entity { this.clipConfiguration = clipConfiguration; this.resolvedTiming = { start: clipConfiguration.start, length: clipConfiguration.length }; + this.mediaTimingState = { + status: "ready", + asset: getAssetTimingIdentity(clipConfiguration.asset), + duration: null + }; this.wipeMask = null; @@ -125,11 +134,11 @@ export abstract class Player extends Entity { /** * Reload the asset for this player (e.g., when asset.src changes). - * Override in subclasses that have loadable assets (image, video). - * Default implementation is a no-op. + * Non-temporal players publish the new asset identity immediately. */ public async reloadAsset(): Promise { - // Default: no-op. Override in ImagePlayer, VideoPlayer, etc. + const revision = this.beginMediaTimingLoad(); + this.completeMediaTimingLoad(revision, null); } protected configureKeyframes() { @@ -340,6 +349,9 @@ export abstract class Player extends Entity { } public override dispose(): void { + // Any asset work completing after disposal is stale, regardless of whether the + // source identity happens to be unchanged (for example an A -> B -> A race). + this.mediaTimingRevision += 1; this.wipeMask?.destroy(); this.wipeMask = null; this.wipeFilter?.destroy(); @@ -401,6 +413,44 @@ export abstract class Player extends Entity { return { ...this.resolvedTiming }; } + /** + * Publish loaded media facts through one contract for every asset type. + * Non-temporal assets are immediately ready without an intrinsic duration. + */ + public getMediaTimingState(): MediaTimingState { + return mediaTimingMatchesAsset(this.mediaTimingState, this.clipConfiguration.asset) + ? this.mediaTimingState + : { status: "pending", asset: getAssetTimingIdentity(this.clipConfiguration.asset) }; + } + + /** Mark the current asset revision as loading and return its race-safe revision. */ + protected beginMediaTimingLoad(): number { + this.mediaTimingRevision += 1; + this.mediaTimingState = { status: "pending", asset: getAssetTimingIdentity(this.clipConfiguration.asset) }; + return this.mediaTimingRevision; + } + + /** Whether an asynchronous asset load still represents the Player's current asset revision. */ + protected isMediaTimingLoadCurrent(revision: number): boolean { + return revision === this.mediaTimingRevision; + } + + /** Publish metadata only if the Player still represents the asset revision that loaded it. */ + protected completeMediaTimingLoad(revision: number, duration: Seconds | null): void { + if (!this.isMediaTimingLoadCurrent(revision)) return; + const validDuration = duration !== null && Number.isFinite(duration) && duration > 0 ? duration : null; + this.mediaTimingState = { + status: "ready", + asset: getAssetTimingIdentity(this.clipConfiguration.asset), + duration: validDuration + }; + } + + /** Asset cache identifier held by this Player, used for reference-counted cleanup. */ + public getLoadedResourceIdentifier(): string | null { + return null; + } + public setResolvedTiming(timing: ResolvedTiming): void { this.resolvedTiming = { ...timing }; this.clipConfiguration.start = timing.start; diff --git a/src/components/canvas/players/rich-caption-player.ts b/src/components/canvas/players/rich-caption-player.ts index d5ce53f9..fed8c684 100644 --- a/src/components/canvas/players/rich-caption-player.ts +++ b/src/components/canvas/players/rich-caption-player.ts @@ -2,7 +2,7 @@ import { Player, PlayerType } from "@canvas/players/player"; import { Edit } from "@core/edit-session"; import { parseFontFamily, resolveFontPath, getFontDisplayName } from "@core/fonts/font-config"; import { extractFontNames, isGoogleFontUrl } from "@core/fonts/font-utils"; -import { isAliasReference } from "@core/timing/types"; +import { isAliasReference, sec, type Seconds } from "@core/timing/types"; import { type Size, type Vector } from "@layouts/geometry"; import { RichCaptionAssetSchema, type RichCaptionAsset, type ResolvedClip } from "@schemas"; import { @@ -38,15 +38,17 @@ export class RichCaptionPlayer extends Player { private currentRender: Promise | null = null; // serialises the async paint so captures await a finished frame private texture: pixi.Texture | null = null; private sprite: pixi.Sprite | null = null; + private fallbackText: pixi.Text | null = null; private words: WordTiming[] = []; private loadComplete: boolean = false; private isPlaceholder = false; private readonly fontRegistrationCache = new Map>(); - private lastRegisteredFontKey: string = ""; private pendingLayoutId: number = 0; private resolvedPauseThreshold: number = 500; + private assetLoadInProgress = false; + private renderConfigurationKey: string | null = null; constructor(edit: Edit, clipConfiguration: ResolvedClip) { const { fit, ...configWithoutFit } = clipConfiguration; @@ -78,56 +80,20 @@ export class RichCaptionPlayer extends Player { return words; } + private static getWordsDuration(words: WordTiming[]): Seconds | null { + let maxEnd = Number.NEGATIVE_INFINITY; + for (const word of words) { + const end = Number(word.end); + if (Number.isFinite(end)) maxEnd = Math.max(maxEnd, end); + } + return Number.isFinite(maxEnd) ? sec((maxEnd + 500) / 1000) : null; + } + public override async load(): Promise { + const mediaTimingRevision = this.beginMediaTimingLoad(); await super.load(); - - const richCaptionAsset = this.clipConfiguration.asset as RichCaptionAsset; - - try { - const validationResult = RichCaptionAssetSchema.safeParse(richCaptionAsset); - if (!validationResult.success) { - this.createFallbackGraphic("Invalid caption asset"); - return; - } - - let words: WordTiming[]; - if (richCaptionAsset.src) { - if (isAliasReference(richCaptionAsset.src)) { - words = RichCaptionPlayer.createPlaceholderWords(this.getLength() * 1000); - this.isPlaceholder = true; - this.needsResolution = true; - } else { - words = await this.fetchAndParseSubtitle(richCaptionAsset.src); - this.resolvedPauseThreshold = 5; - } - } else { - words = ((richCaptionAsset as RichCaptionAsset & { words?: WordTiming[] }).words ?? []).map((w: WordTiming) => ({ - text: w.text, - start: w.start, - end: w.end, - confidence: w.confidence - })); - } - - if (words.length === 0) { - this.createFallbackGraphic("No caption words found"); - return; - } - - if (words.length > HARD_WORD_LIMIT) { - this.createFallbackGraphic(`Word count (${words.length}) exceeds limit of ${HARD_WORD_LIMIT}`); - return; - } - if (words.length > SOFT_WORD_LIMIT) { - console.warn(`RichCaptionPlayer: ${words.length} words exceeds soft limit of ${SOFT_WORD_LIMIT}. Performance may degrade.`); - } - - await this.buildRenderPipeline(richCaptionAsset, words); - } catch (error) { - console.error("RichCaptionPlayer load failed:", error); - this.cleanupResources(); - this.createFallbackGraphic("Failed to load caption"); - } + if (!this.isMediaTimingLoadCurrent(mediaTimingRevision)) return; + await this.loadCurrentAsset(false, mediaTimingRevision); } public override update(deltaTime: number, elapsed: number): void { @@ -152,126 +118,190 @@ export class RichCaptionPlayer extends Player { } public override async reloadAsset(): Promise { + await this.loadCurrentAsset(true); + } + + private async loadCurrentAsset(propagateError: boolean, mediaTimingRevision = this.beginMediaTimingLoad()): Promise { + if (!this.isMediaTimingLoadCurrent(mediaTimingRevision)) return; + const pipelineRevision = this.beginPipelineBuild(); const asset = this.clipConfiguration.asset as RichCaptionAsset; - // When src is an alias reference, reset to placeholder if previously resolved - if (!asset.src || isAliasReference(asset.src)) { - if (this.loadComplete && !this.isPlaceholder) { - this.resolvedPauseThreshold = 500; - this.words = RichCaptionPlayer.createPlaceholderWords(this.getLength() * 1000); - this.isPlaceholder = true; - this.needsResolution = true; - await this.reconfigure(); - } + // Re-loading the same unresolved alias only needs to publish its new base identity. + // Styling and length changes use reconfigure(), which preserves the placeholder pipeline. + if (asset.src && isAliasReference(asset.src) && this.loadComplete && this.isPlaceholder) { + this.assetLoadInProgress = false; + this.completeMediaTimingLoad(mediaTimingRevision, null); return; } - this.loadComplete = false; + this.assetLoadInProgress = true; + let timingPublished = false; - if (this.texture) { - this.texture.destroy(); - this.texture = null; - } - if (this.sprite) { - this.sprite.destroy(); - this.sprite = null; - } - this.captionLayout = null; - this.validatedAsset = null; - this.generatorConfig = null; - this.canvas = null; - this.painter = null; + try { + const validationResult = RichCaptionAssetSchema.safeParse(asset); + if (!validationResult.success) { + this.completeMediaTimingLoad(mediaTimingRevision, null); + this.replacePipelineWithFallback("Invalid caption asset"); + return; + } - this.isPlaceholder = false; - this.needsResolution = false; + const isPlaceholder = Boolean(asset.src && isAliasReference(asset.src)); + const pauseThreshold = asset.src && !isPlaceholder ? 5 : 500; + let words: WordTiming[]; + if (isPlaceholder) { + words = RichCaptionPlayer.createPlaceholderWords(this.getLength() * 1000); + } else if (asset.src) { + words = await this.fetchAndParseSubtitle(asset.src); + } else { + words = ((asset as RichCaptionAsset & { words?: WordTiming[] }).words ?? []).map(word => ({ ...word })); + } - const words = await this.fetchAndParseSubtitle(asset.src); - this.resolvedPauseThreshold = 5; + if (!this.isPipelineBuildCurrent(pipelineRevision, mediaTimingRevision)) return; - if (words.length === 0) { - this.createFallbackGraphic("No caption words found"); - return; + if (words.length === 0) { + this.completeMediaTimingLoad(mediaTimingRevision, null); + this.replacePipelineWithFallback("No caption words found"); + return; + } + if (words.length > HARD_WORD_LIMIT) { + this.completeMediaTimingLoad(mediaTimingRevision, null); + this.replacePipelineWithFallback(`Word count (${words.length}) exceeds limit of ${HARD_WORD_LIMIT}`); + return; + } + if (words.length > SOFT_WORD_LIMIT) { + console.warn(`RichCaptionPlayer: ${words.length} words exceeds soft limit of ${SOFT_WORD_LIMIT}. Performance may degrade.`); + } + + this.completeMediaTimingLoad(mediaTimingRevision, isPlaceholder ? null : RichCaptionPlayer.getWordsDuration(words)); + timingPublished = true; + await this.buildRenderPipeline(asset, words, { + pipelineRevision, + mediaTimingRevision, + isPlaceholder, + needsResolution: isPlaceholder, + pauseThreshold, + renderTimeMs: 0 + }); + } catch (error) { + if (!this.isPipelineBuildCurrent(pipelineRevision, mediaTimingRevision)) return; + if (!timingPublished) this.completeMediaTimingLoad(mediaTimingRevision, null); + this.replacePipelineWithFallback("Failed to load caption"); + if (propagateError) throw error; + console.error("RichCaptionPlayer load failed:", error); + } finally { + if (this.isMediaTimingLoadCurrent(mediaTimingRevision)) this.assetLoadInProgress = false; } + } + + private beginPipelineBuild(): number { + this.pendingLayoutId += 1; + return this.pendingLayoutId; + } - await this.buildRenderPipeline(asset, words); + private isPipelineBuildCurrent(pipelineRevision: number, mediaTimingRevision?: number): boolean { + return pipelineRevision === this.pendingLayoutId && (mediaTimingRevision === undefined || this.isMediaTimingLoadCurrent(mediaTimingRevision)); } public override reconfigureAfterRestore(): void { super.reconfigureAfterRestore(); + if (this.loadComplete && this.renderConfigurationKey === this.getRenderConfigurationKey()) return; this.reconfigure(); } + private getRenderConfigurationKey(): string { + const { width, height } = this.getSize(); + return JSON.stringify([this.clipConfiguration.asset, width, height, this.isPlaceholder ? this.getLength() : null]); + } + private async reconfigure(): Promise { - if (!this.loadComplete || !this.layoutEngine || !this.canvas || !this.painter) { + if (this.assetLoadInProgress || !this.loadComplete || !this.layoutEngine || !this.canvas || !this.painter) { return; } + const pipelineRevision = this.beginPipelineBuild(); try { const asset = this.clipConfiguration.asset as RichCaptionAsset; - - // Regenerate placeholder words when clip length changes (e.g. "end" re-resolved after video probed) - if (this.isPlaceholder) { - this.words = RichCaptionPlayer.createPlaceholderWords(this.getLength() * 1000); - } - - const fontKey = `${asset.font?.family ?? "Roboto"}|${asset.font?.weight ?? 400}`; - if (fontKey !== this.lastRegisteredFontKey) { - await this.registerFonts(asset); - this.lastRegisteredFontKey = fontKey; - } - - const canvasPayload = this.buildCanvasPayload(asset, this.words); - const canvasValidation = CanvasRichCaptionAssetSchema.safeParse(canvasPayload); - if (!canvasValidation.success) { - console.error("Caption reconfigure validation failed:", canvasValidation.error?.issues); - return; - } - this.validatedAsset = canvasValidation.data; - - const { width, height } = this.getSize(); - const layoutConfig = buildCaptionLayoutConfig(this.validatedAsset, width, height); - this.captionLayout = await this.layoutEngine.layoutCaption(this.words, layoutConfig); - - this.generatorConfig = createDefaultGeneratorConfig(width, height, 1); - - this.renderFrameSync(this.getPlaybackTime() * 1000); + const words = this.isPlaceholder ? RichCaptionPlayer.createPlaceholderWords(this.getLength() * 1000) : this.words.map(word => ({ ...word })); + await this.buildRenderPipeline(asset, words, { + pipelineRevision, + isPlaceholder: this.isPlaceholder, + needsResolution: this.needsResolution, + pauseThreshold: this.resolvedPauseThreshold, + renderTimeMs: this.getPlaybackTime() * 1000 + }); } catch (error) { - console.error("RichCaptionPlayer reconfigure failed:", error); + if (this.isPipelineBuildCurrent(pipelineRevision)) { + console.error("RichCaptionPlayer reconfigure failed:", error); + } } } - private async buildRenderPipeline(asset: RichCaptionAsset, words: WordTiming[]): Promise { - const canvasPayload = this.buildCanvasPayload(asset, words); + private async buildRenderPipeline( + asset: RichCaptionAsset, + words: WordTiming[], + options: { + pipelineRevision: number; + mediaTimingRevision?: number; + isPlaceholder: boolean; + needsResolution: boolean; + pauseThreshold: number; + renderTimeMs: number; + } + ): Promise { + if (!this.isPipelineBuildCurrent(options.pipelineRevision, options.mediaTimingRevision)) return; + const { width, height } = this.getSize(); + const canvasPayload = this.buildCanvasPayload(asset, words, options.pauseThreshold, { width, height }); const canvasValidation = CanvasRichCaptionAssetSchema.safeParse(canvasPayload); if (!canvasValidation.success) { - console.error("Canvas caption validation failed:", canvasValidation.error?.issues ?? canvasValidation.error); - this.createFallbackGraphic("Caption validation failed"); + if (this.isPipelineBuildCurrent(options.pipelineRevision, options.mediaTimingRevision)) { + console.error("Canvas caption validation failed:", canvasValidation.error?.issues ?? canvasValidation.error); + this.replacePipelineWithFallback("Caption validation failed"); + } return; } - this.validatedAsset = canvasValidation.data; - this.words = words; - - this.fontRegistry = await FontRegistry.getSharedInstance(); - await this.registerFonts(asset); - this.lastRegisteredFontKey = `${asset.font?.family ?? "Roboto"}|${asset.font?.weight ?? 400}`; - - this.layoutEngine = new CaptionLayoutEngine(this.fontRegistry); - - const { width, height } = this.getSize(); - const layoutConfig = buildCaptionLayoutConfig(this.validatedAsset, width, height); - - this.captionLayout = await this.layoutEngine.layoutCaption(words, layoutConfig); - this.generatorConfig = createDefaultGeneratorConfig(width, height, 1); - - this.canvas = document.createElement("canvas"); - this.canvas.width = width; - this.canvas.height = height; - this.painter = createWebPainter(this.canvas); - - this.renderFrameSync(0); - this.configureKeyframes(); - this.loadComplete = true; + let fontRegistry: FontRegistry | null = null; + let committed = false; + try { + fontRegistry = await FontRegistry.getSharedInstance(); + if (!this.isPipelineBuildCurrent(options.pipelineRevision, options.mediaTimingRevision)) return; + + await this.registerFonts(asset, fontRegistry); + if (!this.isPipelineBuildCurrent(options.pipelineRevision, options.mediaTimingRevision)) return; + + const layoutEngine = new CaptionLayoutEngine(fontRegistry); + const layoutConfig = buildCaptionLayoutConfig(canvasValidation.data, width, height); + const captionLayout = await layoutEngine.layoutCaption(words, layoutConfig); + if (!this.isPipelineBuildCurrent(options.pipelineRevision, options.mediaTimingRevision)) return; + + const canvas = document.createElement("canvas"); + canvas.width = width; + canvas.height = height; + const painter = createWebPainter(canvas); + + this.destroyRenderedOutput(); + const previousRegistry = this.fontRegistry; + this.fontRegistry = fontRegistry; + this.layoutEngine = layoutEngine; + this.captionLayout = captionLayout; + this.validatedAsset = canvasValidation.data; + this.generatorConfig = createDefaultGeneratorConfig(width, height, 1); + this.canvas = canvas; + this.painter = painter; + this.words = words; + this.isPlaceholder = options.isPlaceholder; + this.needsResolution = options.needsResolution; + this.resolvedPauseThreshold = options.pauseThreshold; + this.loadComplete = true; + this.renderConfigurationKey = this.getRenderConfigurationKey(); + committed = true; + this.releaseFontRegistry(previousRegistry); + + this.renderFrameSync(options.renderTimeMs); + this.configureKeyframes(); + } finally { + if (fontRegistry && !committed) this.releaseFontRegistry(fontRegistry); + } } /** @@ -282,10 +312,12 @@ export class RichCaptionPlayer extends Player { while (this.currentRender) { await this.currentRender; } - this.currentRender = this.paintFrame(timeMs).finally(() => { - this.currentRender = null; + const pipelineRevision = this.pendingLayoutId; + const render = this.paintFrame(timeMs, pipelineRevision).finally(() => { + if (this.currentRender === render) this.currentRender = null; }); - await this.currentRender; + this.currentRender = render; + await render; } /** Fire-and-forget render for the live tick and lifecycle hooks; the texture updates when the paint settles. */ @@ -298,26 +330,29 @@ export class RichCaptionPlayer extends Player { * asynchronously), so the Pixi texture is only updated once the paint has finished — otherwise a * snapshot captures a half-drawn canvas (a single glyph). */ - private async paintFrame(timeMs: number): Promise { + private async paintFrame(timeMs: number, pipelineRevision: number): Promise { + if (!this.isPipelineBuildCurrent(pipelineRevision)) return; if (!this.layoutEngine || !this.captionLayout || !this.canvas || !this.painter || !this.validatedAsset || !this.generatorConfig) { return; } + const { layoutEngine, captionLayout, canvas, painter, validatedAsset, generatorConfig } = this; try { - const { ops } = generateRichCaptionFrame(this.validatedAsset, this.captionLayout, timeMs, this.layoutEngine, this.generatorConfig); + const { ops } = generateRichCaptionFrame(validatedAsset, captionLayout, timeMs, layoutEngine, generatorConfig); if (ops.length === 0 && this.sprite) { - this.sprite.visible = false; + if (this.isPipelineBuildCurrent(pipelineRevision)) this.sprite.visible = false; return; } - const ctx = this.canvas.getContext("2d"); - if (ctx) ctx.clearRect(0, 0, this.canvas.width, this.canvas.height); + const ctx = canvas.getContext("2d"); + if (ctx) ctx.clearRect(0, 0, canvas.width, canvas.height); - await this.painter.render(ops); + await painter.render(ops); + if (!this.isPipelineBuildCurrent(pipelineRevision)) return; if (!this.texture) { - this.texture = pixi.Texture.from(this.canvas); + this.texture = pixi.Texture.from(canvas); } else { this.texture.source.update(); } @@ -367,9 +402,7 @@ export class RichCaptionPlayer extends Player { }); } - private async registerFonts(asset: RichCaptionAsset): Promise { - if (!this.fontRegistry) return; - + private async registerFonts(asset: RichCaptionAsset, fontRegistry: FontRegistry): Promise { const family = asset.font?.family ?? "Roboto"; const assetWeight = asset.font?.weight ? parseInt(String(asset.font.weight), 10) || 400 : 400; @@ -379,19 +412,18 @@ export class RichCaptionPlayer extends Player { if (resolution.matched) { for (const font of resolution.fonts) { - if (font.src) await this.registerFontFromUrl(font.src, font.family, parseInt(font.weight, 10) || 400); + if (font.src) await this.registerFontFromUrl(fontRegistry, font.src, font.family, parseInt(font.weight, 10) || 400); } return; } const resolved = this.resolveFontWithWeight(family, assetWeight); if (resolved) { - await this.registerFontFromUrl(resolved.url, resolved.baseFontFamily, resolved.fontWeight); + await this.registerFontFromUrl(fontRegistry, resolved.url, resolved.baseFontFamily, resolved.fontWeight); } } - private async registerFontFromUrl(url: string, family: string, weight: number): Promise { - if (!this.fontRegistry) return false; + private async registerFontFromUrl(fontRegistry: FontRegistry, url: string, family: string, weight: number): Promise { const cacheKey = `${url}|${family}|${weight}`; const cached = this.fontRegistrationCache.get(cacheKey); if (cached) return cached; @@ -401,7 +433,7 @@ export class RichCaptionPlayer extends Player { const response = await fetch(url); if (!response.ok) return false; const bytes = await response.arrayBuffer(); - await this.fontRegistry!.registerFromBytes(bytes, { family, weight: weight.toString() }); + await fontRegistry.registerFromBytes(bytes, { family, weight: weight.toString() }); try { const fontFace = new FontFace(family, bytes, { @@ -514,8 +546,13 @@ export class RichCaptionPlayer extends Player { return []; } - private buildCanvasPayload(asset: RichCaptionAsset, words: WordTiming[]): Record { - const { width, height } = this.getSize(); + private buildCanvasPayload( + asset: RichCaptionAsset, + words: WordTiming[], + pauseThreshold: number, + size: { width: number; height: number } + ): Record { + const { width, height } = size; // Use the same resolution as registration, so the rendered family matches the registered face. const resolution = this.resolveFontsForAsset(asset); const resolvedFamily = resolution.matched ? resolution.resolvedFamily : getFontDisplayName(asset.font?.family ?? "Roboto"); @@ -541,7 +578,7 @@ export class RichCaptionPlayer extends Player { style: asset.style, animation: asset.animation, align: asset.align, - pauseThreshold: this.resolvedPauseThreshold + pauseThreshold }; for (const [key, value] of Object.entries(optionalFields)) { @@ -569,23 +606,54 @@ export class RichCaptionPlayer extends Player { wordWrapWidth: width }); - const fallbackText = new pixi.Text(message, style); - fallbackText.anchor.set(0.5, 0.5); - fallbackText.x = width / 2; - fallbackText.y = height / 2; + this.fallbackText = new pixi.Text(message, style); + this.fallbackText.anchor.set(0.5, 0.5); + this.fallbackText.x = width / 2; + this.fallbackText.y = height / 2; - this.contentContainer.addChild(fallbackText); + this.contentContainer.addChild(this.fallbackText); } - private cleanupResources(): void { - if (this.fontRegistry) { - try { - this.fontRegistry.release(); - } catch (e) { - console.warn("Error releasing font registry:", e); - } - this.fontRegistry = null; + private destroyRenderedOutput(): void { + if (this.sprite) { + this.contentContainer.removeChild(this.sprite); + this.sprite.destroy(); + this.sprite = null; } + this.texture?.destroy(); + this.texture = null; + + if (this.fallbackText) { + this.contentContainer.removeChild(this.fallbackText); + this.fallbackText.destroy(); + this.fallbackText = null; + } + } + + private replacePipelineWithFallback(message: string): void { + this.beginPipelineBuild(); + this.loadComplete = false; + this.destroyRenderedOutput(); + this.cleanupResources(); + this.words = []; + this.isPlaceholder = false; + this.needsResolution = false; + this.createFallbackGraphic(message); + } + + private releaseFontRegistry(fontRegistry: FontRegistry | null): void { + if (!fontRegistry) return; + try { + fontRegistry.release(); + } catch (error) { + console.warn("Error releasing font registry:", error); + } + } + + private cleanupResources(): void { + this.releaseFontRegistry(this.fontRegistry); + this.fontRegistry = null; + this.fontRegistrationCache.clear(); this.layoutEngine = null; this.captionLayout = null; @@ -593,23 +661,16 @@ export class RichCaptionPlayer extends Player { this.generatorConfig = null; this.canvas = null; this.painter = null; + this.renderConfigurationKey = null; } public override dispose(): void { - super.dispose(); + this.beginPipelineBuild(); this.loadComplete = false; - - if (this.texture) { - this.texture.destroy(); - } - this.texture = null; - - if (this.sprite) { - this.sprite.destroy(); - this.sprite = null; - } - + this.assetLoadInProgress = false; + this.destroyRenderedOutput(); this.cleanupResources(); + super.dispose(); } public override getSize(): Size { @@ -639,66 +700,7 @@ export class RichCaptionPlayer extends Player { protected override onDimensionsChanged(): void { if (this.words.length === 0) return; - this.rebuildForCurrentSize(); - } - - private async rebuildForCurrentSize(): Promise { - const currentTimeMs = this.getPlaybackTime() * 1000; - - if (this.texture) { - this.texture.destroy(); - this.texture = null; - } - if (this.sprite) { - this.contentContainer.removeChild(this.sprite); - this.sprite.destroy(); - this.sprite = null; - } - if (this.contentContainer.mask) { - const { mask } = this.contentContainer; - this.contentContainer.mask = null; - if (mask instanceof pixi.Graphics) { - mask.destroy(); - } - } - - this.captionLayout = null; - this.validatedAsset = null; - this.generatorConfig = null; - this.canvas = null; - this.painter = null; - - const { width, height } = this.getSize(); - const asset = this.clipConfiguration.asset as RichCaptionAsset; - - const canvasPayload = this.buildCanvasPayload(asset, this.words); - const canvasValidation = CanvasRichCaptionAssetSchema.safeParse(canvasPayload); - if (!canvasValidation.success) { - return; - } - this.validatedAsset = canvasValidation.data; - - this.generatorConfig = createDefaultGeneratorConfig(width, height, 1); - - this.canvas = document.createElement("canvas"); - this.canvas.width = width; - this.canvas.height = height; - this.painter = createWebPainter(this.canvas); - - if (!this.layoutEngine) return; - - const layoutConfig = buildCaptionLayoutConfig(this.validatedAsset, width, height); - - this.pendingLayoutId += 1; - const layoutId = this.pendingLayoutId; - - const layout = await this.layoutEngine.layoutCaption(this.words, layoutConfig); - - if (layoutId !== this.pendingLayoutId) return; - - this.captionLayout = layout; - - this.renderFrameSync(currentTimeMs); + this.reconfigure(); } public override supportsEdgeResize(): boolean { diff --git a/src/components/canvas/players/svg-player.ts b/src/components/canvas/players/svg-player.ts index 54b027fa..a57257d6 100644 --- a/src/components/canvas/players/svg-player.ts +++ b/src/components/canvas/players/svg-player.ts @@ -67,6 +67,7 @@ export class SvgPlayer extends Player { } public override async reloadAsset(): Promise { + await super.reloadAsset(); await this.rerenderAtCurrentDimensions(); } diff --git a/src/components/canvas/players/video-player.ts b/src/components/canvas/players/video-player.ts index 1c29b873..c330ee50 100644 --- a/src/components/canvas/players/video-player.ts +++ b/src/components/canvas/players/video-player.ts @@ -1,5 +1,6 @@ import { KeyframeBuilder } from "@animations/keyframe-builder"; import type { Edit } from "@core/edit-session"; +import { sec } from "@core/timing/types"; import { type Size } from "@layouts/geometry"; import { type ResolvedClip, type VideoAsset } from "@schemas"; import * as pixi from "pixi.js"; @@ -18,6 +19,7 @@ export class VideoPlayer extends Player { private syncTimer: number; private activeSyncTimer: number; private skipVideoUpdate: boolean; + private cancelVideoReadyWait: (() => void) | null; constructor(edit: Edit, clipConfiguration: ResolvedClip) { super(edit, clipConfiguration, PlayerType.Video); @@ -33,14 +35,21 @@ export class VideoPlayer extends Player { this.syncTimer = 0; this.activeSyncTimer = 0; this.skipVideoUpdate = false; + this.cancelVideoReadyWait = null; } public override async load(): Promise { + const mediaTimingRevision = this.beginMediaTimingLoad(); + this.cancelPendingVideoReadyWait(); await super.load(); + if (!this.isMediaTimingLoadCurrent(mediaTimingRevision)) return; try { - await this.loadVideo(); + if (!(await this.loadVideo(mediaTimingRevision))) return; + this.completeMediaTimingLoad(mediaTimingRevision, this.getLoadedDuration()); this.configureKeyframes(); } catch (error) { + if (!this.isMediaTimingLoadCurrent(mediaTimingRevision)) return; + this.completeMediaTimingLoad(mediaTimingRevision, null); console.warn(`[VideoPlayer.load] FAILED clipId=${this.clipId}:`, error); this.createFallbackGraphic(); } @@ -115,6 +124,7 @@ export class VideoPlayer extends Player { } public override dispose(): void { + this.cancelPendingVideoReadyWait(); this.disposeVideo(); this.clearPlaceholder(); super.dispose(); @@ -141,6 +151,8 @@ export class VideoPlayer extends Player { /** Reload the video asset when asset.src changes (e.g., merge field update) */ public override async reloadAsset(): Promise { + const mediaTimingRevision = this.beginMediaTimingLoad(); + this.cancelPendingVideoReadyWait(); this.skipVideoUpdate = true; this.isPlaying = false; this.syncTimer = 0; @@ -149,12 +161,15 @@ export class VideoPlayer extends Player { try { this.disposeVideo(); this.clearPlaceholder(); - await this.loadVideo(); + if (!(await this.loadVideo(mediaTimingRevision))) return; + this.completeMediaTimingLoad(mediaTimingRevision, this.getLoadedDuration()); } catch (error) { + if (!this.isMediaTimingLoadCurrent(mediaTimingRevision)) return; + this.completeMediaTimingLoad(mediaTimingRevision, null); console.warn(`[VideoPlayer.reloadAsset] FAILED clipId=${this.clipId}:`, error); this.createFallbackGraphic(); } finally { - this.skipVideoUpdate = false; + if (this.isMediaTimingLoadCurrent(mediaTimingRevision)) this.skipVideoUpdate = false; } } @@ -165,7 +180,7 @@ export class VideoPlayer extends Player { this.volumeKeyframeBuilder = new KeyframeBuilder(videoAsset.volume ?? 1, this.getLength()); } - private async loadVideo(): Promise { + private async loadVideo(mediaTimingRevision: number): Promise { const videoAsset = this.clipConfiguration.asset as VideoAsset; const { src } = videoAsset; if (!src) { @@ -187,42 +202,92 @@ export class VideoPlayer extends Player { if (!texture || !(texture.source instanceof pixi.VideoSource)) { throw new Error(`Invalid video source '${src}'.`); } - - this.clearPlaceholder(); + if (!this.isMediaTimingLoadCurrent(mediaTimingRevision)) { + this.destroyVideoTexture(texture); + return false; + } // Fix alpha channel rendering for WebM VP9 videos (PixiJS 8 auto-detection is buggy) texture.source.alphaMode = "no-premultiply-alpha"; - this.texture = this.createCroppedTexture(texture); + const loadedTexture = this.createCroppedTexture(texture); // Ensure the video has at least one decoded frame before adding to render tree // This prevents WebGL errors when GPU tries to upload uninitialized texture data - const video = (this.texture.source as pixi.VideoSource).resource; - if (video instanceof HTMLVideoElement && video.readyState < 2) { - await new Promise(resolve => { - const onReady = () => { - video.removeEventListener("loadeddata", onReady); - resolve(); - }; - video.addEventListener("loadeddata", onReady); - if (video.readyState >= 2) resolve(); - }); + const video = loadedTexture.source.resource; + try { + if (!(await this.waitForVideoReady(video, mediaTimingRevision)) || !this.isMediaTimingLoadCurrent(mediaTimingRevision)) { + this.destroyVideoTexture(loadedTexture); + return false; + } + } catch (error) { + this.destroyVideoTexture(loadedTexture); + throw error; } - this.sprite = new pixi.Sprite(this.texture); + this.clearPlaceholder(); + this.texture = loadedTexture; + this.sprite = new pixi.Sprite(loadedTexture); this.contentContainer.addChild(this.sprite); // Set initial volume immediately so the element never sits at the browser default of 1.0 this.texture.source.resource.volume = this.getVolume(); + return true; } - private disposeVideo(): void { - if (this.texture?.source?.resource) { - this.texture.source.resource.pause(); - // Release video resource - each player owns its own video element - this.texture.source.resource.src = ""; - this.texture.source.resource.load(); + private waitForVideoReady(video: HTMLVideoElement, mediaTimingRevision: number): Promise { + if (!(video instanceof HTMLVideoElement) || video.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA) { + return Promise.resolve(this.isMediaTimingLoadCurrent(mediaTimingRevision)); } + + return new Promise((resolve, reject) => { + let settled = false; + let onLoadedData: () => void; + let onError: () => void; + let onAbort: () => void; + let cancel: () => void; + + const cleanup = () => { + video.removeEventListener("loadeddata", onLoadedData); + video.removeEventListener("error", onError); + video.removeEventListener("abort", onAbort); + if (this.cancelVideoReadyWait === cancel) this.cancelVideoReadyWait = null; + }; + const settle = (ready: boolean, error?: Error) => { + if (settled) return; + settled = true; + cleanup(); + if (error) reject(error); + else resolve(ready); + }; + onLoadedData = () => { + settle(this.isMediaTimingLoadCurrent(mediaTimingRevision)); + }; + onError = () => { + settle(false, new Error("Video failed while waiting for its first decoded frame.")); + }; + onAbort = () => { + settle(false, new Error("Video loading was aborted before its first decoded frame.")); + }; + cancel = () => { + settle(false); + }; + + this.cancelVideoReadyWait = cancel; + video.addEventListener("loadeddata", onLoadedData); + video.addEventListener("error", onError); + video.addEventListener("abort", onAbort); + + if (!this.isMediaTimingLoadCurrent(mediaTimingRevision)) cancel(); + else if (video.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA) onLoadedData(); + }); + } + + private cancelPendingVideoReadyWait(): void { + this.cancelVideoReadyWait?.(); + } + + private disposeVideo(): void { if (this.sprite) { this.contentContainer.removeChild(this.sprite); this.sprite.destroy(); @@ -230,11 +295,24 @@ export class VideoPlayer extends Player { } // Destroy the texture since we own it (created via loadVideoUnique) if (this.texture) { - this.texture.destroy(true); + this.destroyVideoTexture(this.texture); this.texture = null; } } + private destroyVideoTexture(texture: pixi.Texture): void { + const { resource } = texture.source; + resource.pause(); + resource.src = ""; + resource.load(); + texture.destroy(true); + } + + private getLoadedDuration(): ReturnType | null { + const duration = this.texture?.source.resource.duration; + return duration !== undefined && Number.isFinite(duration) ? sec(duration) : null; + } + private clearPlaceholder(): void { if (!this.placeholder) { return; diff --git a/src/components/timeline/media-thumbnail-renderer.ts b/src/components/timeline/media-thumbnail-renderer.ts index ef3a43a4..11c43823 100644 --- a/src/components/timeline/media-thumbnail-renderer.ts +++ b/src/components/timeline/media-thumbnail-renderer.ts @@ -8,6 +8,7 @@ * - Image: Loads image directly and uses original URL */ +import type { GifThumbnail } from "@loaders/asset-loader"; import type { ResolvedClip, ImageAsset, VideoAsset } from "@schemas"; import type { ThumbnailGenerator } from "./thumbnail-generator"; @@ -26,6 +27,7 @@ const THUMBNAIL_HEIGHT = 72; export class MediaThumbnailRenderer implements ClipRenderer { private readonly generator: ThumbnailGenerator; private readonly onRendered: () => void; + private readonly getGifThumbnail?: (src: string) => Promise; // Track state per clip identity (src|trim|start) - NOT by element reference // This prevents state aliasing when elements are recycled or clips move @@ -33,9 +35,10 @@ export class MediaThumbnailRenderer implements ClipRenderer { private appliedToElement = new WeakMap(); - constructor(generator: ThumbnailGenerator, onRendered: () => void = () => {}) { + constructor(generator: ThumbnailGenerator, onRendered: () => void = () => {}, getGifThumbnail?: (src: string) => Promise) { this.generator = generator; this.onRendered = onRendered; + this.getGifThumbnail = getGifThumbnail; } /** @@ -125,7 +128,6 @@ export class MediaThumbnailRenderer implements ClipRenderer { private async generateAndApplyVideo(element: HTMLElement, asset: VideoAsset, clipKey: string): Promise { const state: ThumbnailState = { loading: true, thumbnails: [], thumbnailWidth: 0, failed: false }; this.clipStates.set(clipKey, state); - if (!asset.src) { state.loading = false; state.failed = true; @@ -161,7 +163,6 @@ export class MediaThumbnailRenderer implements ClipRenderer { private async generateAndApplyImage(element: HTMLElement, asset: ImageAsset, clipKey: string): Promise { const state: ThumbnailState = { loading: true, thumbnails: [], thumbnailWidth: 0, failed: false }; this.clipStates.set(clipKey, state); - if (!asset.src) { state.loading = false; state.failed = true; @@ -170,7 +171,19 @@ export class MediaThumbnailRenderer implements ClipRenderer { } try { - const result = await this.loadImageThumbnail(asset.src); + const gifThumbnail = await this.getGifThumbnail?.(asset.src); + let result: { url: string; thumbnailWidth: number } | null; + if (gifThumbnail?.isGif) { + result = + gifThumbnail.dataUrl && gifThumbnail.height > 0 + ? { + url: gifThumbnail.dataUrl, + thumbnailWidth: Math.round(THUMBNAIL_HEIGHT * (gifThumbnail.width / gifThumbnail.height)) + } + : null; + } else { + result = await this.loadImageThumbnail(asset.src); + } // Check if element is still in DOM (might have been disposed) if (!element.isConnected) return; diff --git a/src/components/timeline/timeline.ts b/src/components/timeline/timeline.ts index b606a842..61213546 100644 --- a/src/components/timeline/timeline.ts +++ b/src/components/timeline/timeline.ts @@ -92,15 +92,19 @@ export class Timeline { // Initialize media thumbnail generation (video and image) this.thumbnailGenerator = new ThumbnailGenerator(); - this.mediaThumbnailRenderer = new MediaThumbnailRenderer(this.thumbnailGenerator, () => { - if (!this.thumbnailRenderPending) { - this.thumbnailRenderPending = true; - requestAnimationFrame(() => { - this.thumbnailRenderPending = false; - this.requestRender(); - }); - } - }); + this.mediaThumbnailRenderer = new MediaThumbnailRenderer( + this.thumbnailGenerator, + () => { + if (!this.thumbnailRenderPending) { + this.thumbnailRenderPending = true; + requestAnimationFrame(() => { + this.thumbnailRenderPending = false; + this.requestRender(); + }); + } + }, + src => this.edit.assetLoader.getGifThumbnail(src) + ); this.clipRenderers.set("video", this.mediaThumbnailRenderer); this.clipRenderers.set("image", this.mediaThumbnailRenderer); diff --git a/src/core/commands/add-clip-command.ts b/src/core/commands/add-clip-command.ts index 57268ff2..5b789c44 100644 --- a/src/core/commands/add-clip-command.ts +++ b/src/core/commands/add-clip-command.ts @@ -19,7 +19,7 @@ export class AddClipCommand implements EditCommand { private clip: ClipType ) {} - execute(context?: CommandContext): CommandResult { + async execute(context?: CommandContext): Promise { if (!context) throw new Error("AddClipCommand.execute: context is required"); const document = context.getDocument(); @@ -38,6 +38,10 @@ export class AddClipCommand implements EditCommand { // Resolve triggers reconciler → creates Player (must happen before duration calc) context.resolve(); + if (addedClip.length === "auto" && this.addedClipId) { + const player = context.getPlayerByClipId(this.addedClipId); + if (player) await context.resolveClipAutoLength(player); + } context.updateDuration(); diff --git a/src/core/commands/types.ts b/src/core/commands/types.ts index 4f75e713..8ac999d3 100644 --- a/src/core/commands/types.ts +++ b/src/core/commands/types.ts @@ -108,7 +108,7 @@ export type CommandContext = { * * @param trackIdx - Track index * @param clipIdx - Clip index on that track - * @returns Context with previousClipEnd, timelineEnd, intrinsicDuration + * @returns Context with previousClipEnd, timelineEnd, and canonical auto length */ buildResolutionContext(trackIdx: number, clipIdx: number): ResolutionContext; diff --git a/src/core/debug/state-assertions.ts b/src/core/debug/state-assertions.ts index a0f740f7..97c2b659 100644 --- a/src/core/debug/state-assertions.ts +++ b/src/core/debug/state-assertions.ts @@ -65,7 +65,7 @@ export function assertTimingConsistency(intent: TimingIntent, actual: ResolvedTi throw new Error( `INVARIANT VIOLATION: Length mismatch\n` + ` Intent: ${formatTimingValue(intent.length)}\n` + - ` Context: timelineEnd=${context.timelineEnd}, intrinsicDuration=${context.intrinsicDuration}\n` + + ` Context: timelineEnd=${context.timelineEnd}, autoLength=${context.autoLength}\n` + ` Expected: ${expected.length}\n` + ` Actual: ${actual.length}\n` + ` Diff: ${lengthDiff}` diff --git a/src/core/edit-session.ts b/src/core/edit-session.ts index 10083f04..96d52f18 100644 --- a/src/core/edit-session.ts +++ b/src/core/edit-session.ts @@ -31,7 +31,7 @@ import { calculateSizeFromPreset, OutputSettingsManager } from "@core/output-set import { SelectionManager } from "@core/selection-manager"; import { findEligibleSourceClips, ensureClipAlias } from "@core/shared/source-clip-finder"; import { deepMerge, nextFrame, setNestedValue } from "@core/shared/utils"; -import { calculateTimelineEnd, resolveAutoLength, resolveAutoStart } from "@core/timing/resolver"; +import { calculateTimelineEnd, mediaTimingMatchesAsset, resolveAutoLength } from "@core/timing/resolver"; import { type Milliseconds, type ResolutionContext, type Seconds, sec, isAliasReference } from "@core/timing/types"; import { TimingManager } from "@core/timing-manager"; import type { Size } from "@layouts/geometry"; @@ -58,7 +58,7 @@ import { CommandQueue } from "./commands/command-queue"; import { CommandNoop, type EditCommand, type CommandContext, type CommandResult } from "./commands/types"; import { EditDocument } from "./edit-document"; import { PlayerReconciler } from "./player-reconciler"; -import { resolve as resolveDocument, resolveClip as resolveClipById, type SingleClipContext } from "./resolver"; +import { resolve as resolveDocument, resolveClip as resolveClipById, type ResolveContext, type SingleClipContext } from "./resolver"; import { InvalidAssetUrlError, extractClipUrls, extractTrackUrls } from "./url-validation"; /** Internal type for clips with hydrated IDs during edit updates */ @@ -121,6 +121,8 @@ export class Edit { private isBatchingEvents: boolean = false; private isExporting: boolean = false; private lastResolved: ResolvedEdit | null = null; + private pendingAutoLengthPlayers = new Set(); + private autoLengthResolutionPromise: Promise | null = null; /** * Create an Edit instance from a template configuration. @@ -222,13 +224,17 @@ export class Edit { } } - // 9. Resolve async timing (auto-length for videos, etc.) + // 9. Re-resolve with intrinsic metadata learned while Players loaded. This + // keeps aliases, auto starts, end lengths, and the resolved cache aligned. + this.resolve(); + + // 10. Resolve remaining async timing (auto-length for videos, etc.) await this.timingManager.resolveAllTiming(); - // 10. Update total duration + // 11. Update total duration this.updateTotalDuration(); - // 11. Load soundtrack if present + // 12. Load soundtrack if present if (parsedEdit.timeline.soundtrack) { await this.loadSoundtrack(parsedEdit.timeline.soundtrack); } @@ -496,13 +502,25 @@ export class Edit { */ public getResolvedEdit(): ResolvedEdit { if (!this.lastResolved) { - this.lastResolved = resolveDocument(this.document, { - mergeFields: this.mergeFieldService - }); + this.lastResolved = resolveDocument(this.document, this.getResolverContext()); } return this.lastResolved; } + private getResolverContext(): ResolveContext { + const mediaTiming = new Map>(); + for (const [clipId, player] of this.playerByClipId) { + if (player.getTimingIntent().length === "auto") { + mediaTiming.set(clipId, player.getMediaTimingState()); + } + } + + return { + mergeFields: this.mergeFieldService, + mediaTimingByClipId: mediaTiming + }; + } + /** * Get a specific clip from the resolved edit. * @internal @@ -553,9 +571,7 @@ export class Edit { /** @internal Resolve the document to a ResolvedEdit and emit the Resolved event. */ public resolve(): ResolvedEdit { - this.lastResolved = resolveDocument(this.document, { - mergeFields: this.mergeFieldService - }); + this.lastResolved = resolveDocument(this.document, this.getResolverContext()); // Emit event for components to react this.internalEvents.emit(InternalEvent.Resolved, { edit: this.lastResolved }); @@ -603,7 +619,8 @@ export class Edit { const context: SingleClipContext = { mergeFields: this.mergeFieldService, previousClipEnd, - cachedTimelineEnd: this.timingManager.getTimelineEnd() + cachedTimelineEnd: this.timingManager.getTimelineEnd(), + mediaTiming: player.getMediaTimingState() }; // Resolve just this one clip @@ -1691,23 +1708,23 @@ export class Edit { // 2. Timeline end excluding "end" clips (for length: "end") const timelineEnd = calculateTimelineEnd(this.tracks); - // 3. Intrinsic duration if available (for length: "auto") - // Note: This may be null if asset metadata hasn't loaded yet - let intrinsicDuration: Seconds | null = null; + // 3. Final asset-aware length if loaded metadata is available. + let autoLength: Seconds | null = null; const player = this.getClipAt(trackIdx, clipIdx); if (player) { const intent = player.getTimingIntent(); - // Only lookup intrinsic duration if the clip uses "auto" length if (intent.length === "auto") { - // The player's resolved length IS the intrinsic duration after async load - intrinsicDuration = player.getLength(); + const state = player.getMediaTimingState(); + if (state.status === "ready" && mediaTimingMatchesAsset(state, player.clipConfiguration.asset)) { + autoLength = resolveAutoLength(player.clipConfiguration.asset, state.duration); + } } } return { previousClipEnd, timelineEnd, - intrinsicDuration + autoLength }; }, resolve: () => this.resolve(), @@ -1855,12 +1872,12 @@ export class Edit { } private unloadClipAssets(clip: Player): void { - const { asset } = clip.clipConfiguration; - if (asset && "src" in asset && typeof asset.src === "string") { - const safeToUnload = this.assetLoader.decrementRef(asset.src); - if (safeToUnload && pixi.Assets.cache.has(asset.src)) { - pixi.Assets.unload(asset.src); - } + const identifier = clip.getLoadedResourceIdentifier(); + if (!identifier) return; + if (typeof this.assetLoader.release === "function") { + this.assetLoader.release(identifier); + } else if (this.assetLoader.decrementRef(identifier) && pixi.Assets.cache.has(identifier)) { + pixi.Assets.unload(identifier); } } @@ -1906,28 +1923,39 @@ export class Edit { } /** @internal */ - public async resolveClipAutoLength(clip: Player): Promise { - const intent = clip.getTimingIntent(); - if (intent.length !== "auto") return; - - // Find clip indices first (needed if start is also auto) - const indices = this.findClipIndices(clip); - - // Resolve auto start if needed, otherwise use current start - let resolvedStart = clip.getStart(); - if (intent.start === "auto" && indices) { - resolvedStart = resolveAutoStart(indices.trackIndex, indices.clipIndex, this.tracks); + public resolveClipAutoLength(clip: Player): Promise { + if (clip.getTimingIntent().length !== "auto") return Promise.resolve(); + this.pendingAutoLengthPlayers.add(clip); + + if (!this.autoLengthResolutionPromise) { + this.autoLengthResolutionPromise = Promise.resolve() + .then(() => this.flushAutoLengthResolutions()) + .finally(() => { + this.autoLengthResolutionPromise = null; + }); } - const newLength = await resolveAutoLength(clip.clipConfiguration.asset); - clip.setResolvedTiming({ - start: resolvedStart, - length: newLength - }); - clip.reconfigureAfterRestore(); + return this.autoLengthResolutionPromise; + } + + private async flushAutoLengthResolutions(): Promise { + while (this.pendingAutoLengthPlayers.size > 0) { + const batch = [...this.pendingAutoLengthPlayers]; + const pendingPlayers = batch.filter(player => player.getMediaTimingState().status === "pending"); + await Promise.all(pendingPlayers.map(player => this.playerReconciler.whenPlayerSettled(player))); + for (const player of batch) this.pendingAutoLengthPlayers.delete(player); - if (indices) { - this.propagateTimingChanges(indices.trackIndex, indices.clipIndex); + const currentPlayers = batch.filter( + player => + player.getTimingIntent().length === "auto" && + (!player.clipId || this.playerByClipId.get(player.clipId) === player) && + this.findClipIndices(player) !== null + ); + if (currentPlayers.length > 0) { + const indices = this.findClipIndices(currentPlayers[0]); + this.resolve(); + if (indices) this.propagateTimingChanges(indices.trackIndex, indices.clipIndex); + } } } diff --git a/src/core/loaders/asset-loader.ts b/src/core/loaders/asset-loader.ts index e68bc5c9..c34c6fb8 100644 --- a/src/core/loaders/asset-loader.ts +++ b/src/core/loaders/asset-loader.ts @@ -2,7 +2,18 @@ import * as pixi from "pixi.js"; import { AssetLoadTracker, type AssetLoadInfoStatus } from "../events/asset-load-tracker"; +import { GifImageSource } from "./gif-image-source"; +import { appendCorsQuery, isGifUrl } from "./gif-url"; + +export interface GifThumbnail { + readonly isGif: boolean; + readonly dataUrl: string | null; + readonly width: number; + readonly height: number; +} + export class AssetLoader { + private static readonly GIF_DETECTION_TIMEOUT_MS = 5_000; private static readonly VIDEO_EXTENSIONS = [".mp4", ".m4v", ".webm", ".ogg", ".ogv"]; private static readonly VIDEO_MIME: Record = { ".mp4": "video/mp4", @@ -15,6 +26,8 @@ export class AssetLoader { /** Reference counts for loaded assets - prevents premature unloading during transforms */ private refCounts = new Map(); + private gifDetections = new Map>(); + private gifSources = new Map>(); /** * Increment reference count for an asset. @@ -29,8 +42,9 @@ export class AssetLoader { * @returns true if asset can be safely unloaded (count reached zero) */ public decrementRef(src: string): boolean { - const count = this.refCounts.get(src) ?? 0; - if (count <= 1) { + const count = this.refCounts.get(src); + if (!count) return false; + if (count === 1) { this.refCounts.delete(src); return true; // Safe to unload } @@ -38,6 +52,22 @@ export class AssetLoader { return false; // Still in use } + /** Release a cached asset once its final Player reference is disposed. */ + public release(identifier: string): void { + if (!this.decrementRef(identifier)) return; + + const gifSource = this.gifSources.get(identifier); + if (gifSource) { + this.gifSources.delete(identifier); + gifSource.then(source => source.destroy()).catch(() => undefined); + } + this.gifDetections.delete(identifier); + + if (pixi.Assets.cache.has(identifier)) { + pixi.Assets.unload(identifier); + } + } + constructor() { pixi.Assets.setPreferences({ crossOrigin: "anonymous" }); } @@ -62,8 +92,8 @@ export class AssetLoader { const resolvedAsset = useSafari ? await this.loadVideoForSafari(identifier, loadOptions) : await pixi.Assets.load(loadOptions, progress => { - this.updateAssetLoadMetadata(identifier, "loading", progress); - }); + this.updateAssetLoadMetadata(identifier, "loading", progress); + }); if (resolvedAsset == null) { console.warn(`[AssetLoader.load] Empty asset returned for "${identifier}"`); @@ -82,6 +112,98 @@ export class AssetLoader { } } + /** Detect GIFs without relying on Pixi's suffix-only parser selection. */ + public isGif(identifier: string, requestUrl: string = appendCorsQuery(identifier)): Promise { + const cached = this.gifDetections.get(identifier); + if (cached) return cached; + + const detection = isGifUrl(identifier) ? Promise.resolve(true) : this.hasGifMagic(requestUrl); + + this.gifDetections.set(identifier, detection); + detection.catch(() => { + if (this.gifDetections.get(identifier) === detection) this.gifDetections.delete(identifier); + }); + return detection; + } + + private async hasGifMagic(requestUrl: string): Promise { + let reader: ReadableStreamDefaultReader | null = null; + try { + const response = await fetch(requestUrl, { + headers: { Range: "bytes=0-5" }, + signal: AbortSignal.timeout(AssetLoader.GIF_DETECTION_TIMEOUT_MS) + }); + if (!response.ok) { + await response.body?.cancel().catch(() => undefined); + throw new Error(`Unable to inspect image source (${response.status}).`); + } + if (!response.body) throw new Error("Image response body is not available as a readable stream."); + reader = response.body.getReader(); + const signature = new Uint8Array(6); + let offset = 0; + while (offset < signature.length) { + const { done, value } = await reader.read(); + if (done || !value) break; + const length = Math.min(value.byteLength, signature.length - offset); + signature.set(value.subarray(0, length), offset); + offset += length; + } + + if (offset !== signature.length) throw new Error("Image response ended before its signature could be inspected."); + const value = String.fromCharCode(...signature); + return value === "GIF87a" || value === "GIF89a"; + } finally { + if (reader) { + await reader.cancel().catch(() => undefined); + reader.releaseLock(); + } + } + } + + /** Load and share an eagerly decoded GIF source across clips using the same URL. */ + public async loadGif(identifier: string, requestUrl: string = appendCorsQuery(identifier)): Promise { + this.updateAssetLoadMetadata(identifier, "pending", 0); + this.incrementRef(identifier); + + let sourcePromise = this.gifSources.get(identifier); + if (!sourcePromise) { + sourcePromise = GifImageSource.fetch(requestUrl); + this.gifSources.set(identifier, sourcePromise); + } + + try { + this.updateAssetLoadMetadata(identifier, "loading", 0.5); + const source = await sourcePromise; + this.updateAssetLoadMetadata(identifier, "success", 1); + return source; + } catch (error) { + console.warn(`[AssetLoader.loadGif] Failed to load "${identifier}":`, error); + this.updateAssetLoadMetadata(identifier, "failed", 1); + this.release(identifier); + return null; + } + } + + /** Return the decoded first GIF frame for a non-animating timeline thumbnail. */ + public async getGifThumbnail(identifier: string): Promise { + const isGif = await this.isGif(identifier); + if (!isGif) return { isGif: false, dataUrl: null, width: 0, height: 0 }; + + const source = await this.loadGif(identifier); + if (!source) return { isGif: true, dataUrl: null, width: 0, height: 0 }; + try { + return { + isGif: true, + dataUrl: source.getFirstFrameDataUrl(), + width: source.width, + height: source.height + }; + } finally { + // The timeline only retains the generated PNG; the Player owns the decoded source. + this.release(identifier); + } + } + /** * Load a video with a unique HTMLVideoElement (not cached). * Each call creates an independent video element, allowing multiple VideoPlayers @@ -146,11 +268,14 @@ export class AssetLoader { } private async cleanupFailedLoad(identifier: string): Promise { - this.decrementRef(identifier); - try { - await pixi.Assets.unload(identifier); - } catch { - // Ignore unload errors for already-failed assets + const cached = pixi.Assets.cache.has(identifier); + this.release(identifier); + if (!cached) { + try { + await pixi.Assets.unload(identifier); + } catch { + // Ignore unload errors for assets that never reached the cache. + } } } diff --git a/src/core/loaders/gif-image-load-error.ts b/src/core/loaders/gif-image-load-error.ts new file mode 100644 index 00000000..9191042c --- /dev/null +++ b/src/core/loaders/gif-image-load-error.ts @@ -0,0 +1,6 @@ +export class GifImageLoadError extends Error { + constructor(message: string, options?: ErrorOptions) { + super(message, options); + this.name = "GifImageLoadError"; + } +} diff --git a/src/core/loaders/gif-image-source.ts b/src/core/loaders/gif-image-source.ts new file mode 100644 index 00000000..150dac53 --- /dev/null +++ b/src/core/loaders/gif-image-source.ts @@ -0,0 +1,441 @@ +import * as pixi from "pixi.js"; +import { GifSource as PixiGifSource } from "pixi.js/gif"; + +export const GIF_DECODE_LIMITS = Object.freeze({ + maxCompressedBytes: 50 * 1024 * 1024, + maxDecodedBytes: 128 * 1024 * 1024, + maxDimension: 4096, + maxFrames: 1000, + maxCycleDurationMs: 300_000 +}); + +export interface GifFrameTexture { + readonly start: number; + readonly end: number; + readonly texture: pixi.Texture; +} + +export interface GifInspection { + readonly width: number; + readonly height: number; + readonly frameCount: number; + readonly decodedBytes: number; +} + +interface GifScanResult { + readonly inspection: GifInspection; + readonly durationMs: number; + readonly imagesWithoutControlExtension: number[]; +} + +const GIF_REQUEST_TIMEOUT_MS = 15_000; +const DEFAULT_FRAME_DELAY_MS = 100; +const DEFAULT_GRAPHIC_CONTROL_EXTENSION = Uint8Array.of(0x21, 0xf9, 0x04, 0x00, 0x0a, 0x00, 0x00, 0x00); + +interface GifReader { + readonly position: number; + readonly hasRemaining: boolean; + readByte(): number; + readUint16(): number; + skip(byteLength: number): void; + skipSubBlocks(): void; +} + +function createGifReader(bytes: Uint8Array): GifReader { + let offset = 0; + const requireBytes = (byteLength: number): void => { + if (!Number.isSafeInteger(byteLength) || byteLength < 0 || offset + byteLength > bytes.byteLength) { + throw new Error("GIF data is truncated or malformed."); + } + }; + const readByte = (): number => { + requireBytes(1); + const value = bytes[offset]; + offset += 1; + return value; + }; + const skip = (byteLength: number): void => { + requireBytes(byteLength); + offset += byteLength; + }; + + return { + get position(): number { + return offset; + }, + get hasRemaining(): boolean { + return offset < bytes.byteLength; + }, + readByte, + readUint16: () => readByte() + readByte() * 256, + skip, + skipSubBlocks: () => { + for (;;) { + const byteLength = readByte(); + if (byteLength === 0) return; + skip(byteLength); + } + } + }; +} + +function validateCompressedSize(byteLength: number): void { + if (byteLength <= 0) { + throw new Error("GIF data is empty."); + } + if (byteLength > GIF_DECODE_LIMITS.maxCompressedBytes) { + throw new Error(`GIF exceeds the ${GIF_DECODE_LIMITS.maxCompressedBytes} byte compressed-size limit.`); + } +} + +function getColorTableByteLength(packedFields: number): number { + return 3 * 2 ** ((packedFields % 8) + 1); +} + +function validateGifSignature(bytes: Uint8Array): void { + if (bytes.byteLength < 6) throw new Error("GIF data is truncated or malformed."); + const signature = String.fromCharCode(...bytes.subarray(0, 6)); + if (signature !== "GIF87a" && signature !== "GIF89a") { + throw new Error("Asset does not contain a valid GIF signature."); + } +} + +function validateFrameDescriptor(left: number, top: number, width: number, height: number, screenWidth: number, screenHeight: number): void { + const right = left + width; + const bottom = top + height; + if ( + ![left, top, width, height, right, bottom].every(Number.isSafeInteger) || + width <= 0 || + height <= 0 || + right > screenWidth || + bottom > screenHeight + ) { + throw new Error("GIF image frame descriptor is invalid or exceeds the logical screen."); + } +} + +export function validateGifCycleDuration(durationMs: number): void { + if (!Number.isFinite(durationMs) || durationMs <= 0) { + throw new Error("GIF has an invalid animation cycle duration."); + } + if (durationMs > GIF_DECODE_LIMITS.maxCycleDurationMs) { + throw new Error(`GIF animation cycle exceeds the ${GIF_DECODE_LIMITS.maxCycleDurationMs}ms limit.`); + } +} + +function scanGif(buffer: ArrayBuffer): GifScanResult { + validateCompressedSize(buffer.byteLength); + const bytes = new Uint8Array(buffer); + validateGifSignature(bytes); + + const reader = createGifReader(bytes); + reader.skip(6); + const width = reader.readUint16(); + const height = reader.readUint16(); + const screenPackedFields = reader.readByte(); + reader.skip(2); + + if (width <= 0 || height <= 0) { + throw new Error("GIF has invalid dimensions or no image frames."); + } + if (width > GIF_DECODE_LIMITS.maxDimension || height > GIF_DECODE_LIMITS.maxDimension) { + throw new Error(`GIF dimensions exceed the ${GIF_DECODE_LIMITS.maxDimension}px limit.`); + } + if (screenPackedFields >= 0x80) { + reader.skip(getColorTableByteLength(screenPackedFields)); + } + + const fullFrameBytes = width * height * 4; + const imagesWithoutControlExtension: number[] = []; + let frameCount = 0; + let patchBytes = 0; + let maxPatchBytes = 0; + let decodedBytes = fullFrameBytes * 3; + let durationMs = 0; + let pendingFrameDelayMs: number | null = null; + let foundTrailer = false; + + while (reader.hasRemaining) { + const blockOffset = reader.position; + const introducer = reader.readByte(); + + if (introducer === 0x3b) { + foundTrailer = true; + break; + } + + if (introducer === 0x21) { + const extensionLabel = reader.readByte(); + if (extensionLabel === 0xf9) { + if (reader.readByte() !== 4) throw new Error("GIF graphic control extension is malformed."); + reader.readByte(); + const delayCentiseconds = reader.readUint16(); + reader.readByte(); + if (reader.readByte() !== 0) throw new Error("GIF graphic control extension is malformed."); + pendingFrameDelayMs = (delayCentiseconds || 10) * 10; + } else { + reader.skipSubBlocks(); + // A graphic control extension applies to the next rendered graphic, including plain text. + if (extensionLabel === 0x01) pendingFrameDelayMs = null; + } + } else if (introducer === 0x2c) { + if (pendingFrameDelayMs === null) imagesWithoutControlExtension.push(blockOffset); + durationMs += pendingFrameDelayMs ?? DEFAULT_FRAME_DELAY_MS; + pendingFrameDelayMs = null; + + const left = reader.readUint16(); + const top = reader.readUint16(); + const frameWidth = reader.readUint16(); + const frameHeight = reader.readUint16(); + validateFrameDescriptor(left, top, frameWidth, frameHeight, width, height); + + const imagePackedFields = reader.readByte(); + if (imagePackedFields >= 0x80) { + reader.skip(getColorTableByteLength(imagePackedFields)); + } + reader.readByte(); + reader.skipSubBlocks(); + + frameCount += 1; + const framePatchBytes = frameWidth * frameHeight * 4; + patchBytes += framePatchBytes; + maxPatchBytes = Math.max(maxPatchBytes, framePatchBytes); + if (frameCount > GIF_DECODE_LIMITS.maxFrames) { + throw new Error(`GIF exceeds the ${GIF_DECODE_LIMITS.maxFrames} frame limit.`); + } + // Pixi retains each composited frame plus patch arrays, and uses three + // full-frame and two patch-sized buffers while compositing. + decodedBytes = fullFrameBytes * (frameCount + 3) + patchBytes + maxPatchBytes * 2; + if (!Number.isSafeInteger(decodedBytes) || decodedBytes > GIF_DECODE_LIMITS.maxDecodedBytes) { + throw new Error(`GIF decoded frames exceed the ${GIF_DECODE_LIMITS.maxDecodedBytes} byte limit.`); + } + if (durationMs > GIF_DECODE_LIMITS.maxCycleDurationMs) { + throw new Error(`GIF animation cycle exceeds the ${GIF_DECODE_LIMITS.maxCycleDurationMs}ms limit.`); + } + } else { + throw new Error("GIF contains an unsupported or malformed block."); + } + } + + if (!foundTrailer) throw new Error("GIF data is truncated or malformed."); + if (frameCount === 0) throw new Error("GIF has invalid dimensions or no image frames."); + validateGifCycleDuration(durationMs); + + return { + inspection: { + width, + height, + frameCount, + decodedBytes + }, + durationMs, + imagesWithoutControlExtension + }; +} + +function addDefaultGraphicControlExtensions(buffer: ArrayBuffer, imageOffsets: readonly number[]): ArrayBuffer { + if (imageOffsets.length === 0) return buffer; + + const input = new Uint8Array(buffer); + const output = new Uint8Array(input.byteLength + imageOffsets.length * DEFAULT_GRAPHIC_CONTROL_EXTENSION.byteLength); + let inputOffset = 0; + let outputOffset = 0; + + for (const imageOffset of imageOffsets) { + const segment = input.subarray(inputOffset, imageOffset); + output.set(segment, outputOffset); + outputOffset += segment.byteLength; + output.set(DEFAULT_GRAPHIC_CONTROL_EXTENSION, outputOffset); + outputOffset += DEFAULT_GRAPHIC_CONTROL_EXTENSION.byteLength; + inputOffset = imageOffset; + } + + output.set(input.subarray(inputOffset), outputOffset); + return output.buffer; +} + +export function inspectGif(buffer: ArrayBuffer): GifInspection { + return scanGif(buffer).inspection; +} + +export function validateGifContentLength(contentLength: string | null): void { + if (!contentLength) return; + + const byteLength = Number(contentLength); + if (Number.isFinite(byteLength)) { + validateCompressedSize(byteLength); + } +} + +export async function readGifResponse(response: Response): Promise { + try { + validateGifContentLength(response.headers.get("content-length")); + } catch (error) { + await response.body?.cancel(error).catch(() => undefined); + throw error; + } + if (!response.body) { + throw new Error("GIF response body is not available as a readable stream."); + } + + const reader = response.body.getReader(); + const chunks: Uint8Array[] = []; + let byteLength = 0; + + try { + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + if (value) { + byteLength += value.byteLength; + validateCompressedSize(byteLength); + chunks.push(value); + } + } + } catch (error) { + await reader.cancel(error).catch(() => undefined); + throw error; + } finally { + reader.releaseLock(); + } + + validateCompressedSize(byteLength); + const bytes = new Uint8Array(byteLength); + let offset = 0; + for (const chunk of chunks) { + bytes.set(chunk, offset); + offset += chunk.byteLength; + } + return bytes.buffer; +} + +export function selectGifFrame(frames: readonly Pick[], timeMs: number, durationMs: number): number { + if (frames.length === 0 || durationMs <= 0) return 0; + + const localTime = ((timeMs % durationMs) + durationMs) % durationMs; + let low = 0; + let high = frames.length - 1; + + while (low <= high) { + const middle = Math.floor((low + high) / 2); + const frame = frames[middle]; + if (localTime < frame.start) { + high = middle - 1; + } else if (localTime >= frame.end) { + low = middle + 1; + } else { + return middle; + } + } + + return Math.min(low, frames.length - 1); +} + +export function getAnimatedGifDurationMs(source: Pick | null): number | null { + return source && source.totalFrames > 1 ? source.duration : null; +} + +export class GifImageSource { + public readonly width: number; + public readonly height: number; + public readonly duration: number; + public readonly frames: GifFrameTexture[]; + public readonly totalFrames: number; + + private firstFrameDataUrl: string | null = null; + private destroyed = false; + + private constructor(source: PixiGifSource) { + this.width = source.width; + this.height = source.height; + this.totalFrames = source.totalFrames; + this.duration = source.duration; + this.frames = []; + + try { + for (const frame of source.frames) { + const { resource } = frame.texture.source; + if (!resource) throw new Error("GIF decoder returned a frame without a canvas resource."); + + this.frames.push({ + start: frame.start, + end: frame.end, + texture: new pixi.Texture({ + source: new pixi.CanvasSource({ resource }) + }) + }); + } + } catch (error) { + for (const frame of this.frames) frame.texture.destroy(true); + this.frames.length = 0; + throw error; + } finally { + // The decoder is bundled, while these replacement textures belong to the host Pixi runtime. + source.destroy(); + } + } + + public static async fetch(url: string): Promise { + const response = await fetch(url, { signal: AbortSignal.timeout(GIF_REQUEST_TIMEOUT_MS) }); + if (!response.ok) { + throw new Error(`GIF request failed with HTTP ${response.status}.`); + } + + const buffer = await readGifResponse(response); + return GifImageSource.from(buffer); + } + + public static from(buffer: ArrayBuffer): GifImageSource { + const { inspection, durationMs, imagesWithoutControlExtension } = scanGif(buffer); + const normalizedBuffer = addDefaultGraphicControlExtensions(buffer, imagesWithoutControlExtension); + const source = PixiGifSource.from(normalizedBuffer, { fps: 10 }); + + if ( + source.width !== inspection.width || + source.height !== inspection.height || + source.totalFrames !== inspection.frameCount || + source.duration !== durationMs + ) { + source.destroy(); + throw new Error("GIF frame metadata changed during decoding."); + } + + return new GifImageSource(source); + } + + public frameIndexAt(timeMs: number): number { + return selectGifFrame(this.frames, timeMs, this.duration); + } + + public getFirstFrameDataUrl(maxDimension = 128): string | null { + if (this.firstFrameDataUrl) return this.firstFrameDataUrl; + + const resource = this.frames[0]?.texture.source.resource; + if (typeof HTMLCanvasElement !== "undefined" && resource instanceof HTMLCanvasElement) { + const scale = Math.min(1, maxDimension / Math.max(resource.width, resource.height)); + if (scale < 1) { + const thumbnail = document.createElement("canvas"); + thumbnail.width = Math.max(1, Math.round(resource.width * scale)); + thumbnail.height = Math.max(1, Math.round(resource.height * scale)); + const context = thumbnail.getContext("2d"); + if (context) { + context.drawImage(resource, 0, 0, thumbnail.width, thumbnail.height); + this.firstFrameDataUrl = thumbnail.toDataURL("image/png"); + } + thumbnail.width = 0; + thumbnail.height = 0; + } else { + this.firstFrameDataUrl = resource.toDataURL("image/png"); + } + } + return this.firstFrameDataUrl; + } + + public destroy(): void { + if (this.destroyed) return; + this.destroyed = true; + for (const frame of this.frames) frame.texture.destroy(true); + this.frames.length = 0; + this.firstFrameDataUrl = null; + } +} diff --git a/src/core/loaders/gif-url.ts b/src/core/loaders/gif-url.ts new file mode 100644 index 00000000..e9892fb8 --- /dev/null +++ b/src/core/loaders/gif-url.ts @@ -0,0 +1,24 @@ +const GIF_DATA_URL_PATTERN = /^data:image\/gif(?:;|,)/i; + +export function getUrlExtension(src: string): string | null { + if (src.startsWith("data:")) return null; + + try { + const { pathname } = new URL(src, typeof window === "undefined" ? "http://localhost" : window.location.origin); + // eslint-disable-next-line prefer-destructuring -- The final path segment is required, not the first. + const filename = pathname.split("/").at(-1) ?? ""; + const extensionStart = filename.lastIndexOf("."); + return extensionStart >= 0 ? filename.slice(extensionStart).toLowerCase() : null; + } catch { + return null; + } +} + +export function isGifUrl(src: string): boolean { + return GIF_DATA_URL_PATTERN.test(src) || getUrlExtension(src) === ".gif"; +} + +export function appendCorsQuery(src: string): string { + if (!/^https?:\/\//i.test(src)) return src; + return `${src}${src.includes("?") ? "&" : "?"}x-cors=1`; +} diff --git a/src/core/player-reconciler.ts b/src/core/player-reconciler.ts index 82c41de0..a1485e7a 100644 --- a/src/core/player-reconciler.ts +++ b/src/core/player-reconciler.ts @@ -14,6 +14,7 @@ import type { ResolvedClip, ResolvedEdit } from "@schemas"; import type { Edit } from "./edit-session"; import { EditEvent, InternalEvent } from "./events/edit-events"; import { isPendingAiAsset } from "./shared/ai-asset-utils"; +import { assetTimingIdentitiesEqual, getAssetTimingIdentity } from "./timing/resolver"; import type { Seconds } from "./timing/types"; export interface ReconcileResult { @@ -39,6 +40,8 @@ export class PlayerReconciler { /** In-flight player load promises, so off-playback captures (captureFrame) can await asset readiness. */ private readonly inFlightLoads = new Set>(); + private readonly playerLoads = new WeakMap>>(); + private isInitialReconcile = false; constructor(private readonly edit: Edit) { this.edit.getInternalEvents().on(InternalEvent.Resolved, this.onResolved); @@ -59,9 +62,14 @@ export class PlayerReconciler { * @returns Promise that resolves when all players are loaded */ public async reconcileInitial(resolved: ResolvedEdit): Promise { - const result = this.reconcile(resolved); - await Promise.all(result.pendingLoads); - return result; + this.isInitialReconcile = true; + try { + const result = this.reconcile(resolved); + await Promise.all(result.pendingLoads); + return result; + } finally { + this.isInitialReconcile = false; + } } /** @@ -76,6 +84,13 @@ export class PlayerReconciler { } } + /** Wait for the current initial load or asset reload for one Player. */ + public async whenPlayerSettled(player: Player): Promise { + while (this.playerLoads.has(player)) { + await Promise.allSettled([...(this.playerLoads.get(player) ?? [])]); + } + } + /** * Reconcile Players to match the ResolvedEdit. * @@ -130,7 +145,7 @@ export class PlayerReconciler { } } else if (this.enableCreation) { // Create new Player - this.createPlayer(clip, clipId, trackIndex, clipIndex); + pendingLoads.push(this.createPlayer(clip, clipId, trackIndex, clipIndex)); result.created.push(clipId); } } @@ -247,9 +262,7 @@ export class PlayerReconciler { }); }); - // Track the load so off-playback captures can await asset readiness (see whenSettled()). - this.inFlightLoads.add(loadPromise); - return loadPromise.finally(() => this.inFlightLoads.delete(loadPromise)); + return this.trackPlayerLoad(player, loadPromise); } /** @@ -358,6 +371,10 @@ export class PlayerReconciler { private updateAsset(player: Player, newAsset: unknown): void { const oldAsset = player.clipConfiguration.asset; const assetType = (newAsset as { type?: string })?.type; + const intrinsicContentChanged = !assetTimingIdentitiesEqual( + getAssetTimingIdentity(oldAsset), + getAssetTimingIdentity(newAsset as ResolvedClip["asset"]) + ); // eslint-disable-next-line no-param-reassign -- Intentional player state update player.clipConfiguration.asset = newAsset as ResolvedClip["asset"]; @@ -366,13 +383,11 @@ export class PlayerReconciler { if (assetType === "html5") { needsReload = JSON.stringify(oldAsset) !== JSON.stringify(newAsset); } else { - const oldSrc = (oldAsset as { src?: string })?.src; - const newSrc = (newAsset as { src?: string })?.src; - needsReload = oldSrc !== newSrc; + needsReload = intrinsicContentChanged; } if (needsReload && player.reloadAsset) { - player + const reloadPromise = player .reloadAsset() .then(() => { player.reconfigureAfterRestore(); @@ -380,11 +395,43 @@ export class PlayerReconciler { .catch(error => { console.error("Failed to reload asset:", error); }); + this.trackPlayerLoad(player, reloadPromise); } else { player.reconfigureAfterRestore(); } } + private trackPlayerLoad(player: Player, loadPromise: Promise): Promise { + this.inFlightLoads.add(loadPromise); + let loads = this.playerLoads.get(player); + if (!loads) { + loads = new Set(); + this.playerLoads.set(player, loads); + } + loads.add(loadPromise); + + const cleanup = (): void => { + this.inFlightLoads.delete(loadPromise); + const currentLoads = this.playerLoads.get(player); + currentLoads?.delete(loadPromise); + if (currentLoads?.size === 0) { + this.playerLoads.delete(player); + const currentPlayer = player.clipId ? this.edit.getPlayerMap().get(player.clipId) : null; + if (!this.isInitialReconcile && currentPlayer === player && player.getTimingIntent().length === "auto") { + queueMicrotask(() => { + if (player.clipId && this.edit.getPlayerMap().get(player.clipId) === player) { + this.edit.resolveClipAutoLength(player).catch(error => { + console.error("Failed to resolve auto clip timing:", error); + }); + } + }); + } + } + }; + loadPromise.then(cleanup, cleanup); + return loadPromise; + } + /** * Sync PIXI track containers to match resolved track count. * Creates new containers for added tracks, removes empty containers for deleted tracks. diff --git a/src/core/resolver.ts b/src/core/resolver.ts index 1df2e621..bc9d31db 100644 --- a/src/core/resolver.ts +++ b/src/core/resolver.ts @@ -18,12 +18,14 @@ import type { EditDocument } from "./edit-document"; import type { MergeFieldService } from "./merge/merge-field-service"; import type { Clip, ResolvedClip, ResolvedEdit, ResolvedTrack } from "./schemas"; -import { type Seconds, sec, isAliasReference, parseAliasName } from "./timing/types"; +import { mediaTimingMatchesAsset, resolveAutoLength, resolveEndLength } from "./timing/resolver"; +import { type MediaTimingState, type Seconds, sec, isAliasReference, parseAliasName } from "./timing/types"; // ─── Types ──────────────────────────────────────────────────────────────────── export interface ResolveContext { mergeFields: MergeFieldService; + mediaTimingByClipId?: ReadonlyMap; } /** @@ -81,7 +83,7 @@ function resolveMergeFieldsInClip(clip: InternalClip, mergeFields: MergeFieldSer return num !== null ? num : mergeFields.resolve(value); } if (Array.isArray(value)) { - return value.map((item) => processValue(item, key)); + return value.map(item => processValue(item, key)); } if (value !== null && typeof value === "object") { const result: Record = {}; @@ -246,7 +248,12 @@ function topologicalSort(dependencies: Map>, allClipIds: str * This is called after topological sorting ensures dependencies are resolved first. * Note: Merge fields should be resolved via resolveMergeFieldsInClip() BEFORE calling this. */ -function resolveClipWithAliases(clip: InternalClip, previousClipEnd: Seconds, resolvedAliases: Map): PartialResolvedClip { +function resolveClipWithAliases( + clip: InternalClip, + previousClipEnd: Seconds, + resolvedAliases: Map, + autoLength?: Seconds +): PartialResolvedClip { // Resolve start let start: Seconds; if (clip.start === "auto") { @@ -269,9 +276,7 @@ function resolveClipWithAliases(clip: InternalClip, previousClipEnd: Seconds, re length = sec(1); // Temporary placeholder pendingEndLength = true; } else if (clip.length === "auto") { - // Use intrinsic duration if available, else fallback - // Note: For now use fallback; intrinsic duration will be provided by Players - length = sec(3); + length = autoLength ?? resolveAutoLength(clip.asset); } else if (isAliasReference(clip.length)) { const aliasName = parseAliasName(clip.length); const aliasValue = resolvedAliases.get(aliasName); @@ -341,6 +346,14 @@ export interface SingleClipContext extends ResolveContext { * Resolved alias values for alias reference resolution. */ resolvedAliases?: Map; + + /** Loaded media timing for this clip; the resolver validates its asset key. */ + mediaTiming?: MediaTimingState; +} + +function getResolvedAutoLength(clip: InternalClip, mediaTiming?: MediaTimingState): Seconds { + const duration = mediaTiming?.status === "ready" && mediaTimingMatchesAsset(mediaTiming, clip.asset) ? mediaTiming.duration : null; + return resolveAutoLength(clip.asset, duration); } /** @@ -379,11 +392,12 @@ export function resolveClip(document: EditDocument, clipId: string, context: Sin // 3. Resolve the single clip using alias-aware logic const resolvedAliases = context.resolvedAliases ?? new Map(); - const resolvedClip = resolveClipWithAliases(processedClip, context.previousClipEnd, resolvedAliases); + const autoLength = processedClip.length === "auto" ? getResolvedAutoLength(processedClip, context.mediaTiming) : undefined; + const resolvedClip = resolveClipWithAliases(processedClip, context.previousClipEnd, resolvedAliases, autoLength); // 4. Handle "end" length (second pass for this single clip) if (resolvedClip.pendingEndLength && context.cachedTimelineEnd !== undefined) { - resolvedClip.length = sec(Math.max(context.cachedTimelineEnd - resolvedClip.start, 0.1)); + resolvedClip.length = resolveEndLength(context.cachedTimelineEnd, resolvedClip.start); } // 5. Clean up and return @@ -453,7 +467,9 @@ export function resolve(document: EditDocument, context: ResolveContext): Resolv const previousClipEnd = previousClipEndByTrack.get(trackIndex) ?? sec(0); // Resolve the clip with alias support - const resolvedClip = resolveClipWithAliases(processedClip, previousClipEnd, resolvedAliases); + const mediaTiming = processedClip.length === "auto" && clip.id ? context.mediaTimingByClipId?.get(clip.id) : undefined; + const autoLength = processedClip.length === "auto" ? getResolvedAutoLength(processedClip, mediaTiming) : undefined; + const resolvedClip = resolveClipWithAliases(processedClip, previousClipEnd, resolvedAliases, autoLength); // Store in map by position key resolvedClipsByPosition.set(`${trackIndex}-${clipIndex}`, resolvedClip); @@ -496,8 +512,7 @@ export function resolve(document: EditDocument, context: ResolveContext): Resolv const tracks: ResolvedTrack[] = partialTracks.map(track => ({ clips: track.clips.map(clip => { if (clip.pendingEndLength) { - // Resolve "end" length now that we know the timeline end - const resolvedLength = sec(Math.max(timelineEnd - clip.start, 0.1)); + const resolvedLength = resolveEndLength(sec(timelineEnd), clip.start); return cleanupPendingFlags({ ...clip, length: resolvedLength }); } return cleanupPendingFlags(clip); diff --git a/src/core/timing-manager.ts b/src/core/timing-manager.ts index cb3a4c25..cd682de5 100644 --- a/src/core/timing-manager.ts +++ b/src/core/timing-manager.ts @@ -4,7 +4,7 @@ import type { Player } from "@canvas/players/player"; import { EditEvent } from "@core/events/edit-events"; -import { calculateTimelineEnd, resolveAutoLength, resolveAutoStart, resolveEndLength } from "@core/timing/resolver"; +import { calculateTimelineEnd, resolveAutoStart, resolveEndLength } from "@core/timing/resolver"; import { type Seconds, isAliasReference, sec } from "@core/timing/types"; import type { Edit } from "./edit-session"; @@ -53,22 +53,15 @@ export class TimingManager { const resolvedClip = resolvedTrack?.clips[clipIdx]; if (resolvedClip) { - const intent = player.getTimingIntent(); - - // Use resolved values from the resolver + const previousStart = player.getStart(); + const previousLength = player.getLength(); const resolvedStart = resolvedClip.start; - let resolvedLength = resolvedClip.length; - - // Special handling for "auto" length - requires async asset loading - if (intent.length === "auto") { - resolvedLength = await resolveAutoLength(player.clipConfiguration.asset); - } + const resolvedLength = resolvedClip.length; player.setResolvedTiming({ start: resolvedStart, length: resolvedLength }); - - // Sync resolved edit cache so timeline UI sees actual timing - resolvedClip.start = resolvedStart; - resolvedClip.length = resolvedLength; + if (resolvedStart !== previousStart || resolvedLength !== previousLength) { + player.reconfigureAfterRestore(); + } } } } @@ -76,32 +69,6 @@ export class TimingManager { // Calculate timeline end and cache it const timelineEnd = calculateTimelineEnd(tracks); this.cachedTimelineEnd = timelineEnd; - - // Resolve "end" clips now that we have the final timeline end - // (accounts for any "auto" length changes from async resolution above) - const endLengthClips = this.getEndLengthClips(); - for (const clip of endLengthClips) { - const currentTiming = clip.getResolvedTiming(); - const endLength = resolveEndLength(currentTiming.start, timelineEnd); - clip.setResolvedTiming({ - start: currentTiming.start, - length: endLength - }); - - // Sync resolved edit cache for "end" clips - const trackIdx = clip.layer - 1; - const clipIdx = tracks[trackIdx]?.indexOf(clip) ?? -1; - const endResolvedClip = resolved.timeline.tracks[trackIdx]?.clips[clipIdx]; - if (endResolvedClip) { - endResolvedClip.start = currentTiming.start; - endResolvedClip.length = endLength; - } - } - - // Reconfigure "end" clips to rebuild keyframes - for (const clip of endLengthClips) { - clip.reconfigureAfterRestore(); - } } // ─── Propagation ───────────────────────────────────────────────────────── @@ -137,7 +104,7 @@ export class TimingManager { const endLengthClips = this.getEndLengthClips(); for (const clip of endLengthClips) { - const newLength = resolveEndLength(clip.getStart(), newTimelineEnd); + const newLength = resolveEndLength(newTimelineEnd, clip.getStart()); const currentLength = clip.getLength(); if (Math.abs(newLength - currentLength) > 0.001) { diff --git a/src/core/timing/resolver.ts b/src/core/timing/resolver.ts index 1dd9067c..a32b9d5d 100644 --- a/src/core/timing/resolver.ts +++ b/src/core/timing/resolver.ts @@ -6,11 +6,58 @@ import type { Player } from "@canvas/players/player"; import type { Asset } from "@schemas"; -import { type ResolutionContext, type ResolvedTiming, type Seconds, type TimingIntent, isAliasReference, sec } from "./types"; +import { + type AssetTimingIdentity, + type MediaTimingState, + type ResolutionContext, + type ResolvedTiming, + type Seconds, + type TimingIntent, + isAliasReference, + sec +} from "./types"; + +export const DEFAULT_AUTO_CLIP_LENGTH = sec(3); +export const MINIMUM_END_LENGTH = sec(0); + +/** Stable identity for intrinsic metadata. Trim is excluded because it is applied by the resolver. */ +export function getAssetTimingIdentity(asset: Asset): AssetTimingIdentity { + const { type } = asset as { type: string }; + const src = "src" in asset && typeof asset.src === "string" ? asset.src : ""; + const identity: AssetTimingIdentity = { type, src: src || null }; + + // URL-backed assets are identified by their source. Rich captions may instead carry + // their timed words inline, so retain an exact revision to invalidate stale duration + // metadata and rendering when any word changes. + if (type === "rich-caption" && !src && "words" in asset && Array.isArray(asset.words)) { + return { ...identity, revision: JSON.stringify(asset.words) }; + } + + return identity; +} + +export function assetTimingIdentitiesEqual(left: AssetTimingIdentity, right: AssetTimingIdentity): boolean { + return left.type === right.type && left.src === right.src && left.revision === right.revision; +} -const DEFAULT_AUTO_LENGTH_FALLBACK = sec(1); +export function mediaTimingMatchesAsset(state: MediaTimingState, asset: Asset): boolean { + return assetTimingIdentitiesEqual(state.asset, getAssetTimingIdentity(asset)); +} + +/** Apply the one asset-aware policy for `length: "auto"`. */ +export function resolveAutoLength(asset: Asset, intrinsicDuration: Seconds | null = null): Seconds { + if (intrinsicDuration === null || !Number.isFinite(intrinsicDuration) || intrinsicDuration <= 0) return DEFAULT_AUTO_CLIP_LENGTH; -const DEFAULT_AUTO_LENGTH_SEC = sec(3); + const supportsTrim = + asset.type === "video" || asset.type === "audio" || asset.type === "luma" || asset.type === "caption" || asset.type === "text-to-speech"; + const trim = supportsTrim && "trim" in asset && typeof asset.trim === "number" && Number.isFinite(asset.trim) && asset.trim >= 0 ? asset.trim : 0; + return sec(Math.max(0, intrinsicDuration - trim)); +} + +export function resolveEndLength(timelineEnd: Seconds, clipStart: Seconds): Seconds { + if (!Number.isFinite(timelineEnd) || !Number.isFinite(clipStart)) return MINIMUM_END_LENGTH; + return sec(Math.max(MINIMUM_END_LENGTH, timelineEnd - clipStart)); +} export function resolveTimingIntent(intent: TimingIntent, context: Readonly): ResolvedTiming { // Alias references must be resolved before calling this function @@ -27,11 +74,9 @@ export function resolveTimingIntent(intent: TimingIntent, context: Readonly { - return new Promise(resolve => { - const video = document.createElement("video"); - video.preload = "metadata"; - video.crossOrigin = "anonymous"; - video.onloadedmetadata = (): void => resolve(video.duration); - video.onerror = (): void => resolve(null); - video.src = src; - }); -} - -export async function resolveAutoLength(asset: Asset): Promise { - const assetWithSrc = asset as { type: string; src?: string; trim?: number }; - - if (["video", "audio", "luma"].includes(assetWithSrc.type) && assetWithSrc.src) { - const duration = await probeMediaDuration(assetWithSrc.src); - if (duration !== null && !Number.isNaN(duration)) { - const trim = assetWithSrc.trim ?? 0; - return sec(duration - trim); - } - } - - return DEFAULT_AUTO_LENGTH_SEC; -} - export function resolveAutoStart(trackIndex: number, clipIndex: number, tracks: Player[][]): Seconds { if (clipIndex === 0) { return sec(0); @@ -77,10 +97,6 @@ export function resolveAutoStart(trackIndex: number, clipIndex: number, tracks: return previousClip.getEnd(); } -export function resolveEndLength(clipStart: Seconds, timelineEnd: Seconds): Seconds { - return sec(Math.max(0, timelineEnd - clipStart)); -} - export function calculateTimelineEnd(tracks: Player[][]): Seconds { let max = sec(0); diff --git a/src/core/timing/types.ts b/src/core/timing/types.ts index b6755dd3..955277c0 100644 --- a/src/core/timing/types.ts +++ b/src/core/timing/types.ts @@ -100,6 +100,29 @@ export interface ResolvedTiming { length: Seconds; } +/** + * Duration metadata published by every Player using the same lifecycle contract. + * A ready `null` duration means the asset has no intrinsic timeline duration and + * the resolver must apply the canonical auto-length fallback. + */ +export interface AssetTimingIdentity { + readonly type: string; + readonly src: string | null; + /** Exact inline content revision when intrinsic timing is not backed by a URL. */ + readonly revision?: string; +} + +export type MediaTimingState = + | { + readonly status: "pending"; + readonly asset: AssetTimingIdentity; + } + | { + readonly status: "ready"; + readonly asset: AssetTimingIdentity; + readonly duration: Seconds | null; + }; + /** * Context required to resolve timing intent to concrete values. * @@ -110,6 +133,6 @@ export interface ResolutionContext { readonly previousClipEnd: Seconds; /** Total duration of timeline excluding "end" clips (for length: "end") */ readonly timelineEnd: Seconds; - /** Intrinsic duration from asset metadata, null if not yet loaded (for length: "auto") */ - readonly intrinsicDuration: Seconds | null; + /** Final asset-aware auto length, null when the canonical fallback applies. */ + readonly autoLength: Seconds | null; } diff --git a/test-package.js b/test-package.js index ff08fe52..204cbf97 100644 --- a/test-package.js +++ b/test-package.js @@ -291,6 +291,9 @@ const checkPackageExports = () => { } } } + if (!packageJson.files?.includes("THIRD_PARTY_NOTICES")) { + errors.push("THIRD_PARTY_NOTICES is not included in the published package."); + } if (errors.length > 0) { failWithDetails("package.json exports contract", errors); @@ -337,6 +340,23 @@ const checkUmdGlobalMappings = () => { printResult("UMD global mappings", true); }; +const checkGifRuntimePackaging = () => { + const errors = []; + + for (const bundlePath of CONTRACT.umdBundles) { + const content = readFileSync(resolve(__dirname, bundlePath), "utf-8"); + const wrapper = content.slice(0, 2048); + if (wrapper.includes('"pixi.js/gif"')) { + errors.push(`${bundlePath}: pixi.js/gif is external, but the PIXI browser global does not export GifSource.`); + } + } + + if (errors.length > 0) { + failWithDetails("GIF runtime packaging", errors); + } + printResult("GIF runtime packaging", true); +}; + const checkBundleSizes = () => { const errors = []; const details = []; @@ -398,6 +418,7 @@ checkInternalDeclarationSurface(); checkNoChunkArtifactsOrImports(); checkPackageExports(); checkUmdGlobalMappings(); +checkGifRuntimePackaging(); checkBundleSizes(); await runRuntimeExportSmokeTest("Runtime export smoke test", "./dist/shotstack-studio.es.js", CONTRACT.runtimeExports); await runRuntimeExportSmokeTest("Internal runtime export smoke test", "./dist/internal.es.js", CONTRACT.internalRuntimeExports); diff --git a/tests/asset-loader.test.ts b/tests/asset-loader.test.ts index 21b3082b..d252b2f5 100644 --- a/tests/asset-loader.test.ts +++ b/tests/asset-loader.test.ts @@ -59,6 +59,18 @@ describe("AssetLoader", () => { }); }); + it("does not release an asset after its final reference was already removed", () => { + const loader = new AssetLoader(); + const url = "https://example.com/shared.png"; + pixiMock.Assets.cache.has.mockReturnValue(true); + loader.incrementRef(url); + + loader.release(url); + loader.release(url); + + expect(pixiMock.Assets.unload).toHaveBeenCalledTimes(1); + }); + describe("loadVideoUnique", () => { /** * Regression test for video playback glitch with overlapping clips. diff --git a/tests/edit-clip-operations.test.ts b/tests/edit-clip-operations.test.ts index f93ae50e..c499abdd 100644 --- a/tests/edit-clip-operations.test.ts +++ b/tests/edit-clip-operations.test.ts @@ -9,6 +9,7 @@ import { Edit } from "@core/edit-session"; import { PlayerType } from "@canvas/players/player"; import type { EventEmitter } from "@core/events/event-emitter"; import type { Clip, ResolvedClip } from "@schemas"; +import { getAssetTimingIdentity } from "@core/timing/resolver"; import { ms, sec } from "@core/timing/types"; // Stub the DOM-dependent svg-clipboard helpers — sanitisation is unit-tested @@ -203,6 +204,8 @@ const createMockPlayer = (edit: Edit, config: ResolvedClip, type: PlayerType) => }; }, getResolvedTiming: () => ({ ...resolvedTiming }), + getMediaTimingState: () => ({ status: "ready", asset: getAssetTimingIdentity(config.asset), duration: null }), + getLoadedResourceIdentifier: () => ("src" in config.asset ? config.asset.src : null), setResolvedTiming: jest.fn((timing: { start: number; length: number }) => { resolvedTiming = { ...timing }; }), diff --git a/tests/edit-load.test.ts b/tests/edit-load.test.ts index 234f3d15..89b2ee06 100644 --- a/tests/edit-load.test.ts +++ b/tests/edit-load.test.ts @@ -30,6 +30,7 @@ import { ShotstackEdit } from "@core/shotstack-edit"; import { PlayerType } from "@canvas/players/player"; import type { EventEmitter } from "@core/events/event-emitter"; import type { ResolvedClip, EditConfig } from "@schemas"; +import { getAssetTimingIdentity } from "@core/timing/resolver"; // Mock pixi-filters jest.mock("pixi-filters", () => ({ @@ -210,6 +211,8 @@ const createMockPlayer = (edit: Edit, config: ResolvedClip, type: PlayerType) => }; }, getResolvedTiming: () => ({ ...resolvedTiming }), + getMediaTimingState: () => ({ status: "ready", asset: getAssetTimingIdentity(config.asset), duration: null }), + getLoadedResourceIdentifier: () => ("src" in config.asset ? config.asset.src : null), setResolvedTiming: jest.fn((timing: { start: number; length: number }) => { resolvedTiming = { ...timing }; }), diff --git a/tests/edit-merge-fields.test.ts b/tests/edit-merge-fields.test.ts index 526009a7..1c587054 100644 --- a/tests/edit-merge-fields.test.ts +++ b/tests/edit-merge-fields.test.ts @@ -12,6 +12,7 @@ import { ShotstackEdit } from "@core/shotstack-edit"; import { PlayerType } from "@canvas/players/player"; import type { EventEmitter } from "@core/events/event-emitter"; import type { Clip, ResolvedClip } from "@schemas"; +import { getAssetTimingIdentity } from "@core/timing/resolver"; // Mock pixi-filters jest.mock("pixi-filters", () => ({ @@ -182,6 +183,8 @@ const createMockPlayer = (edit: ShotstackEdit, config: ResolvedClip, type: Playe }; }, getResolvedTiming: () => ({ ...resolvedTiming }), + getMediaTimingState: () => ({ status: "ready", asset: getAssetTimingIdentity(config.asset), duration: null }), + getLoadedResourceIdentifier: () => ("src" in config.asset ? config.asset.src : null), setResolvedTiming: jest.fn((timing: { start: number; length: number }) => { resolvedTiming = { ...timing }; }), diff --git a/tests/edit-timing.test.ts b/tests/edit-timing.test.ts index c0a0cf80..510f5de8 100644 --- a/tests/edit-timing.test.ts +++ b/tests/edit-timing.test.ts @@ -9,15 +9,9 @@ import { Edit } from "@core/edit-session"; import { PlayerType } from "@canvas/players/player"; import type { EventEmitter } from "@core/events/event-emitter"; import type { ResolvedClip } from "@schemas"; -import { resolveAutoStart, resolveAutoLength, resolveEndLength, calculateTimelineEnd } from "@core/timing/resolver"; +import { calculateTimelineEnd, getAssetTimingIdentity, resolveAutoLength, resolveAutoStart, resolveEndLength } from "@core/timing/resolver"; import { sec } from "@core/timing/types"; -// Mock probeMediaDuration since document.createElement doesn't work in Node -jest.mock("@core/timing/resolver", () => ({ - ...jest.requireActual("@core/timing/resolver"), - probeMediaDuration: jest.fn().mockResolvedValue(5.0) // 5 seconds -})); - // Mock pixi-filters (must be before pixi.js since it extends pixi classes) jest.mock("pixi-filters", () => ({ AdjustmentFilter: jest.fn().mockImplementation(() => ({})), @@ -189,6 +183,8 @@ const createMockPlayer = (edit: Edit, config: ResolvedClip, type: PlayerType) => }; }, getResolvedTiming: () => ({ ...resolvedTiming }), + getMediaTimingState: () => ({ status: "ready", asset: getAssetTimingIdentity(config.asset), duration: null }), + getLoadedResourceIdentifier: () => ("src" in config.asset ? config.asset.src : null), setResolvedTiming: jest.fn((timing: { start: number; length: number }) => { resolvedTiming = { ...timing }; // Sync clipConfiguration to match real Player behavior (Option B) @@ -380,25 +376,25 @@ describe("Timing Resolver Functions", () => { describe("resolveEndLength()", () => { it("returns timeline end minus clip start", () => { - const result = resolveEndLength(sec(2), sec(10)); + const result = resolveEndLength(sec(10), sec(2)); expect(result).toBe(8); }); - it("never returns negative value", () => { - const result = resolveEndLength(sec(15), sec(10)); + it("never returns a negative duration", () => { + const result = resolveEndLength(sec(10), sec(15)); expect(result).toBe(0); }); - it("returns 0 when clip starts at timeline end", () => { + it("returns zero when clip starts at timeline end", () => { const result = resolveEndLength(sec(10), sec(10)); expect(result).toBe(0); }); it("handles clip starting at 0", () => { - const result = resolveEndLength(sec(0), sec(5)); + const result = resolveEndLength(sec(5), sec(0)); expect(result).toBe(5); }); diff --git a/tests/gif-asset-loader.test.ts b/tests/gif-asset-loader.test.ts new file mode 100644 index 00000000..a715ec38 --- /dev/null +++ b/tests/gif-asset-loader.test.ts @@ -0,0 +1,143 @@ +/** @jest-environment jsdom */ + +import { AssetLoader } from "@loaders/asset-loader"; +import { GifImageSource } from "@loaders/gif-image-source"; + +jest.mock("pixi.js", () => ({ + Assets: { + setPreferences: jest.fn(), + load: jest.fn(), + unload: jest.fn(), + cache: { has: jest.fn().mockReturnValue(false) } + } +})); + +jest.mock("@loaders/gif-image-source", () => ({ + GifImageSource: { fetch: jest.fn() } +})); + +const fetchGif = GifImageSource.fetch as jest.MockedFunction; + +describe("AssetLoader GIF support", () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it("detects uppercase GIF URLs without a network probe", async () => { + const fetchMock = jest.fn(); + global.fetch = fetchMock; + const loader = new AssetLoader(); + + await expect(loader.isGif("https://example.com/ANIMATION.GIF?token=abc")).resolves.toBe(true); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it("detects extensionless GIF URLs by signature", async () => { + const cancel = jest.fn().mockResolvedValue(undefined); + const releaseLock = jest.fn(); + const read = jest.fn().mockResolvedValueOnce({ + done: false, + value: Uint8Array.from("GIF89a", character => character.charCodeAt(0)) + }); + const fetchMock = jest.fn().mockResolvedValue({ ok: true, body: { getReader: () => ({ read, cancel, releaseLock }) } }); + global.fetch = fetchMock; + const loader = new AssetLoader(); + + await expect(loader.isGif("https://example.com/assets/123")).resolves.toBe(true); + expect(fetchMock).toHaveBeenCalledWith( + "https://example.com/assets/123?x-cors=1", + expect.objectContaining({ headers: { Range: "bytes=0-5" }, signal: expect.any(AbortSignal) }) + ); + expect(cancel).toHaveBeenCalledTimes(1); + expect(releaseLock).toHaveBeenCalledTimes(1); + }); + + it("classifies a complete non-GIF signature as a static image", async () => { + const cancel = jest.fn().mockResolvedValue(undefined); + const releaseLock = jest.fn(); + const read = jest.fn().mockResolvedValueOnce({ done: false, value: Uint8Array.of(137, 80, 78, 71, 13, 10) }); + global.fetch = jest.fn().mockResolvedValue({ ok: true, body: { getReader: () => ({ read, cancel, releaseLock }) } }); + const loader = new AssetLoader(); + + await expect(loader.isGif("https://example.com/image.png")).resolves.toBe(false); + expect(cancel).toHaveBeenCalledTimes(1); + expect(releaseLock).toHaveBeenCalledTimes(1); + }); + + it("rejects an inconclusive signature probe instead of bypassing GIF safeguards", async () => { + const cancel = jest.fn().mockResolvedValue(undefined); + const releaseLock = jest.fn(); + const read = jest.fn().mockResolvedValueOnce({ done: true }); + global.fetch = jest.fn().mockResolvedValue({ ok: true, body: { getReader: () => ({ read, cancel, releaseLock }) } }); + const loader = new AssetLoader(); + + await expect(loader.isGif("https://example.com/assets/incomplete")).rejects.toThrow(/ended before its signature/); + expect(cancel).toHaveBeenCalledTimes(1); + expect(releaseLock).toHaveBeenCalledTimes(1); + }); + + it("trusts GIF magic over a non-GIF file extension", async () => { + const cancel = jest.fn().mockResolvedValue(undefined); + const releaseLock = jest.fn(); + const read = jest.fn().mockResolvedValueOnce({ + done: false, + value: Uint8Array.from("GIF89a", character => character.charCodeAt(0)) + }); + global.fetch = jest.fn().mockResolvedValueOnce({ ok: true, body: { getReader: () => ({ read, cancel, releaseLock }) } }); + const loader = new AssetLoader(); + + await expect(loader.isGif("https://example.com/mislabeled.png")).resolves.toBe(true); + expect(global.fetch).toHaveBeenCalledTimes(1); + expect(global.fetch).toHaveBeenCalledWith( + "https://example.com/mislabeled.png?x-cors=1", + expect.objectContaining({ headers: { Range: "bytes=0-5" }, signal: expect.any(AbortSignal) }) + ); + expect(cancel).toHaveBeenCalledTimes(1); + expect(releaseLock).toHaveBeenCalledTimes(1); + }); + + it("shares decoded sources and destroys them only after the last release", async () => { + const source = { + destroy: jest.fn(), + getFirstFrameDataUrl: jest.fn().mockReturnValue("data:image/png;base64,frame"), + width: 320, + height: 180 + } as unknown as GifImageSource; + fetchGif.mockResolvedValue(source); + const loader = new AssetLoader(); + const identifier = "https://example.com/shared.gif"; + + const [first, second] = await Promise.all([loader.loadGif(identifier), loader.loadGif(identifier)]); + expect(first).toBe(source); + expect(second).toBe(source); + expect(fetchGif).toHaveBeenCalledTimes(1); + + loader.release(identifier); + expect(source.destroy).not.toHaveBeenCalled(); + + loader.release(identifier); + await Promise.resolve(); + expect(source.destroy).toHaveBeenCalledTimes(1); + }); + + it("can generate a frozen thumbnail before a Player acquires the source", async () => { + const source = { + destroy: jest.fn(), + getFirstFrameDataUrl: jest.fn().mockReturnValue("data:image/png;base64,frame"), + width: 320, + height: 180 + } as unknown as GifImageSource; + fetchGif.mockResolvedValue(source); + const loader = new AssetLoader(); + + await expect(loader.getGifThumbnail("https://example.com/thumbnail.gif")).resolves.toEqual({ + isGif: true, + dataUrl: "data:image/png;base64,frame", + width: 320, + height: 180 + }); + expect(fetchGif).toHaveBeenCalledTimes(1); + await Promise.resolve(); + expect(source.destroy).toHaveBeenCalledTimes(1); + }); +}); diff --git a/tests/gif-image-source.test.ts b/tests/gif-image-source.test.ts new file mode 100644 index 00000000..4c20c27f --- /dev/null +++ b/tests/gif-image-source.test.ts @@ -0,0 +1,253 @@ +/** @jest-environment jsdom */ + +import { + GIF_DECODE_LIMITS, + GifImageSource, + getAnimatedGifDurationMs, + inspectGif, + readGifResponse, + selectGifFrame, + validateGifContentLength, + validateGifCycleDuration +} from "@loaders/gif-image-source"; +import { GifSource as PixiGifSource } from "pixi.js/gif"; +import * as pixi from "pixi.js"; + +jest.mock("pixi.js", () => { + const CanvasSource = jest.fn(function MockCanvasSource(this: { resource: HTMLCanvasElement }, { resource }: { resource: HTMLCanvasElement }): void { + this.resource = resource; + }); + const Texture = jest.fn(function MockTexture(this: { source: unknown; destroy: jest.Mock }, { source }: { source: unknown }): void { + this.source = source; + this.destroy = jest.fn(); + }); + + return { CanvasSource, Texture }; +}); + +jest.mock("pixi.js/gif", () => ({ + GifSource: { from: jest.fn() } +})); + +const pixiGifFrom = PixiGifSource.from as jest.MockedFunction; +const TWO_FRAME_GIF = "R0lGODlhAgACAPAAAP8AAAAAACH/C05FVFNDQVBFMi4wAwEAAAAh+QQAAAAAACwAAAAAAgACAAACAoRRACH5BAAKAAAALAAAAAACAAIAgAAA/wAAAAIChFEAOw=="; +const MISSING_SECOND_GCE_GIF = "R0lGODlhAgACAPAAAP8AAAAAACH/C05FVFNDQVBFMi4wAwEAAAAh+QQAMgAAACwAAAAAAgACAAACAoRRACwAAAAAAgACAIAAAP8AAAACAoRRADs="; +const GCE_BEFORE_COMMENTS_GIF = + "R0lGODlhAwABAPAAAP8AAAAA/yH/C05FVFNDQVBFMi4wAwEAAAAh+QQIMgD/ACH+AUEAIf4BQgAsAAAAAAMAAQAAAgKECwAh+QQAMgD/ACwBAAAAAQABAAACAkwBADs="; + +function decodeBase64(base64: string): ArrayBuffer { + return Uint8Array.from(atob(base64), character => character.charCodeAt(0)).buffer; +} + +function appendUint16(bytes: number[], value: number): void { + bytes.push(value % 256, Math.floor(value / 256) % 256); +} + +function createGif( + width: number, + height: number, + frameCount: number, + frameDescriptor: { left?: number; top?: number; width?: number; height?: number } = {} +): ArrayBuffer { + const bytes = [71, 73, 70, 56, 57, 97]; + appendUint16(bytes, width); + appendUint16(bytes, height); + bytes.push(0, 0, 0); + + for (let frame = 0; frame < frameCount; frame += 1) { + bytes.push(0x2c); + appendUint16(bytes, frameDescriptor.left ?? 0); + appendUint16(bytes, frameDescriptor.top ?? 0); + appendUint16(bytes, frameDescriptor.width ?? width); + appendUint16(bytes, frameDescriptor.height ?? height); + bytes.push(0, 2, 2, 0x44, 0x01, 0); + } + bytes.push(0x3b); + return Uint8Array.from(bytes).buffer; +} + +function createDecodedSource(width: number, height: number, frameDurations: number[]): PixiGifSource & { destroy: jest.Mock } { + let time = 0; + const frames = frameDurations.map(duration => { + const start = time; + time += duration; + return { + start, + end: time, + texture: { source: { resource: document.createElement("canvas") } } + }; + }); + return { + width, + height, + duration: time, + frames, + textures: frames.map(frame => frame.texture), + totalFrames: frames.length, + destroy: jest.fn() + } as unknown as PixiGifSource & { destroy: jest.Mock }; +} + +describe("GIF decode safeguards", () => { + it("calculates Pixi's retained frames and eager patch allocation before decoding", () => { + expect(inspectGif(createGif(320, 180, 20))).toEqual({ + width: 320, + height: 180, + frameCount: 20, + decodedBytes: 320 * 180 * 45 * 4 + }); + }); + + it("rejects decoded GIFs over the allocation budget", () => { + expect(() => inspectGif(createGif(4096, 4096, 5))).toThrow(/decoded frames exceed/); + }); + + it("rejects excessive dimensions and frame counts", () => { + expect(() => inspectGif(createGif(GIF_DECODE_LIMITS.maxDimension + 1, 1, 1))).toThrow(/dimensions exceed/); + expect(() => inspectGif(createGif(1, 1, GIF_DECODE_LIMITS.maxFrames + 1))).toThrow(/frame limit/); + }); + + it("rejects a frame descriptor outside the logical screen before decoding", () => { + expect(() => inspectGif(createGif(1, 1, 1, { width: 65535 }))).toThrow(/frame descriptor/); + }); + + it("rejects malformed and truncated container data before Pixi sees it", () => { + expect(() => inspectGif(Uint8Array.from([71, 73, 70]).buffer)).toThrow(/truncated/); + expect(() => inspectGif(Uint8Array.from([78, 79, 84, 71, 73, 70, 56, 57, 97]).buffer)).toThrow(/valid GIF signature/); + }); + + it("rejects oversized Content-Length values before downloading", () => { + expect(() => validateGifContentLength(String(GIF_DECODE_LIMITS.maxCompressedBytes + 1))).toThrow(/compressed-size limit/); + expect(() => validateGifContentLength(null)).not.toThrow(); + }); + + it("cancels an oversized response before reading its body", async () => { + const cancel = jest.fn().mockResolvedValue(undefined); + const response = { + headers: new Headers({ "content-length": String(GIF_DECODE_LIMITS.maxCompressedBytes + 1) }), + body: { cancel } + } as unknown as Response; + + await expect(readGifResponse(response)).rejects.toThrow(/compressed-size limit/); + expect(cancel).toHaveBeenCalledTimes(1); + }); + + it("cancels a chunked response as soon as it crosses the compressed limit", async () => { + const cancel = jest.fn().mockResolvedValue(undefined); + const releaseLock = jest.fn(); + const read = jest + .fn() + .mockResolvedValueOnce({ done: false, value: { byteLength: GIF_DECODE_LIMITS.maxCompressedBytes + 1 } }) + .mockResolvedValueOnce({ done: true }); + const response = { + headers: new Headers(), + body: { getReader: () => ({ read, cancel, releaseLock }) } + } as unknown as Response; + + await expect(readGifResponse(response)).rejects.toThrow(/compressed-size limit/); + expect(cancel).toHaveBeenCalledTimes(1); + expect(releaseLock).toHaveBeenCalledTimes(1); + }); + + it("rejects cycles longer than the backend's five-minute limit", () => { + expect(() => validateGifCycleDuration(GIF_DECODE_LIMITS.maxCycleDurationMs + 1)).toThrow(/cycle exceeds/); + expect(() => validateGifCycleDuration(GIF_DECODE_LIMITS.maxCycleDurationMs)).not.toThrow(); + }); +}); + +describe("GIF playhead frame selection", () => { + const frames = [ + { start: 0, end: 40 }, + { start: 40, end: 140 }, + { start: 140, end: 200 } + ]; + + it("honours variable frame delays", () => { + expect(selectGifFrame(frames, 39, 200)).toBe(0); + expect(selectGifFrame(frames, 40, 200)).toBe(1); + expect(selectGifFrame(frames, 139, 200)).toBe(1); + expect(selectGifFrame(frames, 140, 200)).toBe(2); + }); + + it("repeats deterministically for clips longer than one cycle", () => { + expect(selectGifFrame(frames, 240, 200)).toBe(1); + expect(selectGifFrame(frames, 400, 200)).toBe(0); + }); +}); + +describe("GIF intrinsic duration", () => { + it("uses one full cycle only for animated GIFs", () => { + expect(getAnimatedGifDurationMs({ totalFrames: 3, duration: 2400 })).toBe(2400); + expect(getAnimatedGifDurationMs({ totalFrames: 1, duration: 100 })).toBeNull(); + }); +}); + +describe("Pixi GIF source adapter", () => { + beforeEach(() => { + pixiGifFrom.mockReset(); + }); + + it("delegates decoding of a valid two-frame GIF to Pixi", () => { + const decoded = createDecodedSource(2, 2, [100, 100]); + const decodedTextures = decoded.frames.map(frame => frame.texture); + const decodedCanvases = decoded.frames.map(frame => frame.texture.source.resource); + pixiGifFrom.mockReturnValue(decoded); + const source = GifImageSource.from(decodeBase64(TWO_FRAME_GIF)); + + expect(source.totalFrames).toBe(2); + expect(source.width).toBe(2); + expect(source.height).toBe(2); + expect(source.duration).toBe(200); + expect(source.frameIndexAt(source.frames[0].end)).toBe(1); + expect(source.frames.every(frame => frame.texture instanceof pixi.Texture)).toBe(true); + expect(source.frames.every(frame => frame.texture.source instanceof pixi.CanvasSource)).toBe(true); + expect(source.frames.every((frame, index) => frame.texture !== decodedTextures[index])).toBe(true); + expect(source.frames.map(frame => frame.texture.source.resource)).toEqual(decodedCanvases); + expect(decoded.destroy).toHaveBeenCalledTimes(1); + expect(pixiGifFrom).toHaveBeenCalledWith(expect.any(ArrayBuffer), { fps: 10 }); + }); + + it("adds a default control extension instead of inheriting an earlier frame delay", () => { + const buffer = decodeBase64(MISSING_SECOND_GCE_GIF); + pixiGifFrom.mockReturnValue(createDecodedSource(2, 2, [500, 100])); + const source = GifImageSource.from(buffer); + const normalizedBuffer = pixiGifFrom.mock.calls[0][0]; + + expect(normalizedBuffer.byteLength).toBe(buffer.byteLength + 8); + expect(source.duration).toBe(600); + expect(source.frames.map(frame => frame.end - frame.start)).toEqual([500, 100]); + }); + + it("preserves a control extension across intervening comment blocks", () => { + const buffer = decodeBase64(GCE_BEFORE_COMMENTS_GIF); + pixiGifFrom.mockReturnValue(createDecodedSource(3, 1, [500, 500])); + const source = GifImageSource.from(buffer); + const normalizedBuffer = pixiGifFrom.mock.calls[0][0]; + + expect(normalizedBuffer.byteLength).toBe(buffer.byteLength); + expect(source.duration).toBe(1000); + }); + + it("destroys a Pixi source whose decoded metadata differs from preflight", () => { + const decoded = createDecodedSource(3, 2, [100, 100]); + pixiGifFrom.mockReturnValue(decoded); + + expect(() => GifImageSource.from(decodeBase64(TWO_FRAME_GIF))).toThrow(/metadata changed/); + expect(decoded.destroy).toHaveBeenCalledTimes(1); + }); + + it("transfers frame ownership and destroys each host texture exactly once", () => { + const decoded = createDecodedSource(2, 2, [100, 100]); + pixiGifFrom.mockReturnValue(decoded); + const source = GifImageSource.from(decodeBase64(TWO_FRAME_GIF)); + const textureDestroyers = source.frames.map(frame => jest.spyOn(frame.texture, "destroy")); + + source.destroy(); + source.destroy(); + expect(decoded.destroy).toHaveBeenCalledTimes(1); + textureDestroyers.forEach(destroy => { + expect(destroy).toHaveBeenCalledTimes(1); + expect(destroy).toHaveBeenCalledWith(true); + }); + }); +}); diff --git a/tests/gif-url.test.ts b/tests/gif-url.test.ts new file mode 100644 index 00000000..f21268e0 --- /dev/null +++ b/tests/gif-url.test.ts @@ -0,0 +1,29 @@ +import { appendCorsQuery, getUrlExtension, isGifUrl } from "@loaders/gif-url"; + +describe("GIF URL classification", () => { + it.each([ + "https://cdn.example.com/animation.gif", + "https://cdn.example.com/ANIMATION.GIF?token=abc", + "data:image/gif;base64,R0lGODlhAQABAIAAAAUEBA==" + ])("recognises %s", src => { + expect(isGifUrl(src)).toBe(true); + }); + + it.each(["https://cdn.example.com/image.png", "data:image/png;base64,abc", "https://cdn.example.com/assets/image"])( + "does not infer GIF from %s", + src => { + expect(isGifUrl(src)).toBe(false); + } + ); + + it("extracts a case-normalised extension without query parameters", () => { + expect(getUrlExtension("https://example.com/path/FILE.GIF?v=2")).toBe(".gif"); + expect(getUrlExtension("https://example.com/path/asset")).toBeNull(); + }); + + it("adds the CORS query only to HTTP URLs", () => { + expect(appendCorsQuery("https://example.com/a.gif")).toBe("https://example.com/a.gif?x-cors=1"); + expect(appendCorsQuery("https://example.com/a.gif?v=1")).toBe("https://example.com/a.gif?v=1&x-cors=1"); + expect(appendCorsQuery("data:image/gif;base64,abc")).toBe("data:image/gif;base64,abc"); + }); +}); diff --git a/tests/media-auto-timing.test.ts b/tests/media-auto-timing.test.ts new file mode 100644 index 00000000..131f372a --- /dev/null +++ b/tests/media-auto-timing.test.ts @@ -0,0 +1,81 @@ +import type { Player } from "@canvas/players/player"; +import type { Edit } from "@core/edit-session"; +import { TimingManager } from "@core/timing-manager"; +import { resolveAutoLength, resolveEndLength, resolveTimingIntent } from "@core/timing/resolver"; +import { sec } from "@core/timing/types"; +import type { Asset, ResolvedEdit } from "@schemas"; + +describe("uniform auto-length timing", () => { + it("uses the same fallback and minimum end policy in the context resolver", () => { + expect(resolveTimingIntent({ start: sec(0), length: "auto" }, { previousClipEnd: sec(0), timelineEnd: sec(0), autoLength: null })).toEqual({ + start: 0, + length: 3 + }); + expect(resolveTimingIntent({ start: sec(5), length: "end" }, { previousClipEnd: sec(0), timelineEnd: sec(5), autoLength: null })).toEqual({ + start: 5, + length: resolveEndLength(sec(5), sec(5)) + }); + expect(resolveEndLength(sec(5), sec(5))).toBe(0); + }); + + it.each([ + ["animated GIF image", { type: "image", src: "animation.gif", trim: 1 }, 2.4, 2.4], + ["static image", { type: "image", src: "image.png" }, null, 3], + ["video", { type: "video", src: "video.mp4", trim: 2 }, 8, 6], + ["audio", { type: "audio", src: "audio.mp3", trim: 0.5 }, 4.5, 4], + ["video luma", { type: "luma", src: "luma.mp4" }, 5, 5], + ["text", { type: "text", text: "Hello" }, null, 3] + ] as const)("applies the same resolver policy to %s", (_name, asset, intrinsicDuration, expected) => { + expect(resolveAutoLength(asset as Asset, intrinsicDuration === null ? null : sec(intrinsicDuration))).toBe(expected); + }); + + it("uses the full fallback and ignores invalid trim values", () => { + const asset = { type: "video", src: "video.mp4", trim: 2 } as Asset; + expect(resolveAutoLength(asset, null)).toBe(3); + expect(resolveAutoLength({ ...asset, trim: Number.NaN } as Asset, sec(8))).toBe(8); + expect(resolveAutoLength({ ...asset, trim: -2 } as Asset, sec(8))).toBe(8); + }); + + it("returns zero for non-finite end inputs", () => { + expect(resolveEndLength(sec(Number.NaN), sec(1))).toBe(0); + expect(resolveEndLength(sec(10), sec(Number.POSITIVE_INFINITY))).toBe(0); + }); + + it("applies one pure resolver result to auto, dependent start, and end clips", async () => { + const timings = [ + { start: sec(0), length: sec(2.4) }, + { start: sec(2.4), length: sec(1) }, + { start: sec(0), length: sec(3.4) } + ]; + const reconfigure = jest.fn(); + const players = timings.map((initial, index) => { + let timing = index === 0 ? { start: sec(0), length: sec(3) } : initial; + return { + getStart: () => timing.start, + getLength: () => timing.length, + getEnd: () => sec(timing.start + timing.length), + getTimingIntent: () => ({ start: timing.start, length: timing.length }), + setResolvedTiming: (next: typeof timing) => { + timing = next; + }, + reconfigureAfterRestore: reconfigure + } as unknown as Player; + }); + const resolved = { + timeline: { + tracks: [{ clips: [timings[0], timings[1]] }, { clips: [timings[2]] }] + } + } as unknown as ResolvedEdit; + const edit = { + getTracks: () => [[players[0], players[1]], [players[2]]], + getResolvedEdit: () => resolved + } as unknown as Edit; + + await new TimingManager(edit).resolveAllTiming(); + + expect(players[0].getLength()).toBe(2.4); + expect(players[1].getStart()).toBe(2.4); + expect(players[2].getLength()).toBe(3.4); + expect(reconfigure).toHaveBeenCalledTimes(1); + }); +}); diff --git a/tests/media-player-fallback.test.ts b/tests/media-player-fallback.test.ts index a0324633..10447ba3 100644 --- a/tests/media-player-fallback.test.ts +++ b/tests/media-player-fallback.test.ts @@ -149,8 +149,14 @@ jest.mock("pixi.js", () => { // eslint-disable-next-line import/first import { ImagePlayer } from "@canvas/players/image-player"; // eslint-disable-next-line import/first +import { LumaPlayer } from "@canvas/players/luma-player"; +// eslint-disable-next-line import/first +import { Player, PlayerType } from "@canvas/players/player"; +// eslint-disable-next-line import/first import { VideoPlayer } from "@canvas/players/video-player"; // eslint-disable-next-line import/first +import { sec } from "@core/timing/types"; +// eslint-disable-next-line import/first import type { ResolvedClip } from "@schemas"; // eslint-disable-next-line import/first import * as pixi from "pixi.js"; @@ -161,9 +167,12 @@ function createEdit() { playbackTime: 0, isPlaying: false, assetLoader: { + isGif: jest.fn().mockResolvedValue(false), + loadGif: jest.fn(), load: jest.fn(), loadVideoUnique: jest.fn(), - rejectAsset: jest.fn() + rejectAsset: jest.fn(), + release: jest.fn() } }; } @@ -192,9 +201,21 @@ function createVideoClip(): ResolvedClip { } as ResolvedClip; } -function createVideoTexture(width: number, height: number) { +function createLumaClip(): ResolvedClip { + return { + asset: { + type: "luma", + src: "https://example.com/luma-a.mp4" + }, + start: 0, + length: 5 + } as ResolvedClip; +} + +function createVideoTexture(width: number, height: number, duration = 8) { const resource = { volume: 1, + duration, currentTime: 0, pause: jest.fn(), play: jest.fn().mockResolvedValue(undefined), @@ -209,6 +230,60 @@ function createVideoTexture(width: number, height: number) { } as ConstructorParameters[0]); } +function createHtmlVideoTexture(width: number, height: number, duration = 8, initialReadyState = 0) { + const video = document.createElement("video"); + let readyState = initialReadyState; + + Object.defineProperties(video, { + readyState: { configurable: true, get: () => readyState }, + duration: { configurable: true, get: () => duration }, + pause: { configurable: true, value: jest.fn() }, + play: { configurable: true, value: jest.fn().mockResolvedValue(undefined) }, + load: { configurable: true, value: jest.fn() } + }); + + return { + video, + texture: new pixi.Texture({ + source: new pixi.VideoSource({ resource: video }), + width, + height + } as ConstructorParameters[0]), + setReadyState: (value: number) => { + readyState = value; + } + }; +} + +function createDeferred() { + let resolve!: (value: T | PromiseLike) => void; + let reject!: (reason?: unknown) => void; + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise; + reject = rejectPromise; + }); + + return { promise, resolve, reject }; +} + +class TimingStatePlayer extends Player { + constructor(edit: ReturnType, clip: ResolvedClip) { + super(edit as never, clip, PlayerType.Image); + } + + public beginTimingLoad(): number { + return this.beginMediaTimingLoad(); + } + + public completeTimingLoad(revision: number, duration: number): void { + this.completeMediaTimingLoad(revision, sec(duration)); + } + + public override getSize(): { width: number; height: number } { + return this.edit.size; + } +} + describe("media player fallbacks", () => { let warnSpy: jest.SpyInstance; @@ -227,7 +302,12 @@ describe("media player fallbacks", () => { const player = new ImagePlayer(edit as never, createImageClip()); - await player.load(); + const loading = player.load(); + expect(player.getMediaTimingState()).toEqual({ + status: "pending", + asset: { type: "image", src: "https://example.com/missing.png" } + }); + await loading; expect(mockCreatePlaceholderGraphic).toHaveBeenCalledWith(400, 300); expect(edit.assetLoader.rejectAsset).not.toHaveBeenCalled(); @@ -236,6 +316,38 @@ describe("media player fallbacks", () => { expect(Number.isFinite(player.getScale())).toBe(true); expect(player.getContentContainer().children).toHaveLength(1); expect((player.getContentContainer().children[0] as { __placeholder?: boolean }).__placeholder).toBe(true); + expect(player.getMediaTimingState()).toEqual({ + status: "ready", + asset: { type: "image", src: "https://example.com/missing.png" }, + duration: null + }); + }); + + it("routes a mislabeled GIF through the bounded decoder and publishes its cycle duration", async () => { + const edit = createEdit(); + const frameTexture = new pixi.Texture({ width: 2, height: 2 } as ConstructorParameters[0]); + edit.assetLoader.isGif.mockResolvedValueOnce(true); + edit.assetLoader.loadGif.mockResolvedValueOnce({ + totalFrames: 2, + duration: 2400, + frames: [ + { start: 0, end: 1200, texture: frameTexture }, + { start: 1200, end: 2400, texture: frameTexture } + ], + frameIndexAt: jest.fn().mockReturnValue(0) + }); + + const player = new ImagePlayer(edit as never, createImageClip()); + await player.load(); + + expect(edit.assetLoader.isGif).toHaveBeenCalledWith("https://example.com/missing.png", "https://example.com/missing.png?x-cors=1"); + expect(edit.assetLoader.load).not.toHaveBeenCalled(); + expect(edit.assetLoader.loadGif).toHaveBeenCalledTimes(1); + expect(player.getMediaTimingState()).toEqual({ + status: "ready", + asset: { type: "image", src: "https://example.com/missing.png" }, + duration: 2.4 + }); }); it("replaces a failed video placeholder after a successful reload", async () => { @@ -250,6 +362,11 @@ describe("media player fallbacks", () => { expect(player.getContentContainer().children).toHaveLength(1); expect((player.getContentContainer().children[0] as { __placeholder?: boolean }).__placeholder).toBe(true); expect(player.getSize()).toEqual({ width: 1080, height: 1920 }); + expect(player.getMediaTimingState()).toEqual({ + status: "ready", + asset: { type: "video", src: "https://example.com/video.mp4" }, + duration: null + }); await player.reloadAsset(); @@ -257,5 +374,207 @@ describe("media player fallbacks", () => { expect((player.getContentContainer().children[0] as { __placeholder?: boolean }).__placeholder).not.toBe(true); expect(player.getSize()).toEqual({ width: 1280, height: 720 }); expect(Number.isFinite(player.getScale())).toBe(true); + expect(player.getMediaTimingState()).toEqual({ + status: "ready", + asset: { type: "video", src: "https://example.com/video.mp4" }, + duration: 8 + }); + }); + + it("ignores a stale video reload failure after the current source succeeds", async () => { + const edit = createEdit(); + const staleLoad = createDeferred(); + const currentTexture = createVideoTexture(1920, 1080, 12); + edit.assetLoader.loadVideoUnique.mockReturnValueOnce(staleLoad.promise).mockResolvedValueOnce(currentTexture); + const player = new VideoPlayer(edit as never, createVideoClip()); + + const staleReload = player.reloadAsset(); + player.clipConfiguration.asset = { type: "video", src: "https://example.com/current.mp4" }; + await player.reloadAsset(); + + staleLoad.reject(new Error("stale request failed")); + await staleReload; + + expect(mockCreatePlaceholderGraphic).not.toHaveBeenCalled(); + expect(player.getContentContainer().children).toHaveLength(1); + expect((player.getContentContainer().children[0] as { __placeholder?: boolean }).__placeholder).not.toBe(true); + expect(player.getSize()).toEqual({ width: 1920, height: 1080 }); + expect(player.getMediaTimingState()).toEqual({ + status: "ready", + asset: { type: "video", src: "https://example.com/current.mp4" }, + duration: 12 + }); + }); + + it("keeps video updates suppressed when a stale reload fails before the current reload settles", async () => { + const edit = createEdit(); + const staleLoad = createDeferred(); + const currentLoad = createDeferred>(); + edit.assetLoader.loadVideoUnique.mockReturnValueOnce(staleLoad.promise).mockReturnValueOnce(currentLoad.promise); + const player = new VideoPlayer(edit as never, createVideoClip()); + + const staleReload = player.reloadAsset(); + player.clipConfiguration.asset = { type: "video", src: "https://example.com/current.mp4" }; + const currentReload = player.reloadAsset(); + + staleLoad.reject(new Error("stale request failed")); + await staleReload; + + // @ts-expect-error - private state is asserted to guard the async reload contract + expect(player.skipVideoUpdate).toBe(true); + expect(mockCreatePlaceholderGraphic).not.toHaveBeenCalled(); + + currentLoad.resolve(createVideoTexture(1280, 720, 6)); + await currentReload; + + // @ts-expect-error - private state is asserted to guard the async reload contract + expect(player.skipVideoUpdate).toBe(false); + }); + + it("waits for loadeddata from an HTML video and removes every readiness listener", async () => { + const edit = createEdit(); + const pending = createHtmlVideoTexture(1920, 1080, 11); + const removeListenerSpy = jest.spyOn(pending.video, "removeEventListener"); + edit.assetLoader.loadVideoUnique.mockResolvedValueOnce(pending.texture); + const player = new VideoPlayer(edit as never, createVideoClip()); + + const reload = player.reloadAsset(); + await Promise.resolve(); + expect(player.getContentContainer().children).toHaveLength(0); + + pending.setReadyState(HTMLMediaElement.HAVE_CURRENT_DATA); + pending.video.dispatchEvent(new Event("loadeddata")); + await reload; + + expect(removeListenerSpy.mock.calls.map(([event]) => event)).toEqual(expect.arrayContaining(["loadeddata", "error", "abort"])); + expect(player.getContentContainer().children).toHaveLength(1); + expect(player.getMediaTimingState()).toMatchObject({ status: "ready", duration: 11 }); + }); + + it.each(["error", "abort"])("handles an HTML video %s event and removes every readiness listener", async eventName => { + const edit = createEdit(); + const pending = createHtmlVideoTexture(1920, 1080); + const removeListenerSpy = jest.spyOn(pending.video, "removeEventListener"); + edit.assetLoader.loadVideoUnique.mockResolvedValueOnce(pending.texture); + const player = new VideoPlayer(edit as never, createVideoClip()); + + const reload = player.reloadAsset(); + await Promise.resolve(); + pending.video.dispatchEvent(new Event(eventName)); + await reload; + + expect(removeListenerSpy.mock.calls.map(([event]) => event)).toEqual(expect.arrayContaining(["loadeddata", "error", "abort"])); + expect(pending.texture.destroy).toHaveBeenCalledWith(true); + expect(player.getContentContainer().children).toHaveLength(1); + expect((player.getContentContainer().children[0] as { __placeholder?: boolean }).__placeholder).toBe(true); + expect(player.getMediaTimingState()).toMatchObject({ status: "ready", duration: null }); + }); + + it("cancels an HTML video readiness wait when a newer reload starts", async () => { + const edit = createEdit(); + const stale = createHtmlVideoTexture(640, 360); + const current = createHtmlVideoTexture(1920, 1080, 12, HTMLMediaElement.HAVE_CURRENT_DATA); + const removeListenerSpy = jest.spyOn(stale.video, "removeEventListener"); + edit.assetLoader.loadVideoUnique.mockResolvedValueOnce(stale.texture).mockResolvedValueOnce(current.texture); + const player = new VideoPlayer(edit as never, createVideoClip()); + + const staleReload = player.reloadAsset(); + await Promise.resolve(); + player.clipConfiguration.asset = { type: "video", src: "https://example.com/current.mp4" }; + const currentReload = player.reloadAsset(); + await Promise.all([staleReload, currentReload]); + + expect(removeListenerSpy.mock.calls.map(([event]) => event)).toEqual(expect.arrayContaining(["loadeddata", "error", "abort"])); + expect(stale.texture.destroy).toHaveBeenCalledWith(true); + expect(current.texture.destroy).not.toHaveBeenCalled(); + expect(player.getContentContainer().children).toHaveLength(1); + expect(player.getSize()).toEqual({ width: 1920, height: 1080 }); + }); + + it("rejects an HTML video generation superseded after loadeddata resolves", async () => { + const edit = createEdit(); + const stale = createHtmlVideoTexture(640, 360); + const current = createHtmlVideoTexture(1280, 720, 6, HTMLMediaElement.HAVE_CURRENT_DATA); + edit.assetLoader.loadVideoUnique.mockResolvedValueOnce(stale.texture).mockResolvedValueOnce(current.texture); + const player = new VideoPlayer(edit as never, createVideoClip()); + + const staleReload = player.reloadAsset(); + await Promise.resolve(); + stale.setReadyState(HTMLMediaElement.HAVE_CURRENT_DATA); + stale.video.dispatchEvent(new Event("loadeddata")); + + player.clipConfiguration.asset = { type: "video", src: "https://example.com/current.mp4" }; + const currentReload = player.reloadAsset(); + await Promise.all([staleReload, currentReload]); + + expect(stale.texture.destroy).toHaveBeenCalledWith(true); + expect(current.texture.destroy).not.toHaveBeenCalled(); + expect(player.getContentContainer().children).toHaveLength(1); + expect(player.getSize()).toEqual({ width: 1280, height: 720 }); + expect(player.getMediaTimingState()).toMatchObject({ status: "ready", duration: 6 }); + }); + + it("reloads luma metadata and releases the resource actually replaced", async () => { + const edit = createEdit(); + const firstTexture = createVideoTexture(1280, 720, 4); + const secondTexture = createVideoTexture(1280, 720, 6); + edit.assetLoader.load.mockResolvedValueOnce(firstTexture).mockResolvedValueOnce(secondTexture); + const player = new LumaPlayer(edit as never, createLumaClip()); + + await player.load(); + expect(player.getMediaTimingState()).toMatchObject({ status: "ready", duration: 4 }); + expect(player.getLoadedResourceIdentifier()).toBe("https://example.com/luma-a.mp4"); + + player.clipConfiguration.asset = { type: "luma", src: "https://example.com/luma-b.mp4" }; + const reloading = player.reloadAsset(); + expect(player.getMediaTimingState()).toMatchObject({ status: "pending" }); + await reloading; + + expect(edit.assetLoader.release).toHaveBeenCalledWith("https://example.com/luma-a.mp4"); + expect(player.getLoadedResourceIdentifier()).toBe("https://example.com/luma-b.mp4"); + expect(player.getMediaTimingState()).toMatchObject({ status: "ready", duration: 6 }); + }); + + it("seeks luma video from its trim offset", async () => { + const edit = createEdit(); + edit.playbackTime = 1.25; + const texture = createVideoTexture(1280, 720, 8); + edit.assetLoader.load.mockResolvedValueOnce(texture); + const clip = createLumaClip(); + clip.asset = { type: "luma", src: "https://example.com/luma.mp4", trim: 2 }; + const player = new LumaPlayer(edit as never, clip); + + await player.load(); + player.update(0, 0); + + expect(texture.source.resource.currentTime).toBe(3.25); + }); + + it("rejects stale metadata when an asset changes A to B to A", () => { + const player = new TimingStatePlayer(createEdit(), createImageClip()); + const firstA = player.beginTimingLoad(); + player.clipConfiguration.asset = { type: "image", src: "https://example.com/b.png" }; + player.beginTimingLoad(); + player.clipConfiguration.asset = { type: "image", src: "https://example.com/missing.png" }; + const currentA = player.beginTimingLoad(); + + player.completeTimingLoad(firstA, 9); + expect(player.getMediaTimingState()).toMatchObject({ status: "pending" }); + + player.completeTimingLoad(currentA, 2.5); + expect(player.getMediaTimingState()).toMatchObject({ status: "ready", duration: 2.5 }); + }); + + it("publishes a changed non-temporal asset through the base timing contract", async () => { + const player = new TimingStatePlayer(createEdit(), createImageClip()); + player.clipConfiguration.asset = { type: "shape", shape: "rectangle", width: 100, height: 100 } as never; + + await player.reloadAsset(); + + expect(player.getMediaTimingState()).toEqual({ + status: "ready", + asset: { type: "shape", src: null }, + duration: null + }); }); }); diff --git a/tests/media-thumbnail-renderer.test.ts b/tests/media-thumbnail-renderer.test.ts index a196b89c..eac08bdc 100644 --- a/tests/media-thumbnail-renderer.test.ts +++ b/tests/media-thumbnail-renderer.test.ts @@ -215,6 +215,29 @@ describe("MediaThumbnailRenderer", () => { // the code path doesn't throw and handles the type correctly }); + it("uses a frozen first frame for GIF thumbnails", async () => { + const generator = createMockGenerator(); + const firstFrame = "data:image/png;base64,first-frame"; + const getGifThumbnail = jest.fn().mockResolvedValue({ + isGif: true, + dataUrl: firstFrame, + width: 320, + height: 180 + }); + const renderer = new MediaThumbnailRenderer(generator, () => {}, getGifThumbnail); + const element = createMockElement(); + const clip = createMockClip({ + asset: { type: "image", src: "https://example.com/animated.gif" } as never + }); + + renderer.render(clip, element); + await delay(10); + + expect(getGifThumbnail).toHaveBeenCalledWith("https://example.com/animated.gif"); + expect(element.style.backgroundImage).toBe(`url("${firstFrame}")`); + expect(element.style.backgroundImage).not.toContain("animated.gif"); + }); + it("applies thumbnail for video type", async () => { const generator = createMockGenerator({ dataUrl: "data:image/png;base64,thumb", thumbnailWidth: 120 }); const onRendered = jest.fn(); diff --git a/tests/player-reconciler-loads.test.ts b/tests/player-reconciler-loads.test.ts new file mode 100644 index 00000000..2deb8a55 --- /dev/null +++ b/tests/player-reconciler-loads.test.ts @@ -0,0 +1,127 @@ +import type { Player } from "@canvas/players/player"; +import { PlayerReconciler } from "@core/player-reconciler"; +import type { Edit } from "@core/edit-session"; +import type { ResolvedEdit } from "@schemas"; + +describe("PlayerReconciler initial loads", () => { + it("does not resolve initial reconciliation before newly-created Players load", async () => { + let finishLoad!: () => void; + const loadPromise = new Promise(resolve => { + finishLoad = resolve; + }); + const player = { + layer: 0, + clipId: null, + needsResolution: false, + getTimingIntent: () => ({ start: 0, length: 3 }), + load: jest.fn().mockReturnValue(loadPromise) + } as unknown as Player; + const players = new Map(); + const tracks: Player[][] = [[]]; + const events = { on: jest.fn(), emit: jest.fn() }; + const edit = { + getInternalEvents: () => events, + getPlayerByClipId: (id: string) => players.get(id) ?? null, + createPlayerFromAssetType: () => player, + registerPlayerByClipId: (id: string, value: Player) => players.set(id, value), + addPlayerToTracksArray: (trackIndex: number, value: Player) => tracks[trackIndex].push(value), + addPlayerToContainer: jest.fn(), + getTracks: () => tracks, + getPlayerMap: () => players, + removeEmptyTrack: jest.fn(), + ensureTrackExists: jest.fn() + } as unknown as Edit; + const resolved = { + timeline: { + tracks: [ + { + clips: [ + { + id: "gif-clip", + asset: { type: "image", src: "https://example.com/animation.gif" }, + start: 0, + length: 3 + } + ] + } + ] + } + } as unknown as ResolvedEdit; + const reconciler = new PlayerReconciler(edit); + let settled = false; + + const reconciliation = reconciler.reconcileInitial(resolved).then(() => { + settled = true; + }); + await Promise.resolve(); + + expect(player.load).toHaveBeenCalledTimes(1); + expect(settled).toBe(false); + + finishLoad(); + await reconciliation; + expect(settled).toBe(true); + }); + + it("waits for every overlapping load for the same Player", async () => { + let finishFirst!: () => void; + let finishSecond!: () => void; + const first = new Promise(resolve => { + finishFirst = resolve; + }); + const second = new Promise(resolve => { + finishSecond = resolve; + }); + const events = { on: jest.fn(), off: jest.fn() }; + const reconciler = new PlayerReconciler({ getInternalEvents: () => events } as unknown as Edit); + const player = {} as Player; + const trackPlayerLoad = ( + reconciler as unknown as { trackPlayerLoad: (value: Player, promise: Promise) => Promise } + ).trackPlayerLoad.bind(reconciler); + trackPlayerLoad(player, first); + trackPlayerLoad(player, second); + + let settled = false; + const waiting = reconciler.whenPlayerSettled(player).then(() => { + settled = true; + }); + finishSecond(); + await Promise.resolve(); + await Promise.resolve(); + expect(settled).toBe(false); + + finishFirst(); + await waiting; + expect(settled).toBe(true); + }); + + it("re-resolves auto timing after any current Player finishes loading", async () => { + let finishLoad!: () => void; + const load = new Promise(resolve => { + finishLoad = resolve; + }); + const player = { + clipId: "gif-clip", + getTimingIntent: () => ({ start: 0, length: "auto" }) + } as unknown as Player; + const players = new Map([["gif-clip", player]]); + const resolveClipAutoLength = jest.fn().mockResolvedValue(undefined); + const edit = { + getInternalEvents: () => ({ on: jest.fn(), off: jest.fn() }), + getPlayerMap: () => players, + resolveClipAutoLength + } as unknown as Edit; + const reconciler = new PlayerReconciler(edit); + const trackPlayerLoad = ( + reconciler as unknown as { trackPlayerLoad: (value: Player, promise: Promise) => Promise } + ).trackPlayerLoad.bind(reconciler); + + trackPlayerLoad(player, load); + finishLoad(); + await load; + await Promise.resolve(); + await Promise.resolve(); + + expect(resolveClipAutoLength).toHaveBeenCalledWith(player); + }); +}); diff --git a/tests/player-sync.test.ts b/tests/player-sync.test.ts index 868c43c5..9e90e390 100644 --- a/tests/player-sync.test.ts +++ b/tests/player-sync.test.ts @@ -58,10 +58,23 @@ jest.mock("pixi.js", () => { Container: jest.fn().mockImplementation(createMockContainer), Graphics: jest.fn().mockImplementation(() => ({ rect: jest.fn().mockReturnThis(), + roundRect: jest.fn().mockReturnThis(), fill: jest.fn().mockReturnThis(), clear: jest.fn().mockReturnThis(), destroy: jest.fn() })), + Text: jest.fn().mockImplementation(() => ({ + visible: false, + filters: [], + text: "", + x: 0, + y: 0, + width: 100, + height: 30, + position: { set: jest.fn() }, + destroy: jest.fn() + })), + TextStyle: jest.fn().mockImplementation(() => ({})), Sprite: jest.fn().mockImplementation(() => ({ texture: {}, width: 1920, @@ -156,7 +169,7 @@ jest.mock("@loaders/subtitle-load-parser", () => ({ // Mock font config jest.mock("@core/fonts/font-config", () => ({ - parseFontFamily: jest.fn().mockReturnValue("Arial"), + parseFontFamily: jest.fn().mockReturnValue({ baseFontFamily: "Arial", fontWeight: 400 }), resolveFontPath: jest.fn().mockReturnValue(null) })); @@ -188,8 +201,10 @@ function createMockEdit(playbackTimeSec: number): Edit { load: jest.fn(), loadVideoUnique: jest.fn(), incrementRef: jest.fn(), - decrementRef: jest.fn() + decrementRef: jest.fn(), + release: jest.fn() }, + size: { width: 1920, height: 1080 }, events: { emit: jest.fn(), on: jest.fn(), off: jest.fn() }, output: { size: { width: 1920, height: 1080 } } } as unknown as Edit; @@ -383,6 +398,101 @@ describe("CaptionPlayer", () => { expect(player.getPlaybackTime()).toBe(7.5); }); }); + + describe("intrinsic media timing", () => { + it("does not resume loading into disposed containers", async () => { + const edit = createMockEdit(0); + const player = new CaptionPlayer(edit, createCaptionClipConfig(0)); + const graphics = pixi.Graphics as unknown as jest.Mock; + graphics.mockClear(); + + const loading = player.load(); + expect(player.getMediaTimingState()).toMatchObject({ status: "pending" }); + player.dispose(); + await loading; + + expect(graphics).not.toHaveBeenCalled(); + expect(edit.assetLoader.load).not.toHaveBeenCalled(); + }); + + it("publishes the final parsed cue end in seconds", async () => { + const edit = createMockEdit(0); + const load = edit.assetLoader.load as jest.Mock; + load.mockResolvedValue({ + cues: [ + { start: 0, end: 1.2, text: "First" }, + { start: 1.5, end: 4.25, text: "Last" } + ] + }); + const player = new CaptionPlayer(edit, createCaptionClipConfig(0)); + + await player.load(); + + expect(player.getMediaTimingState()).toEqual({ + status: "ready", + asset: { type: "caption", src: "captions.srt" }, + duration: 4.25 + }); + }); + + it("publishes ready without a duration when subtitle loading fails", async () => { + const warnSpy = jest.spyOn(console, "warn").mockImplementation(); + const edit = createMockEdit(0); + const load = edit.assetLoader.load as jest.Mock; + load.mockRejectedValue(new Error("network error")); + const player = new CaptionPlayer(edit, createCaptionClipConfig(0)); + + await player.load(); + + expect(player.getMediaTimingState()).toEqual({ + status: "ready", + asset: { type: "caption", src: "captions.srt" }, + duration: null + }); + warnSpy.mockRestore(); + }); + + it("does not let a stale reload replace newer cue timing or state", async () => { + const edit = createMockEdit(0); + const load = edit.assetLoader.load as jest.Mock; + load.mockResolvedValueOnce({ cues: [{ start: 0, end: 1, text: "Initial" }] }); + const player = new CaptionPlayer(edit, createCaptionClipConfig(0)); + await player.load(); + + let resolveFirst!: (value: unknown) => void; + let resolveSecond!: (value: unknown) => void; + load.mockImplementation( + (src: string) => + new Promise(resolve => { + if (src === "first.srt") resolveFirst = resolve; + if (src === "second.srt") resolveSecond = resolve; + }) + ); + + (player as unknown as { clipConfiguration: ResolvedClip }).clipConfiguration.asset = { + type: "caption", + src: "first.srt" + } as CaptionAsset; + const firstReload = player.reloadAsset(); + (player as unknown as { clipConfiguration: ResolvedClip }).clipConfiguration.asset = { + type: "caption", + src: "second.srt" + } as CaptionAsset; + const secondReload = player.reloadAsset(); + + resolveSecond({ cues: [{ start: 0, end: 6, text: "Second" }] }); + await secondReload; + resolveFirst({ cues: [{ start: 0, end: 9, text: "First" }] }); + await firstReload; + + expect(player.getMediaTimingState()).toEqual({ + status: "ready", + asset: { type: "caption", src: "second.srt" }, + duration: 6 + }); + expect((player as unknown as { state: { cues: Array<{ text: string }> } }).state.cues[0]?.text).toBe("Second"); + }); + }); }); // ───────────────────────────────────────────────────────────────────────────── diff --git a/tests/resolver.test.ts b/tests/resolver.test.ts index eaaec278..550eb5b4 100644 --- a/tests/resolver.test.ts +++ b/tests/resolver.test.ts @@ -2,6 +2,7 @@ import { EditDocument } from "@core/edit-document"; import { EventEmitter } from "@core/events/event-emitter"; import { MergeFieldService } from "@core/merge/merge-field-service"; import { resolve } from "@core/resolver"; +import { sec } from "@core/timing/types"; import type { Edit } from "@schemas"; function createMergeFieldService(): MergeFieldService { @@ -79,6 +80,48 @@ describe("Resolver", () => { expect(clips[2].start).toBe(5); // After second clip (2 + 3) }); + it("uses source-matched intrinsic GIF timing for dependants", () => { + const src = "https://example.com/animation.gif"; + const doc = new EditDocument({ + timeline: { + tracks: [ + { + clips: [ + { alias: "animation", asset: { type: "image", src }, start: 0, length: "auto" }, + { asset: { type: "image", src: "https://example.com/next.jpg" }, start: "auto", length: 1 } + ] + }, + { + clips: [{ asset: { type: "image", src: "https://example.com/dependent.jpg" }, start: 0, length: "alias://animation" }] + }, + { + clips: [{ asset: { type: "image", src: "https://example.com/background.jpg" }, start: 0, length: "end" }] + } + ] + }, + output: { format: "mp4", size: { width: 1920, height: 1080 } } + }); + const clipId = doc.getClipId(0, 0)!; + const mergeFields = createMergeFieldService(); + const resolved = resolve(doc, { + mergeFields, + mediaTimingByClipId: new Map([[clipId, { status: "ready", asset: { type: "image", src }, duration: sec(2.4) }]]) + }); + + expect(resolved.timeline.tracks[0].clips[0].length).toBe(2.4); + expect(resolved.timeline.tracks[0].clips[1].start).toBe(2.4); + expect(resolved.timeline.tracks[1].clips[0].length).toBe(2.4); + expect(resolved.timeline.tracks[2].clips[0].length).toBe(3.4); + + const stale = resolve(doc, { + mergeFields, + mediaTimingByClipId: new Map([ + [clipId, { status: "ready", asset: { type: "image", src: "https://example.com/old.gif" }, duration: sec(9) }] + ]) + }); + expect(stale.timeline.tracks[0].clips[0].length).toBe(3); + }); + it("resolves 'end' length to extend to timeline end", () => { const doc = new EditDocument(createEditWithEndLength()); const mergeFields = createMergeFieldService(); @@ -192,8 +235,7 @@ describe("Resolver", () => { expect(resolved.timeline.tracks[0].clips).toEqual([]); }); - it("handles 'end' length with minimum duration", () => { - // When timeline end equals clip start, ensure minimum length + it("handles an end-only timeline without inventing duration", () => { const edit: Edit = { timeline: { tracks: [ @@ -210,8 +252,7 @@ describe("Resolver", () => { const resolved = resolve(doc, { mergeFields }); - // Should have minimum length of 0.1 - expect(resolved.timeline.tracks[0].clips[0].length).toBeGreaterThanOrEqual(0.1); + expect(resolved.timeline.tracks[0].clips[0].length).toBe(0); }); describe("alias validation", () => { diff --git a/tests/rich-caption-player.test.ts b/tests/rich-caption-player.test.ts index c8b23a5f..0e7e5ffa 100644 --- a/tests/rich-caption-player.test.ts +++ b/tests/rich-caption-player.test.ts @@ -59,7 +59,7 @@ jest.mock("pixi.js", () => { Texture: { from: jest.fn().mockImplementation(() => ({ destroy: jest.fn(), - update: jest.fn() + source: { update: jest.fn() } })), WHITE: {} }, @@ -353,15 +353,26 @@ describe("RichCaptionPlayer", () => { expect((player as unknown as { clipConfiguration: Record }).clipConfiguration["fit"]).toBeUndefined(); }); - it("loads successfully with valid inline words", async () => { + it("loads inline words and publishes their greatest end plus caption tail", async () => { const edit = createMockEdit(); - const player = new RichCaptionPlayer(edit, createClip(createAsset())); + const player = new RichCaptionPlayer( + edit, + createClip( + createAsset({ + words: [ + { text: "Last in time", start: 1000, end: 1400 }, + { text: "Last in array", start: 500, end: 900 } + ] + } as Partial) + ) + ); await player.load(); // @ts-expect-error accessing private property expect(player.loadComplete).toBe(true); expect(mockLayoutCaption).toHaveBeenCalledTimes(1); expect(mockGenerateRichCaptionFrame).toHaveBeenCalledTimes(1); + expect(player.getMediaTimingState()).toMatchObject({ status: "ready", duration: 1.9 }); }); it("falls back to placeholder on invalid asset", async () => { @@ -620,6 +631,19 @@ describe("RichCaptionPlayer", () => { }); describe("Lifecycle", () => { + it("does not resume loading after same-turn disposal", async () => { + const edit = createMockEdit(); + const player = new RichCaptionPlayer(edit, createClip(createAsset())); + + const loading = player.load(); + expect(player.getMediaTimingState()).toMatchObject({ status: "pending" }); + player.dispose(); + await loading; + + expect(mockGetSharedInstance).not.toHaveBeenCalled(); + expect(mockLayoutCaption).not.toHaveBeenCalled(); + }); + it("releases FontRegistry reference on dispose", async () => { const edit = createMockEdit(); const player = new RichCaptionPlayer(edit, createClip(createAsset())); @@ -1104,6 +1128,18 @@ describe("RichCaptionPlayer", () => { }); }); + it("does not rebuild a loaded caption pipeline for a numeric timing-only change", async () => { + const edit = createMockEdit(); + const player = new RichCaptionPlayer(edit, createClip(createAsset())); + await player.load(); + mockLayoutCaption.mockClear(); + + player.setResolvedTiming({ start: sec(1), length: sec(6) }); + player.reconfigureAfterRestore(); + + expect(mockLayoutCaption).not.toHaveBeenCalled(); + }); + describe("Alias Placeholder", () => { it("detects alias reference and sets placeholder flags", async () => { const asset = createAsset({ src: "alias://VIDEO", words: undefined } as Partial); @@ -1513,9 +1549,8 @@ describe("RichCaptionPlayer", () => { setTimeout(resolve, 10); }); - // layoutCaption was called twice - expect(mockLayoutCaption).toHaveBeenCalledTimes(2); - // But only the second layout should have rendered (first was discarded by stale-ID guard) + // The first build becomes stale before expensive layout starts; only the latest renders. + expect(mockLayoutCaption).toHaveBeenCalledTimes(1); expect(mockGenerateRichCaptionFrame).toHaveBeenCalledTimes(1); }); diff --git a/vite.shared.ts b/vite.shared.ts index 47fcf33e..d763ba57 100644 --- a/vite.shared.ts +++ b/vite.shared.ts @@ -20,6 +20,7 @@ export const globals: Record = { }; export function external(id: string): boolean { + if (id === "pixi.js/gif") return false; if (id === "pixi.js" || id.startsWith("pixi.js/")) return true; if (id === "pixi-filters" || id.startsWith("pixi-filters/")) return true; if (id.startsWith("@napi-rs/")) return true;