From 658dcd325979a38d50c4518f427f8e07b3a17289 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Sat, 11 Jul 2026 15:44:33 +0000 Subject: [PATCH] fix(skills): align media guidance with CLI contracts --- skills-manifest.json | 4 +-- .../references/script-format.md | 2 +- .../media-use/audio/references/transcribe.md | 24 +++++++-------- .../audio/references/tts-to-captions.md | 5 ++-- skills/media-use/audio/references/tts.md | 30 +++++++++---------- skills/media-use/scripts/lib/registry.mjs | 3 ++ .../media-use/scripts/lib/registry.test.mjs | 15 ++++++++-- skills/media-use/scripts/resolve.test.mjs | 14 +++++++++ 8 files changed, 63 insertions(+), 34 deletions(-) diff --git a/skills-manifest.json b/skills-manifest.json index 9321089cb1..4e6f187cba 100644 --- a/skills-manifest.json +++ b/skills-manifest.json @@ -30,7 +30,7 @@ "files": 7 }, "hyperframes-core": { - "hash": "690ebb3ba5420b90", + "hash": "c3fdc0fb238d5a56", "files": 14 }, "hyperframes-creative": { @@ -46,7 +46,7 @@ "files": 10 }, "media-use": { - "hash": "49aa5adae0800403", + "hash": "0a0665d20e7ca91d", "files": 122 }, "motion-graphics": { diff --git a/skills/hyperframes-core/references/script-format.md b/skills/hyperframes-core/references/script-format.md index 1946bf9a37..396456b6a7 100644 --- a/skills/hyperframes-core/references/script-format.md +++ b/skills/hyperframes-core/references/script-format.md @@ -46,4 +46,4 @@ A header block, then one section per spoken line. ## To TTS -Feed each line's spoken text to `npx hyperframes tts` (pin `--voice` / `--provider` from the header; capture word timestamps for captions). Real per-word timing replaces the `**Time:**` guides. CLI contract → `media-use/audio/references/tts.md`. +Feed each line's spoken text to the TTS path selected in the header. For local Kokoro, run `npx hyperframes tts` and pin `--voice`; for cloud providers and word timestamps, use the provider-specific scripts. Real per-word timing replaces the `**Time:**` guides. Contract → `media-use/audio/references/tts.md`. diff --git a/skills/media-use/audio/references/transcribe.md b/skills/media-use/audio/references/transcribe.md index 06df2ce445..21d4240060 100644 --- a/skills/media-use/audio/references/transcribe.md +++ b/skills/media-use/audio/references/transcribe.md @@ -4,8 +4,8 @@ Create normalized word-level timestamps. **Always specify `--model` explicitly** ```bash npx hyperframes transcribe audio.mp3 --model small.en # known English -npx hyperframes transcribe video.mp4 --model small --language es # known Spanish -npx hyperframes transcribe audio.mp3 --model small # unknown language (auto-detect) +npx hyperframes transcribe video.mp4 --model large-v3 --language es # known Spanish +npx hyperframes transcribe audio.mp3 --model large-v3 # unknown language (auto-detect) npx hyperframes transcribe subtitles.srt # import existing npx hyperframes transcribe subtitles.vtt npx hyperframes transcribe openai-response.json @@ -16,27 +16,27 @@ npx hyperframes transcribe openai-response.json `.en` models (`tiny.en` / `base.en` / `small.en` / `medium.en`) **translate** non-English audio into English. This silently destroys the original language. 1. **Known English** → `--model small.en` (or `medium.en` for music / noisy audio) -2. **Known non-English** → `--model small --language ` (no `.en` suffix) -3. **Unknown language** → `--model small` (whisper auto-detects) +2. **Known non-English** → `--model large-v3 --language ` +3. **Unknown language** → `--model large-v3` (whisper auto-detects) **CLI default is `small.en`** — do not rely on it; always pass `--model` to make the choice explicit. `--language` also filters out non-target-language segments from mixed-language audio. ## Model Sizes -| Model | Size | Speed | When | -| ---------- | ------ | -------- | ------------------------------------- | -| `tiny` | 75 MB | Fastest | Quick previews, smoke tests | -| `base` | 142 MB | Fast | Short clips, clear audio | -| `small` | 466 MB | Moderate | Default for most multilingual content | -| `medium` | 1.5 GB | Slow | Music with vocals, noisy audio | -| `large-v3` | 3.1 GB | Slowest | Production quality | +| Model | Size | Speed | When | +| ----------- | ------ | -------- | -------------------------------- | +| `tiny.en` | 75 MB | Fastest | English previews and smoke tests | +| `base.en` | 142 MB | Fast | Short, clear English speech | +| `small.en` | 466 MB | Moderate | Default for English speech | +| `medium.en` | 1.5 GB | Slow | Noisy English audio and vocals | +| `large-v3` | 3.1 GB | Slowest | Production quality | ### Picking a model by content type 1. Speech over silence / light background → `small.en` 2. Speech over music, or music with vocals → start with `medium.en` 3. Produced music track (vocals + full instrumentation) → start with `medium.en`; expect to need manual lyrics or an external API ([`captions/transcript-handling.md`](captions/transcript-handling.md) → "Using External Transcription APIs") -4. Multilingual → `medium` or `large-v3` (no `.en` suffix), pair with `--language` +4. Multilingual → `large-v3`, pair with `--language` when known ## Output Shape diff --git a/skills/media-use/audio/references/tts-to-captions.md b/skills/media-use/audio/references/tts-to-captions.md index bad5f86eae..bdb9f6eb84 100644 --- a/skills/media-use/audio/references/tts-to-captions.md +++ b/skills/media-use/audio/references/tts-to-captions.md @@ -7,7 +7,8 @@ When no recorded voiceover exists, generate one and obtain word-level caption ti HeyGen returns word timestamps in the same response as the audio. Pass `--words` and you're done: ```bash -npx hyperframes tts script.txt --provider heygen --output narration.wav --words narration.words.json +node /audio/scripts/heygen-tts.mjs \ + script.txt --output narration.wav --words narration.words.json ``` `narration.words.json` is already in the `[{ id, text, start, end }]` shape the captions pipeline consumes — no separate transcribe pass. @@ -21,4 +22,4 @@ npx hyperframes tts script.txt --voice af_heart --output narration.wav npx hyperframes transcribe narration.wav --model small.en # voice af_heart is American English ``` -Whisper extracts precise word boundaries from the generated audio, so caption timing matches delivery without hand-tuning. Match `--model` to the voice's language (use `small.en` for `a`/`b` prefixes, `small --language ` otherwise). Then consume `transcript.json` via the caption references in `captions/`. +Whisper extracts precise word boundaries from the generated audio, so caption timing matches delivery without hand-tuning. Match `--model` to the voice's language (use `small.en` for `a`/`b` prefixes, `large-v3 --language ` otherwise). Then consume `transcript.json` via the caption references in `captions/`. diff --git a/skills/media-use/audio/references/tts.md b/skills/media-use/audio/references/tts.md index 8763d1b875..984aadb7a4 100644 --- a/skills/media-use/audio/references/tts.md +++ b/skills/media-use/audio/references/tts.md @@ -1,10 +1,13 @@ # Text To Speech -`npx hyperframes tts` auto-detects a provider from env vars; explicit override via `--provider`. +`npx hyperframes tts` is the local Kokoro-82M command. It accepts `--voice`, +`--lang`, `--speed`, and `--output`; it does **not** accept `--provider` or +`--words`. Use the provider-specific scripts below for cloud voices and word +timestamps. > **Run the Preflight first — no credential is not a green light to silently use the local voice.** Before generating a voiceover, complete the sign-in **Preflight** (see `../SKILL.md` → Preflight): run `npx hyperframes auth status`, recommend signing in, and **STOP for the user's choice** (sign in for HeyGen voices, or continue offline with local Kokoro). This applies to a one-off "generate a voiceover" request just as much as inside a full workflow. -## Provider chain +## Media workflow provider chain | Order | Provider | Env trigger | Voice IDs | Word timestamps | Audio format | | ----- | ----------------- | ------------------------------------------- | ------------------------------------------- | ----------------------------------------- | -------------------- | @@ -13,16 +16,8 @@ | 3 | Kokoro-82M | always (local fallback) | `am_michael`, `af_heart`, … (54 voices) | No | wav direct | ```bash -# Auto-detect (HeyGen if key set, else ElevenLabs, else Kokoro) +# Local Kokoro npx hyperframes tts "Welcome to HyperFrames" -o narration.wav - -# Pin the provider explicitly -npx hyperframes tts "Hello" --provider kokoro -npx hyperframes tts "Hello" --provider heygen --voice -npx hyperframes tts "Hello" --provider elevenlabs --voice 21m00Tcm4TlvDq8ikWAM - -# HeyGen path: capture word timestamps in one call (skips a Whisper pass) -npx hyperframes tts "Hi there" --words narration.words.json ``` ## Self-contained HeyGen (no CLI) — `scripts/heygen-tts.mjs` @@ -70,7 +65,11 @@ node skills/media-use/audio/scripts/heygen-tts.mjs --list # public starfish vo ## ffmpeg requirement -HeyGen + ElevenLabs return mp3. The CLI transcodes to wav when `--output` ends in `.wav` (the default and what downstream `ffprobe` + Whisper expect). If you'd rather skip the transcode, pass `-o file.mp3`. Without `ffmpeg` on PATH, `.wav` output from the cloud providers fails — install ffmpeg or use `.mp3`. +HeyGen + ElevenLabs return mp3. Their provider-specific scripts transcode to wav +when `--output` ends in `.wav` (the default and what downstream `ffprobe` + +Whisper expect). If you'd rather skip the transcode, pass `-o file.mp3`. +Without `ffmpeg` on PATH, `.wav` output from cloud providers fails — install +ffmpeg or use `.mp3`. Local `hyperframes tts` writes Kokoro wav directly. ## Voice selection (Kokoro) @@ -103,8 +102,8 @@ The first letter of a Kokoro voice ID picks the phonemizer language; `--lang` ov | `z` | Mandarin | ```bash -npx hyperframes tts "La reunión empieza a las nueve" --voice ef_dora --provider kokoro -npx hyperframes tts "Today is a nice day" --voice af_heart --provider kokoro +npx hyperframes tts "La reunión empieza a las nueve" --voice ef_dora +npx hyperframes tts "Today is a nice day" --voice af_heart ``` Valid `--lang` codes (only needed to override the voice's auto-detected language): `en-us`, `en-gb`, `es`, `fr-fr`, `hi`, `it`, `pt-br`, `ja`, `zh`. @@ -118,7 +117,8 @@ Non-English phonemization requires `espeak-ng` system-wide (`brew install espeak - `1.1-1.2` — intros, transitions, upbeat content - `1.5+` — rarely appropriate, test carefully -Honored by Kokoro + HeyGen; ElevenLabs ignores `--speed` (use voice settings on their dashboard). +`hyperframes tts --speed` controls local Kokoro. Cloud-provider speed support is +documented by each provider-specific script; do not pass CLI-only flags to them. ## Long scripts diff --git a/skills/media-use/scripts/lib/registry.mjs b/skills/media-use/scripts/lib/registry.mjs index 84df07ccf9..4fa7418823 100644 --- a/skills/media-use/scripts/lib/registry.mjs +++ b/skills/media-use/scripts/lib/registry.mjs @@ -83,6 +83,9 @@ const REGISTRY = { // Local design spec, not heygen — reads frame.md / design.md tokens. A("design_spec", { search: brandProvider.search }), ], + // Local/direct-URL ingest only. Declaring video here lets `resolve --from` + // freeze and ledger prepared clips without implying a generative provider. + video: [A("ingest.only", {})], grade: [ // Local deterministic cascade handled by resolve.mjs so grade records can // carry an inline block as well as an optional frozen .cube file. diff --git a/skills/media-use/scripts/lib/registry.test.mjs b/skills/media-use/scripts/lib/registry.test.mjs index 6cd51ac430..796215a0b0 100644 --- a/skills/media-use/scripts/lib/registry.test.mjs +++ b/skills/media-use/scripts/lib/registry.test.mjs @@ -6,7 +6,18 @@ import { getProviders, getProvider, listTypes, runProviders, runCapability } fro test("listTypes exposes the v2 media types", () => { const types = listTypes(); - for (const t of ["bgm", "sfx", "image", "icon", "logo", "voice", "brand", "grade", "lut"]) { + for (const t of [ + "bgm", + "sfx", + "image", + "icon", + "logo", + "voice", + "brand", + "video", + "grade", + "lut", + ]) { assert.ok(types.includes(t), `missing type: ${t}`); } }); @@ -21,7 +32,7 @@ test("heygen provider is first for every type it serves", () => { test("sanctioned providers only: heygen, local mflux/kokoro, codex, design spec, logo tiers", () => { const allowed = - /^heygen|^mflux\.local$|^kokoro\.local$|^codex\.image_gen$|^design_spec$|^svgl$|^simple-icons$|^github\.avatar$|^favicon\.ddg$|^color_grade\.local$|^cube_lut\.local$/; + /^heygen|^mflux\.local$|^kokoro\.local$|^codex\.image_gen$|^design_spec$|^svgl$|^simple-icons$|^github\.avatar$|^favicon\.ddg$|^ingest\.only$|^color_grade\.local$|^cube_lut\.local$/; for (const t of listTypes()) { for (const p of getProviders(t)) { assert.ok(allowed.test(p.name), `${t} lists unsanctioned provider: ${p.name}`); diff --git a/skills/media-use/scripts/resolve.test.mjs b/skills/media-use/scripts/resolve.test.mjs index fea0e3be8c..f5ff3c2593 100644 --- a/skills/media-use/scripts/resolve.test.mjs +++ b/skills/media-use/scripts/resolve.test.mjs @@ -640,6 +640,20 @@ test("--from rejects invalid lut cube without registering or leaving a frozen fi cleanup(); }); +test("--from ingests a local video when type is video", () => { + setup(); + const video = join(tmp, "demo.mp4"); + writeFileSync(video, "video fixture"); + + const out = runResolve(["--from", video, "--type", "video", "--project", tmp, "--json"]); + const parsed = JSON.parse(out); + + assert.equal(parsed.type, "video"); + assert.ok(parsed.path.endsWith(".mp4")); + assert.ok(existsSync(join(tmp, parsed.path))); + cleanup(); +}); + test("grade miss exits explicitly with no partial file", () => { setup(); const missIntent = `zqxv imaginary neutron ${process.pid}`;