diff --git a/electron/stt/snapWordBoundaries.test.ts b/electron/stt/snapWordBoundaries.test.ts new file mode 100644 index 000000000..2668c6966 --- /dev/null +++ b/electron/stt/snapWordBoundaries.test.ts @@ -0,0 +1,76 @@ +import { describe, expect, it } from "vitest"; +import { snapWordBoundariesToAudio } from "./snapWordBoundaries"; +import type { SttWordSegment } from "./transcriptionContract"; + +const SAMPLE_RATE = 16_000; + +/** Mono 16 kHz buffer that is loud everywhere except the given silent spans. */ +function audioWithSilences(durationSec: number, silences: Array<[number, number]>): Float32Array { + const samples = new Float32Array(Math.round(durationSec * SAMPLE_RATE)); + for (let i = 0; i < samples.length; i++) { + const t = i / SAMPLE_RATE; + const silent = silences.some(([from, to]) => t >= from && t < to); + // Alternating ±0.5 gives a flat, non-zero RMS without needing a real tone. + samples[i] = silent ? 0 : i % 2 === 0 ? 0.5 : -0.5; + } + return samples; +} + +const word = (w: Partial = {}): SttWordSegment => ({ + word: "w", + startSec: 0, + endSec: 0.1, + ...w, +}); + +describe("snapWordBoundariesToAudio", () => { + it("pulls a late boundary back into the silence that precedes it", () => { + // Speech stops at 1.0 and resumes at 1.2; whisper reports the next word + // starting at 1.3 — 100 ms after the audio actually resumed. + const samples = audioWithSilences(3, [[1.0, 1.2]]); + const [snapped] = snapWordBoundariesToAudio([word({ startSec: 1.3, endSec: 1.8 })], samples); + expect(snapped.startSec).toBeGreaterThanOrEqual(1.0); + expect(snapped.startSec).toBeLessThan(1.2); + }); + + it("leaves a boundary alone when nothing quieter precedes it", () => { + // A word ending a phrase: whisper is already right, the frames before the + // boundary are all speech, so the quietest frame in the window is the + // boundary itself and it must not drift. + const samples = audioWithSilences(3, [[1.5, 2.0]]); + const [snapped] = snapWordBoundariesToAudio([word({ startSec: 1.0, endSec: 1.5 })], samples); + expect(snapped.endSec).toBeCloseTo(1.5, 2); + }); + + it("never moves a boundary more than the lookback window", () => { + const samples = audioWithSilences(3, [[0.0, 1.0]]); + const [snapped] = snapWordBoundariesToAudio([word({ startSec: 2.0, endSec: 2.5 })], samples); + expect(snapped.startSec).toBeGreaterThanOrEqual(2.0 - 0.15); + }); + + it("keeps a boundary shared by two words shared", () => { + const samples = audioWithSilences(3, [[1.0, 1.2]]); + const [first, second] = snapWordBoundariesToAudio( + [word({ startSec: 0.5, endSec: 1.3 }), word({ startSec: 1.3, endSec: 1.8 })], + samples, + ); + expect(first.endSec).toBeCloseTo(second.startSec, 6); + }); + + it("leaves boundaries that fall outside the decoded audio alone", () => { + // Clamping these into range would collapse every boundary onto the end of + // the buffer instead of leaving the unmeasurable ones untouched. + const samples = audioWithSilences(0.1, []); + const words = [word({ startSec: 5.51, endSec: 6.85 })]; + expect(snapWordBoundariesToAudio(words, samples)).toEqual(words); + }); + + it("keeps degenerate words non-empty and passes words through without audio", () => { + const samples = audioWithSilences(3, [[1.0, 1.2]]); + const [degenerate] = snapWordBoundariesToAudio([word({ startSec: 1.3, endSec: 1.3 })], samples); + expect(degenerate.endSec).toBeGreaterThan(degenerate.startSec); + + const untouched = [word({ startSec: 1.3, endSec: 1.8 })]; + expect(snapWordBoundariesToAudio(untouched, new Float32Array(0))).toEqual(untouched); + }); +}); diff --git a/electron/stt/snapWordBoundaries.ts b/electron/stt/snapWordBoundaries.ts new file mode 100644 index 000000000..d5d0b6f10 --- /dev/null +++ b/electron/stt/snapWordBoundaries.ts @@ -0,0 +1,100 @@ +// Pulls whisper.cpp's DTW word boundaries back onto the audio they describe. +// +// whisper.cpp reports one time per token and derives word spans from it, so a +// word's `start` is the point where the decoder *emitted* the token, not where +// the speaker started saying it. Measured on real recordings, that lands +// consistently 80–150 ms late, and because consecutive words share a boundary +// (`word[i].end === word[i+1].start`) the whole transcript is dragged right by +// roughly a syllable. +// +// That is invisible in captions but not in the transcript editor: deleting +// words there turns the selection into a trim of exactly +// `[firstWord.startSec, lastWord.endSec]`, so a late boundary leaves the attack +// of the first removed word audible and bites into the following kept word. +// +// The correction is to look at the audio instead of guessing an offset: each +// boundary moves back to the quietest 10 ms frame within the preceding +// `LOOKBACK_SEC`. It is self-limiting — on a decaying tail (a word that ends a +// phrase, where whisper is already right) the quietest frame IS the reported +// one, so the boundary doesn't move at all. + +import type { SttWordSegment } from "./transcriptionContract"; + +/** Matches `writeSamplesAsWav` — the samples handed to whisper are mono 16 kHz. */ +const SAMPLE_RATE = 16_000; + +/** RMS envelope resolution; finer than any boundary error worth correcting. */ +const FRAME_SEC = 0.01; + +/** + * How far back a boundary may travel — the calibration knob for this + * correction. Sized to the measured DTW lag (~80–150 ms). Widen it and + * boundaries inside continuous speech start snapping onto the *previous* + * syllable's trough; narrow it and the lag survives. + */ +const LOOKBACK_SEC = 0.15; + +/** Keep degenerate words (whisper sometimes reports end <= start) non-empty. */ +const MIN_WORD_SEC = 0.02; + +/** Per-frame RMS of the mono signal — the cheapest usable "is this speech" proxy. */ +function rmsEnvelope(samples: Float32Array): Float32Array { + const frameLength = Math.round(SAMPLE_RATE * FRAME_SEC); + const frameCount = Math.floor(samples.length / frameLength); + const envelope = new Float32Array(frameCount); + for (let f = 0; f < frameCount; f++) { + const start = f * frameLength; + let sum = 0; + for (let i = start; i < start + frameLength; i++) { + const v = samples[i] ?? 0; + sum += v * v; + } + envelope[f] = Math.sqrt(sum / frameLength); + } + return envelope; +} + +/** + * Move every word boundary back to the quietest frame in the `LOOKBACK_SEC` + * preceding it. Boundaries shared by two words snap identically (same input + * time), so the transcript stays gap-free where whisper made it gap-free. + * Returns the words unchanged when there is no audio to measure against. + */ +export function snapWordBoundariesToAudio( + words: SttWordSegment[], + samples: Float32Array, +): SttWordSegment[] { + const envelope = rmsEnvelope(samples); + if (envelope.length === 0) return words; + + const lookbackFrames = Math.round(LOOKBACK_SEC / FRAME_SEC); + const snap = (timeSec: number): number => { + const hi = Math.round(timeSec / FRAME_SEC); + // A boundary outside the audio we can measure (whisper occasionally reports + // times past the end of the samples) has nothing to snap to — clamping it + // into range would drag it to the end of the buffer instead. + if (hi <= 0 || hi >= envelope.length) return timeSec; + const lo = Math.max(0, hi - lookbackFrames); + let bestFrame = hi; + let bestRms = envelope[hi]; + // Strict `<` while walking backwards keeps the frame CLOSEST to the + // reported boundary when a whole stretch is equally quiet, so digital + // silence can't drag a boundary the full window. + for (let f = hi - 1; f >= lo; f--) { + if (envelope[f] < bestRms) { + bestRms = envelope[f]; + bestFrame = f; + } + } + return bestFrame * FRAME_SEC; + }; + + return words.map((w) => { + const startSec = snap(w.startSec); + return { + ...w, + startSec, + endSec: Math.max(snap(w.endSec), startSec + MIN_WORD_SEC), + }; + }); +} diff --git a/electron/stt/whisperServer.ts b/electron/stt/whisperServer.ts index 7c846555d..4762ebf6d 100644 --- a/electron/stt/whisperServer.ts +++ b/electron/stt/whisperServer.ts @@ -7,6 +7,7 @@ import path from "node:path"; import type { Readable } from "node:stream"; import { resolveBinaryPath } from "./gpuDetector"; +import { snapWordBoundariesToAudio } from "./snapWordBoundaries"; import type { SttBackend, SttPhraseSegment, SttWordSegment } from "./transcriptionContract"; import { cleanupWav, writeSamplesAsWav } from "./wav"; @@ -335,17 +336,23 @@ export class WhisperServerManager { return { text, startSec, endSec: Math.max(endSec, startSec + 0.05) }; }) .filter((s) => s.text.length > 0); - const wordSegments: SttWordSegment[] = raw - .flatMap((seg) => - (seg.words ?? []).map((w) => { - const word = (w.word ?? "").trim(); - const startSec = this.toSec(w.start, 0); - const endSec = this.toSec(w.end, startSec + 0.05); - const confidence = typeof w.probability === "number" ? w.probability : undefined; - return { word, startSec, endSec: Math.max(startSec + 0.02, endSec), confidence }; - }), - ) - .filter((w) => w.word.length > 0); + // whisper.cpp's DTW boundaries run ~80–150 ms behind the audio, which the + // transcript editor turns into imprecise trims (see snapWordBoundaries.ts). + // Re-anchor them on the same samples whisper was given. + const wordSegments: SttWordSegment[] = snapWordBoundariesToAudio( + raw + .flatMap((seg) => + (seg.words ?? []).map((w) => { + const word = (w.word ?? "").trim(); + const startSec = this.toSec(w.start, 0); + const endSec = this.toSec(w.end, startSec + 0.05); + const confidence = typeof w.probability === "number" ? w.probability : undefined; + return { word, startSec, endSec: Math.max(startSec + 0.02, endSec), confidence }; + }), + ) + .filter((w) => w.word.length > 0), + opts.samples, + ); const detectedLanguage = json.detected_language ?? json.language ?? "auto"; const backend = this.toBackend(json.backend); return { segments, wordSegments, detectedLanguage, backend }; diff --git a/technical-documentation/architecture/transcription-and-captions.md b/technical-documentation/architecture/transcription-and-captions.md index c6a0fe5e1..77962e341 100644 --- a/technical-documentation/architecture/transcription-and-captions.md +++ b/technical-documentation/architecture/transcription-and-captions.md @@ -84,6 +84,24 @@ it verbatim in the response. `word.end = t_dtw of the next word's first token` (or the segment's `t1` for the segment's last word). The result is a monotonic, gap-free timeline of word ranges that downstream code can use without rebasing. +5. **Re-anchor on the audio** — `t_dtw` marks where the decoder *emitted* a + token, not where the speaker started saying it, so every boundary lands + 80–150 ms late (measured across real recordings; the mean correction on + the reference clip is 83 ms). Because consecutive words share a boundary, + the whole transcript is dragged right by roughly a syllable. + [`electron/stt/snapWordBoundaries.ts`](../../electron/stt/snapWordBoundaries.ts) + pulls each boundary back to the quietest 10 ms frame in the preceding + 150 ms of the same samples whisper was given. It is self-limiting: on a + decaying tail — a word ending a phrase, where whisper is already right — + the quietest frame *is* the reported one and nothing moves. Boundaries + past the end of the decoded audio are left untouched. + +> Why this matters beyond caption timing: the transcript editor turns a word +> selection into a trim of exactly `[firstWord.startSec, lastWord.endSec]`, +> so a late boundary leaves the attack of the first removed word audible and +> bites into the following kept word. `LOOKBACK_SEC` in that module is the +> calibration knob — widen it and boundaries inside continuous speech start +> snapping onto the previous syllable's trough. The recognition call returns **both** the phrase segments and the per-word segments in one pass @@ -118,6 +136,7 @@ picked up as a usable model. | Module | Role | |---|---| | `electron/stt/whisperServer.ts` | Server lifecycle; `POST /inference` client; verbose_json parser. | +| `electron/stt/snapWordBoundaries.ts` | Re-anchors DTW word boundaries on the audio's RMS envelope (see step 5 above). | | `electron/stt/wav.ts` | WAV write + temp-file cleanup helpers. | | `electron/stt/gpuDetector.ts` | Per-platform binary resolver (no GPU probing). | | `electron/stt/modelManager.ts` | Single GGML file download, SHA-256 verify, atomic write. |