From ffd639dd931329ca59791f101833e45d9917a3cb Mon Sep 17 00:00:00 2001 From: Etienne Lescot Date: Mon, 27 Jul 2026 12:24:29 +0200 Subject: [PATCH] fix(stt): re-anchor whisper's word boundaries on the audio, so transcript trims cut where the words are MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deleting words in the transcript editor produced a trim whose edges were audibly wrong every time: the attack of the first removed word stayed in, and the cut bit into the following kept word. The editor is not at fault - it turns a selection into exactly `[firstWord.startSec, lastWord.endSec]`. The word times are. whisper.cpp's `t_dtw` marks where the decoder *emitted* a token, not where the speaker started saying it, and since `word.end` is the next word's `t_dtw`, every boundary in the transcript is late by the same amount. The whole thing is dragged right by roughly a syllable. Measured on a real recording (33.6 s of French, `ggml-small-q8_0`) by comparing the stored word times against the signal's own RMS envelope: the lag is 80-150 ms, mean 83 ms. Two independent post-pause onsets confirm it - "Alors" reported at 11.96 while the audio resumes at 11.78, "Voilà" at 25.04 against 24.85 - and phrase-*ends* are already correct (11.20 against 11.23), so a constant offset would have traded one artifact for another. So measure instead of guessing: each boundary moves back to the quietest 10 ms frame within the preceding 150 ms of the same samples whisper was given. That is self-limiting - on a decaying tail the quietest frame IS the reported one, so the boundaries whisper already got right do not move. The correction is applied in the server adapter, upstream of everything, so trims, caption cues and the transcript pane's playhead highlight are all fixed in one place. On the trim that prompted the report ("vraiment il est bien"), 21.94-22.68 becomes 21.80-22.55; the audio has the phrase at 21.86-22.56. The post-pause onsets land at 11.82 and 24.89 against 11.78 and 24.85, and the two phrase-ends stay put. One real bug came out of the existing whisperServer tests while writing this: clamping an out-of-range boundary into the envelope dragged it onto the end of the buffer. A boundary past the decoded audio now stays untouched - there is nothing to measure it against. Existing transcripts keep their old times; re-run transcription on a project to pick the correction up. Phrase segments (the fallback whisper only returns when it reports no words at all) are deliberately left alone. --- electron/stt/snapWordBoundaries.test.ts | 76 +++++++++++++ electron/stt/snapWordBoundaries.ts | 100 ++++++++++++++++++ electron/stt/whisperServer.ts | 29 +++-- .../transcription-and-captions.md | 19 ++++ 4 files changed, 213 insertions(+), 11 deletions(-) create mode 100644 electron/stt/snapWordBoundaries.test.ts create mode 100644 electron/stt/snapWordBoundaries.ts diff --git a/electron/stt/snapWordBoundaries.test.ts b/electron/stt/snapWordBoundaries.test.ts new file mode 100644 index 000000000..2668c6966 --- /dev/null +++ b/electron/stt/snapWordBoundaries.test.ts @@ -0,0 +1,76 @@ +import { describe, expect, it } from "vitest"; +import { snapWordBoundariesToAudio } from "./snapWordBoundaries"; +import type { SttWordSegment } from "./transcriptionContract"; + +const SAMPLE_RATE = 16_000; + +/** Mono 16 kHz buffer that is loud everywhere except the given silent spans. */ +function audioWithSilences(durationSec: number, silences: Array<[number, number]>): Float32Array { + const samples = new Float32Array(Math.round(durationSec * SAMPLE_RATE)); + for (let i = 0; i < samples.length; i++) { + const t = i / SAMPLE_RATE; + const silent = silences.some(([from, to]) => t >= from && t < to); + // Alternating ±0.5 gives a flat, non-zero RMS without needing a real tone. + samples[i] = silent ? 0 : i % 2 === 0 ? 0.5 : -0.5; + } + return samples; +} + +const word = (w: Partial = {}): SttWordSegment => ({ + word: "w", + startSec: 0, + endSec: 0.1, + ...w, +}); + +describe("snapWordBoundariesToAudio", () => { + it("pulls a late boundary back into the silence that precedes it", () => { + // Speech stops at 1.0 and resumes at 1.2; whisper reports the next word + // starting at 1.3 — 100 ms after the audio actually resumed. + const samples = audioWithSilences(3, [[1.0, 1.2]]); + const [snapped] = snapWordBoundariesToAudio([word({ startSec: 1.3, endSec: 1.8 })], samples); + expect(snapped.startSec).toBeGreaterThanOrEqual(1.0); + expect(snapped.startSec).toBeLessThan(1.2); + }); + + it("leaves a boundary alone when nothing quieter precedes it", () => { + // A word ending a phrase: whisper is already right, the frames before the + // boundary are all speech, so the quietest frame in the window is the + // boundary itself and it must not drift. + const samples = audioWithSilences(3, [[1.5, 2.0]]); + const [snapped] = snapWordBoundariesToAudio([word({ startSec: 1.0, endSec: 1.5 })], samples); + expect(snapped.endSec).toBeCloseTo(1.5, 2); + }); + + it("never moves a boundary more than the lookback window", () => { + const samples = audioWithSilences(3, [[0.0, 1.0]]); + const [snapped] = snapWordBoundariesToAudio([word({ startSec: 2.0, endSec: 2.5 })], samples); + expect(snapped.startSec).toBeGreaterThanOrEqual(2.0 - 0.15); + }); + + it("keeps a boundary shared by two words shared", () => { + const samples = audioWithSilences(3, [[1.0, 1.2]]); + const [first, second] = snapWordBoundariesToAudio( + [word({ startSec: 0.5, endSec: 1.3 }), word({ startSec: 1.3, endSec: 1.8 })], + samples, + ); + expect(first.endSec).toBeCloseTo(second.startSec, 6); + }); + + it("leaves boundaries that fall outside the decoded audio alone", () => { + // Clamping these into range would collapse every boundary onto the end of + // the buffer instead of leaving the unmeasurable ones untouched. + const samples = audioWithSilences(0.1, []); + const words = [word({ startSec: 5.51, endSec: 6.85 })]; + expect(snapWordBoundariesToAudio(words, samples)).toEqual(words); + }); + + it("keeps degenerate words non-empty and passes words through without audio", () => { + const samples = audioWithSilences(3, [[1.0, 1.2]]); + const [degenerate] = snapWordBoundariesToAudio([word({ startSec: 1.3, endSec: 1.3 })], samples); + expect(degenerate.endSec).toBeGreaterThan(degenerate.startSec); + + const untouched = [word({ startSec: 1.3, endSec: 1.8 })]; + expect(snapWordBoundariesToAudio(untouched, new Float32Array(0))).toEqual(untouched); + }); +}); diff --git a/electron/stt/snapWordBoundaries.ts b/electron/stt/snapWordBoundaries.ts new file mode 100644 index 000000000..d5d0b6f10 --- /dev/null +++ b/electron/stt/snapWordBoundaries.ts @@ -0,0 +1,100 @@ +// Pulls whisper.cpp's DTW word boundaries back onto the audio they describe. +// +// whisper.cpp reports one time per token and derives word spans from it, so a +// word's `start` is the point where the decoder *emitted* the token, not where +// the speaker started saying it. Measured on real recordings, that lands +// consistently 80–150 ms late, and because consecutive words share a boundary +// (`word[i].end === word[i+1].start`) the whole transcript is dragged right by +// roughly a syllable. +// +// That is invisible in captions but not in the transcript editor: deleting +// words there turns the selection into a trim of exactly +// `[firstWord.startSec, lastWord.endSec]`, so a late boundary leaves the attack +// of the first removed word audible and bites into the following kept word. +// +// The correction is to look at the audio instead of guessing an offset: each +// boundary moves back to the quietest 10 ms frame within the preceding +// `LOOKBACK_SEC`. It is self-limiting — on a decaying tail (a word that ends a +// phrase, where whisper is already right) the quietest frame IS the reported +// one, so the boundary doesn't move at all. + +import type { SttWordSegment } from "./transcriptionContract"; + +/** Matches `writeSamplesAsWav` — the samples handed to whisper are mono 16 kHz. */ +const SAMPLE_RATE = 16_000; + +/** RMS envelope resolution; finer than any boundary error worth correcting. */ +const FRAME_SEC = 0.01; + +/** + * How far back a boundary may travel — the calibration knob for this + * correction. Sized to the measured DTW lag (~80–150 ms). Widen it and + * boundaries inside continuous speech start snapping onto the *previous* + * syllable's trough; narrow it and the lag survives. + */ +const LOOKBACK_SEC = 0.15; + +/** Keep degenerate words (whisper sometimes reports end <= start) non-empty. */ +const MIN_WORD_SEC = 0.02; + +/** Per-frame RMS of the mono signal — the cheapest usable "is this speech" proxy. */ +function rmsEnvelope(samples: Float32Array): Float32Array { + const frameLength = Math.round(SAMPLE_RATE * FRAME_SEC); + const frameCount = Math.floor(samples.length / frameLength); + const envelope = new Float32Array(frameCount); + for (let f = 0; f < frameCount; f++) { + const start = f * frameLength; + let sum = 0; + for (let i = start; i < start + frameLength; i++) { + const v = samples[i] ?? 0; + sum += v * v; + } + envelope[f] = Math.sqrt(sum / frameLength); + } + return envelope; +} + +/** + * Move every word boundary back to the quietest frame in the `LOOKBACK_SEC` + * preceding it. Boundaries shared by two words snap identically (same input + * time), so the transcript stays gap-free where whisper made it gap-free. + * Returns the words unchanged when there is no audio to measure against. + */ +export function snapWordBoundariesToAudio( + words: SttWordSegment[], + samples: Float32Array, +): SttWordSegment[] { + const envelope = rmsEnvelope(samples); + if (envelope.length === 0) return words; + + const lookbackFrames = Math.round(LOOKBACK_SEC / FRAME_SEC); + const snap = (timeSec: number): number => { + const hi = Math.round(timeSec / FRAME_SEC); + // A boundary outside the audio we can measure (whisper occasionally reports + // times past the end of the samples) has nothing to snap to — clamping it + // into range would drag it to the end of the buffer instead. + if (hi <= 0 || hi >= envelope.length) return timeSec; + const lo = Math.max(0, hi - lookbackFrames); + let bestFrame = hi; + let bestRms = envelope[hi]; + // Strict `<` while walking backwards keeps the frame CLOSEST to the + // reported boundary when a whole stretch is equally quiet, so digital + // silence can't drag a boundary the full window. + for (let f = hi - 1; f >= lo; f--) { + if (envelope[f] < bestRms) { + bestRms = envelope[f]; + bestFrame = f; + } + } + return bestFrame * FRAME_SEC; + }; + + return words.map((w) => { + const startSec = snap(w.startSec); + return { + ...w, + startSec, + endSec: Math.max(snap(w.endSec), startSec + MIN_WORD_SEC), + }; + }); +} diff --git a/electron/stt/whisperServer.ts b/electron/stt/whisperServer.ts index 7c846555d..4762ebf6d 100644 --- a/electron/stt/whisperServer.ts +++ b/electron/stt/whisperServer.ts @@ -7,6 +7,7 @@ import path from "node:path"; import type { Readable } from "node:stream"; import { resolveBinaryPath } from "./gpuDetector"; +import { snapWordBoundariesToAudio } from "./snapWordBoundaries"; import type { SttBackend, SttPhraseSegment, SttWordSegment } from "./transcriptionContract"; import { cleanupWav, writeSamplesAsWav } from "./wav"; @@ -335,17 +336,23 @@ export class WhisperServerManager { return { text, startSec, endSec: Math.max(endSec, startSec + 0.05) }; }) .filter((s) => s.text.length > 0); - const wordSegments: SttWordSegment[] = raw - .flatMap((seg) => - (seg.words ?? []).map((w) => { - const word = (w.word ?? "").trim(); - const startSec = this.toSec(w.start, 0); - const endSec = this.toSec(w.end, startSec + 0.05); - const confidence = typeof w.probability === "number" ? w.probability : undefined; - return { word, startSec, endSec: Math.max(startSec + 0.02, endSec), confidence }; - }), - ) - .filter((w) => w.word.length > 0); + // whisper.cpp's DTW boundaries run ~80–150 ms behind the audio, which the + // transcript editor turns into imprecise trims (see snapWordBoundaries.ts). + // Re-anchor them on the same samples whisper was given. + const wordSegments: SttWordSegment[] = snapWordBoundariesToAudio( + raw + .flatMap((seg) => + (seg.words ?? []).map((w) => { + const word = (w.word ?? "").trim(); + const startSec = this.toSec(w.start, 0); + const endSec = this.toSec(w.end, startSec + 0.05); + const confidence = typeof w.probability === "number" ? w.probability : undefined; + return { word, startSec, endSec: Math.max(startSec + 0.02, endSec), confidence }; + }), + ) + .filter((w) => w.word.length > 0), + opts.samples, + ); const detectedLanguage = json.detected_language ?? json.language ?? "auto"; const backend = this.toBackend(json.backend); return { segments, wordSegments, detectedLanguage, backend }; diff --git a/technical-documentation/architecture/transcription-and-captions.md b/technical-documentation/architecture/transcription-and-captions.md index c6a0fe5e1..77962e341 100644 --- a/technical-documentation/architecture/transcription-and-captions.md +++ b/technical-documentation/architecture/transcription-and-captions.md @@ -84,6 +84,24 @@ it verbatim in the response. `word.end = t_dtw of the next word's first token` (or the segment's `t1` for the segment's last word). The result is a monotonic, gap-free timeline of word ranges that downstream code can use without rebasing. +5. **Re-anchor on the audio** — `t_dtw` marks where the decoder *emitted* a + token, not where the speaker started saying it, so every boundary lands + 80–150 ms late (measured across real recordings; the mean correction on + the reference clip is 83 ms). Because consecutive words share a boundary, + the whole transcript is dragged right by roughly a syllable. + [`electron/stt/snapWordBoundaries.ts`](../../electron/stt/snapWordBoundaries.ts) + pulls each boundary back to the quietest 10 ms frame in the preceding + 150 ms of the same samples whisper was given. It is self-limiting: on a + decaying tail — a word ending a phrase, where whisper is already right — + the quietest frame *is* the reported one and nothing moves. Boundaries + past the end of the decoded audio are left untouched. + +> Why this matters beyond caption timing: the transcript editor turns a word +> selection into a trim of exactly `[firstWord.startSec, lastWord.endSec]`, +> so a late boundary leaves the attack of the first removed word audible and +> bites into the following kept word. `LOOKBACK_SEC` in that module is the +> calibration knob — widen it and boundaries inside continuous speech start +> snapping onto the previous syllable's trough. The recognition call returns **both** the phrase segments and the per-word segments in one pass @@ -118,6 +136,7 @@ picked up as a usable model. | Module | Role | |---|---| | `electron/stt/whisperServer.ts` | Server lifecycle; `POST /inference` client; verbose_json parser. | +| `electron/stt/snapWordBoundaries.ts` | Re-anchors DTW word boundaries on the audio's RMS envelope (see step 5 above). | | `electron/stt/wav.ts` | WAV write + temp-file cleanup helpers. | | `electron/stt/gpuDetector.ts` | Per-platform binary resolver (no GPU probing). | | `electron/stt/modelManager.ts` | Single GGML file download, SHA-256 verify, atomic write. |