fix(stt): re-anchor whisper's word boundaries on the audio, so transcript trims cut where the words are

electron/stt/snapWordBoundaries.test.ts

@@ -0,0 +1,76 @@
+import { describe, expect, it } from "vitest";
+import { snapWordBoundariesToAudio } from "./snapWordBoundaries";
+import type { SttWordSegment } from "./transcriptionContract";
+
+const SAMPLE_RATE = 16_000;
+
+/** Mono 16 kHz buffer that is loud everywhere except the given silent spans. */
+function audioWithSilences(durationSec: number, silences: Array<[number, number]>): Float32Array {
+	const samples = new Float32Array(Math.round(durationSec * SAMPLE_RATE));
+	for (let i = 0; i < samples.length; i++) {
+		const t = i / SAMPLE_RATE;
+		const silent = silences.some(([from, to]) => t >= from && t < to);
+		// Alternating ±0.5 gives a flat, non-zero RMS without needing a real tone.
+		samples[i] = silent ? 0 : i % 2 === 0 ? 0.5 : -0.5;
+	}
+	return samples;
+}
+
+const word = (w: Partial<SttWordSegment> = {}): SttWordSegment => ({
+	word: "w",
+	startSec: 0,
+	endSec: 0.1,
+	...w,
+});
+
+describe("snapWordBoundariesToAudio", () => {
+	it("pulls a late boundary back into the silence that precedes it", () => {
+		// Speech stops at 1.0 and resumes at 1.2; whisper reports the next word
+		// starting at 1.3 — 100 ms after the audio actually resumed.
+		const samples = audioWithSilences(3, [[1.0, 1.2]]);
+		const [snapped] = snapWordBoundariesToAudio([word({ startSec: 1.3, endSec: 1.8 })], samples);
+		expect(snapped.startSec).toBeGreaterThanOrEqual(1.0);
+		expect(snapped.startSec).toBeLessThan(1.2);
+	});
+
+	it("leaves a boundary alone when nothing quieter precedes it", () => {
+		// A word ending a phrase: whisper is already right, the frames before the
+		// boundary are all speech, so the quietest frame in the window is the
+		// boundary itself and it must not drift.
+		const samples = audioWithSilences(3, [[1.5, 2.0]]);
+		const [snapped] = snapWordBoundariesToAudio([word({ startSec: 1.0, endSec: 1.5 })], samples);
+		expect(snapped.endSec).toBeCloseTo(1.5, 2);
+	});
+
+	it("never moves a boundary more than the lookback window", () => {
+		const samples = audioWithSilences(3, [[0.0, 1.0]]);
+		const [snapped] = snapWordBoundariesToAudio([word({ startSec: 2.0, endSec: 2.5 })], samples);
+		expect(snapped.startSec).toBeGreaterThanOrEqual(2.0 - 0.15);
+	});
+
+	it("keeps a boundary shared by two words shared", () => {
+		const samples = audioWithSilences(3, [[1.0, 1.2]]);
+		const [first, second] = snapWordBoundariesToAudio(
+			[word({ startSec: 0.5, endSec: 1.3 }), word({ startSec: 1.3, endSec: 1.8 })],
+			samples,
+		);
+		expect(first.endSec).toBeCloseTo(second.startSec, 6);
+	});
+
+	it("leaves boundaries that fall outside the decoded audio alone", () => {
+		// Clamping these into range would collapse every boundary onto the end of
+		// the buffer instead of leaving the unmeasurable ones untouched.
+		const samples = audioWithSilences(0.1, []);
+		const words = [word({ startSec: 5.51, endSec: 6.85 })];
+		expect(snapWordBoundariesToAudio(words, samples)).toEqual(words);
+	});
+
+	it("keeps degenerate words non-empty and passes words through without audio", () => {
+		const samples = audioWithSilences(3, [[1.0, 1.2]]);
+		const [degenerate] = snapWordBoundariesToAudio([word({ startSec: 1.3, endSec: 1.3 })], samples);
+		expect(degenerate.endSec).toBeGreaterThan(degenerate.startSec);
+
+		const untouched = [word({ startSec: 1.3, endSec: 1.8 })];
+		expect(snapWordBoundariesToAudio(untouched, new Float32Array(0))).toEqual(untouched);
+	});
+});

electron/stt/snapWordBoundaries.ts

@@ -0,0 +1,100 @@
+// Pulls whisper.cpp's DTW word boundaries back onto the audio they describe.
+//
+// whisper.cpp reports one time per token and derives word spans from it, so a
+// word's `start` is the point where the decoder *emitted* the token, not where
+// the speaker started saying it. Measured on real recordings, that lands
+// consistently 80–150 ms late, and because consecutive words share a boundary
+// (`word[i].end === word[i+1].start`) the whole transcript is dragged right by
+// roughly a syllable.
+//
+// That is invisible in captions but not in the transcript editor: deleting
+// words there turns the selection into a trim of exactly
+// `[firstWord.startSec, lastWord.endSec]`, so a late boundary leaves the attack
+// of the first removed word audible and bites into the following kept word.
+//
+// The correction is to look at the audio instead of guessing an offset: each
+// boundary moves back to the quietest 10 ms frame within the preceding
+// `LOOKBACK_SEC`. It is self-limiting — on a decaying tail (a word that ends a
+// phrase, where whisper is already right) the quietest frame IS the reported
+// one, so the boundary doesn't move at all.
+
+import type { SttWordSegment } from "./transcriptionContract";
+
+/** Matches `writeSamplesAsWav` — the samples handed to whisper are mono 16 kHz. */
+const SAMPLE_RATE = 16_000;
+
+/** RMS envelope resolution; finer than any boundary error worth correcting. */
+const FRAME_SEC = 0.01;
+
+/**
+ * How far back a boundary may travel — the calibration knob for this
+ * correction. Sized to the measured DTW lag (~80–150 ms). Widen it and
+ * boundaries inside continuous speech start snapping onto the *previous*
+ * syllable's trough; narrow it and the lag survives.
+ */
+const LOOKBACK_SEC = 0.15;
+
+/** Keep degenerate words (whisper sometimes reports end <= start) non-empty. */
+const MIN_WORD_SEC = 0.02;
+
+/** Per-frame RMS of the mono signal — the cheapest usable "is this speech" proxy. */
+function rmsEnvelope(samples: Float32Array): Float32Array {
+	const frameLength = Math.round(SAMPLE_RATE * FRAME_SEC);
+	const frameCount = Math.floor(samples.length / frameLength);
+	const envelope = new Float32Array(frameCount);
+	for (let f = 0; f < frameCount; f++) {
+		const start = f * frameLength;
+		let sum = 0;
+		for (let i = start; i < start + frameLength; i++) {
+			const v = samples[i] ?? 0;
+			sum += v * v;
+		}
+		envelope[f] = Math.sqrt(sum / frameLength);
+	}
+	return envelope;
+}
+
+/**
+ * Move every word boundary back to the quietest frame in the `LOOKBACK_SEC`
+ * preceding it. Boundaries shared by two words snap identically (same input
+ * time), so the transcript stays gap-free where whisper made it gap-free.
+ * Returns the words unchanged when there is no audio to measure against.
+ */
+export function snapWordBoundariesToAudio(
+	words: SttWordSegment[],
+	samples: Float32Array,
+): SttWordSegment[] {
+	const envelope = rmsEnvelope(samples);
+	if (envelope.length === 0) return words;
+
+	const lookbackFrames = Math.round(LOOKBACK_SEC / FRAME_SEC);
+	const snap = (timeSec: number): number => {
+		const hi = Math.round(timeSec / FRAME_SEC);
+		// A boundary outside the audio we can measure (whisper occasionally reports
+		// times past the end of the samples) has nothing to snap to — clamping it
+		// into range would drag it to the end of the buffer instead.
+		if (hi <= 0 || hi >= envelope.length) return timeSec;
+		const lo = Math.max(0, hi - lookbackFrames);
+		let bestFrame = hi;
+		let bestRms = envelope[hi];
+		// Strict `<` while walking backwards keeps the frame CLOSEST to the
+		// reported boundary when a whole stretch is equally quiet, so digital
+		// silence can't drag a boundary the full window.
+		for (let f = hi - 1; f >= lo; f--) {
+			if (envelope[f] < bestRms) {
+				bestRms = envelope[f];
+				bestFrame = f;
+			}
+		}
+		return bestFrame * FRAME_SEC;
+	};
+
+	return words.map((w) => {
+		const startSec = snap(w.startSec);
+		return {
+			...w,
+			startSec,
+			endSec: Math.max(snap(w.endSec), startSec + MIN_WORD_SEC),
+		};
+	});
+}

electron/stt/whisperServer.ts

@@ -7,6 +7,7 @@ import path from "node:path";
 import type { Readable } from "node:stream";
 
 import { resolveBinaryPath } from "./gpuDetector";
+import { snapWordBoundariesToAudio } from "./snapWordBoundaries";
 import type { SttBackend, SttPhraseSegment, SttWordSegment } from "./transcriptionContract";
 import { cleanupWav, writeSamplesAsWav } from "./wav";
 
@@ -335,17 +336,23 @@ export class WhisperServerManager {
 					return { text, startSec, endSec: Math.max(endSec, startSec + 0.05) };
 				})
 				.filter((s) => s.text.length > 0);
-			const wordSegments: SttWordSegment[] = raw
-				.flatMap((seg) =>
-					(seg.words ?? []).map((w) => {
-						const word = (w.word ?? "").trim();
-						const startSec = this.toSec(w.start, 0);
-						const endSec = this.toSec(w.end, startSec + 0.05);
-						const confidence = typeof w.probability === "number" ? w.probability : undefined;
-						return { word, startSec, endSec: Math.max(startSec + 0.02, endSec), confidence };
-					}),
-				)
-				.filter((w) => w.word.length > 0);
+			// whisper.cpp's DTW boundaries run ~80–150 ms behind the audio, which the
+			// transcript editor turns into imprecise trims (see snapWordBoundaries.ts).
+			// Re-anchor them on the same samples whisper was given.
+			const wordSegments: SttWordSegment[] = snapWordBoundariesToAudio(
+				raw
+					.flatMap((seg) =>
+						(seg.words ?? []).map((w) => {
+							const word = (w.word ?? "").trim();
+							const startSec = this.toSec(w.start, 0);
+							const endSec = this.toSec(w.end, startSec + 0.05);
+							const confidence = typeof w.probability === "number" ? w.probability : undefined;
+							return { word, startSec, endSec: Math.max(startSec + 0.02, endSec), confidence };
+						}),
+					)
+					.filter((w) => w.word.length > 0),
+				opts.samples,
+			);
 			const detectedLanguage = json.detected_language ?? json.language ?? "auto";
 			const backend = this.toBackend(json.backend);
 			return { segments, wordSegments, detectedLanguage, backend };

technical-documentation/architecture/transcription-and-captions.md

@@ -84,6 +84,24 @@ it verbatim in the response.
    `word.end = t_dtw of the next word's first token` (or the segment's `t1`
    for the segment's last word). The result is a monotonic, gap-free timeline
    of word ranges that downstream code can use without rebasing.
+5. **Re-anchor on the audio** — `t_dtw` marks where the decoder *emitted* a
+   token, not where the speaker started saying it, so every boundary lands
+   80–150 ms late (measured across real recordings; the mean correction on
+   the reference clip is 83 ms). Because consecutive words share a boundary,
+   the whole transcript is dragged right by roughly a syllable.
+   [`electron/stt/snapWordBoundaries.ts`](../../electron/stt/snapWordBoundaries.ts)
+   pulls each boundary back to the quietest 10 ms frame in the preceding
+   150 ms of the same samples whisper was given. It is self-limiting: on a
+   decaying tail — a word ending a phrase, where whisper is already right —
+   the quietest frame *is* the reported one and nothing moves. Boundaries
+   past the end of the decoded audio are left untouched.
+
+> Why this matters beyond caption timing: the transcript editor turns a word
+> selection into a trim of exactly `[firstWord.startSec, lastWord.endSec]`,
+> so a late boundary leaves the attack of the first removed word audible and
+> bites into the following kept word. `LOOKBACK_SEC` in that module is the
+> calibration knob — widen it and boundaries inside continuous speech start
+> snapping onto the previous syllable's trough.
 
 The recognition call returns **both** the phrase segments and the per-word
 segments in one pass
@@ -118,6 +136,7 @@ picked up as a usable model.
 | Module | Role |
 |---|---|
 | `electron/stt/whisperServer.ts` | Server lifecycle; `POST /inference` client; verbose_json parser. |
+| `electron/stt/snapWordBoundaries.ts` | Re-anchors DTW word boundaries on the audio's RMS envelope (see step 5 above). |
 | `electron/stt/wav.ts` | WAV write + temp-file cleanup helpers. |
 | `electron/stt/gpuDetector.ts` | Per-platform binary resolver (no GPU probing). |
 | `electron/stt/modelManager.ts` | Single GGML file download, SHA-256 verify, atomic write. |