daily-cache: never finalize history against a degraded session parse

src/daily-cache.ts

@@ -150,6 +150,14 @@ export type DailyCache = {
   /// as incomplete and is fully re-backfilled. Absent on caches written before
   /// this field existed → treated as incomplete (one self-healing re-backfill).
   complete?: boolean
+  /// True once a COMPLETE parse finalized this watermark. The pull-back below
+  /// only distrusts caches WITHOUT this stamp: a degraded parse can no longer
+  /// set `complete`, so a stamped cache whose watermark sits past its newest
+  /// populated day is a legitimately idle tail (recent days had no activity),
+  /// not a frozen hole, and re-deriving it every launch is pure waste. Absent
+  /// on caches written before this field: distrusted once (one healing
+  /// pull-back), then stamped.
+  watermarkTrusted?: boolean
 }
 
 function getCacheDir(): string {
@@ -294,7 +302,7 @@ function migrateDays(days: Record<string, unknown>[]): DailyEntry[] {
     }))
 }
 
-function migratedFrom(parsed: { version: number; lastComputedDate: string | null; savingsConfigHash?: string; tzKey?: string; days: Record<string, unknown>[]; complete?: boolean }): DailyCache {
+function migratedFrom(parsed: { version: number; lastComputedDate: string | null; savingsConfigHash?: string; tzKey?: string; days: Record<string, unknown>[]; complete?: boolean; watermarkTrusted?: boolean }): DailyCache {
   return {
     version: DAILY_CACHE_VERSION,
     savingsConfigHash: parsed.savingsConfigHash ?? '',
@@ -306,6 +314,9 @@ function migratedFrom(parsed: { version: number; lastComputedDate: string | null
     // Only a cache explicitly marked complete stays trusted; one written before
     // the marker existed reads false and is re-backfilled once.
     complete: parsed.complete === true,
+    // Absent on a pre-fix cache: the watermark is distrusted once (healing
+    // pull-back), then re-stamped by the finalize that follows.
+    watermarkTrusted: parsed.watermarkTrusted === true,
   }
 }
 
@@ -409,6 +420,7 @@ async function adoptOlderDailyCaches(): Promise<DailyCache> {
     // accounting: leave complete unset so the next hydration re-derives every
     // day whose sources survive (the merge keeps the rest).
     complete: rest.length === candidates.length ? false : base.complete,
+    watermarkTrusted: rest.length === candidates.length ? false : base.watermarkTrusted,
   }
   await saveDailyCache(adopted).catch(() => {})
   return adopted
@@ -451,6 +463,7 @@ export function addNewDays(cache: DailyCache, incoming: DailyEntry[], newestDate
     lastComputedDate: nextLast,
     days: applyRetention(merged, newestDate),
     complete: cache.complete,
+    watermarkTrusted: cache.watermarkTrusted,
   }
 }
 
@@ -671,6 +684,27 @@ export async function ensureCacheHydrated(
       c = { ...c, days: freshDays, lastComputedDate: latestFresh }
     }
 
+    // A cache can claim `complete` while its watermark points PAST its newest
+    // populated day — what a run finalizing off a degraded (read-only) parse
+    // leaves behind: it advanced lastComputedDate over days the parse never
+    // covered. Since gapStart is lastComputedDate + 1, that hole is invisible
+    // to the gap logic forever. Trust the DATA over the marker: pull the
+    // watermark back to the newest day actually present so the ordinary gap
+    // parse re-derives the tail. Nothing is dropped — the cached days all stay.
+    //
+    // Only UNSTAMPED caches are distrusted here. A degraded parse can no longer
+    // set `complete` (that is this fix), so the corrupt state can only be
+    // written by pre-fix code: an unstamped cache. A stamped one whose watermark
+    // outruns its newest day is a legitimately idle tail (recent days had no
+    // activity), and re-deriving that empty tail on every launch is the
+    // regression this guard avoids. A cache with NO days is exempt: it has no
+    // newest day to trust, and a machine with no history at all must still be
+    // able to finalize (below) rather than re-backfill on every launch.
+    const newestCachedDate = c.days.reduce<string | null>((max, d) => (max === null || d.date > max ? d.date : max), null)
+    if (c.watermarkTrusted !== true && newestCachedDate !== null && c.lastComputedDate !== null && c.lastComputedDate > newestCachedDate) {
+      c = { ...c, lastComputedDate: newestCachedDate }
+    }
+
     // Three reasons to re-derive the whole retention window:
     //  1. Savings config changed — cached `savingsUSD` totals are stale.
     //  2. The cache was never finalized against a COMPLETE session parse (an old
@@ -691,6 +725,7 @@ export async function ensureCacheHydrated(
     const tzChanged = c.tzKey !== undefined && c.tzKey !== tzKey
     if (c.savingsConfigHash !== savingsConfigHash || c.complete !== true || tzChanged) {
       const baseline = c.days
+      const priorWatermark = c.lastComputedDate
       const backfillStart = new Date(now.getFullYear(), now.getMonth(), now.getDate() - BACKFILL_DAYS)
       let freshDays: DailyEntry[] = []
       if (backfillStart.getTime() <= yesterdayEnd.getTime()) {
@@ -708,9 +743,18 @@ export async function ensureCacheHydrated(
         version: DAILY_CACHE_VERSION,
         savingsConfigHash,
         tzKey,
-        lastComputedDate: yesterdayStr,
+        // The watermark records how far history has actually been derived, so
+        // only a COMPLETE parse may advance it. A partial one produced no data
+        // for whatever it could not read; moving the watermark to yesterday
+        // anyway would place those days behind the next run's gapStart and
+        // freeze the hole in (retention still anchors on yesterdayStr — the
+        // real calendar edge — so holding the watermark can't evict anything).
+        lastComputedDate: parseWasComplete ? yesterdayStr : priorWatermark,
         days: applyRetention(merged, yesterdayStr),
         complete: parseWasComplete,
+        // Stamp the watermark as trusted only when a COMPLETE parse produced it,
+        // so a later idle tail under this watermark is not distrusted above.
+        watermarkTrusted: parseWasComplete,
       }
       await saveDailyCache(c)
       return c
@@ -733,18 +777,23 @@ export async function ensureCacheHydrated(
       const gapRange: DateRange = { start: gapStart, end: yesterdayEnd }
       const gapProjects = await parseSessions(gapRange)
       const gapDays = aggregateDays(gapProjects)
+      const parseWasComplete = sessionComplete()
+      const priorWatermark = c.lastComputedDate
       c = addNewDays(c, gapDays, yesterdayStr)
       // Finalize as complete ONLY when the session parse that produced these days
       // was itself complete. If it was partial, leave `complete: false` so the
       // next launch (once the session cache is whole) re-backfills instead of
-      // freezing the partial history.
-      c = { ...c, complete: sessionComplete() }
+      // freezing the partial history — and hold the watermark where it was, for
+      // the same reason as the re-derive path above: a partial parse cannot
+      // vouch for the days it never read, and gapStart is the only thing that
+      // will ever bring them back.
+      c = { ...c, lastComputedDate: parseWasComplete ? c.lastComputedDate : priorWatermark, complete: parseWasComplete, watermarkTrusted: parseWasComplete }
       await saveDailyCache(c)
     } else if (c.complete !== true && sessionComplete()) {
       // No gap to fill (already current through yesterday) but not yet marked —
       // e.g. a brand-new machine whose only data is today. Finalize so future
       // launches don't re-backfill the whole window every time.
-      c = { ...c, complete: true }
+      c = { ...c, complete: true, watermarkTrusted: true }
       await saveDailyCache(c)
     }
     return c

src/parser.ts

@@ -1936,6 +1936,7 @@ async function scanProjectDirs(
       const cached = section.files[filePath]
       const action = reconcileFile(fp, cached)
       if (cached && (readOnly || action.action === 'unchanged')) {
+        if (readOnly && action.action !== 'unchanged') readOnlyServedStale = true
         unchangedFiles.push({ filePath, dirName, source, cached: section.files[filePath]! })
       } else if (!readOnly) {
         if (action.action === 'appended') {
@@ -1947,6 +1948,10 @@ async function scanProjectDirs(
           continue
         }
         changedFiles.push({ filePath, info: { dirName, fp, source } })
+      } else {
+        // Read-only with no cache entry at all: this file is dropped from what
+        // we serve, so the snapshot under-reports whatever days it covers.
+        readOnlyServedStale = true
       }
     }
     dirsDone++
@@ -2840,9 +2845,13 @@ async function parseProviderSources(
     // re-read a file that already threw and hasn't changed. It re-parses only
     // when the file changes (then `reconcileFile` reports non-'unchanged').
     if (cached && (readOnly || (action.action === 'unchanged' && (cached.failed || !cachedFileNeedsProviderReparse(providerName, source.path, cached))))) {
+      if (readOnly && action.action !== 'unchanged') readOnlyServedStale = true
       unchangedSources.push({ source, cached })
     } else if (!readOnly) {
       changedSources.push({ source, fp })
+    } else {
+      // Read-only with no cache entry at all — see scanProjectDirs.
+      readOnlyServedStale = true
     }
   }
 
@@ -3539,6 +3548,15 @@ export function isSessionHydrationComplete(): boolean {
   return sessionHydrationComplete
 }
 
+// Set by the read-only serving paths when the snapshot they served did NOT
+// match what is on disk: in read-only mode a changed file is served at its
+// stale fingerprint and a file with no cache entry is skipped entirely. A
+// read-only run under which nothing changed is equivalent to a full parse and
+// stays trustworthy; one that skipped real data is a PARTIAL hydration, and
+// finalizing daily history off it freezes the days it never saw out of the
+// chart (gapStart = lastComputedDate + 1 never looks back at them).
+let readOnlyServedStale = false
+
 export async function parseAllSessions(dateRange?: DateRange, providerFilter?: string): Promise<ProjectSummary[]> {
   const key = cacheKey(dateRange, providerFilter)
   const cached = sessionCache.get(key)
@@ -3608,6 +3626,7 @@ async function runParse(
   options: RunParseOptions = {},
 ): Promise<ProjectSummary[]> {
   const { isCold = false, readOnly = false, refreshLock } = options
+  readOnlyServedStale = false
   const seenMsgIds = new Set<string>()
   const seenKeys = new Set<string>()
   const allSources = await discoverAllSessions(providerFilter)
@@ -3712,7 +3731,10 @@ async function runParse(
       if (refreshLock) throw new RefreshPublicationUnavailableError()
     }
   }
-  sessionHydrationComplete = true
+  // Assigned, not forced true: a read-only run that had to skip or stale real
+  // files reached the end of the scan without hydrating everything, and the
+  // daily backfill must not finalize history off it.
+  sessionHydrationComplete = !readOnly || !readOnlyServedStale
 
   // Merge across providers by normalised project path so the same repository
   // is not double-counted when it was worked on with more than one tool

tests/daily-cache-carry-forward.test.ts

@@ -609,6 +609,7 @@ describe('adoption union across older cache files', () => {
       lastComputedDate: daysAgoStr(1),
       days: [rich],
       complete: true,
+      watermarkTrusted: true,
     }
     await saveDailyCache(cache)
     const loaded = await loadDailyCache()

tests/daily-cache-degraded-completeness.test.ts

@@ -0,0 +1,187 @@
+import { afterEach, beforeEach, describe, expect, it } from 'vitest'
+import { mkdir, rm } from 'fs/promises'
+import { existsSync } from 'fs'
+import { tmpdir } from 'os'
+import { join } from 'path'
+
+import type { DateRange, ProjectSummary } from '../src/types.js'
+
+import {
+  DAILY_CACHE_VERSION,
+  type DailyCache,
+  type DailyEntry,
+  type ProviderDaySlice,
+  currentTzKey,
+  ensureCacheHydrated,
+  saveDailyCache,
+} from '../src/daily-cache.js'
+
+const TMP_CACHE_ROOT = join(tmpdir(), `codeburn-degraded-test-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`)
+
+beforeEach(async () => {
+  process.env['CODEBURN_CACHE_DIR'] = TMP_CACHE_ROOT
+  await mkdir(TMP_CACHE_ROOT, { recursive: true })
+})
+
+afterEach(async () => {
+  if (existsSync(TMP_CACHE_ROOT)) {
+    await rm(TMP_CACHE_ROOT, { recursive: true, force: true })
+  }
+})
+
+function slice(cost: number, calls: number, extra: Partial<ProviderDaySlice> = {}): ProviderDaySlice {
+  return { cost, calls, savingsUSD: 0, ...extra }
+}
+
+function day(date: string, providers: Record<string, ProviderDaySlice>, overrides: Partial<DailyEntry> = {}): DailyEntry {
+  const cost = Object.values(providers).reduce((s, p) => s + p.cost, 0)
+  const calls = Object.values(providers).reduce((s, p) => s + p.calls, 0)
+  return {
+    date,
+    cost,
+    savingsUSD: 0,
+    calls,
+    sessions: 0,
+    inputTokens: 0,
+    outputTokens: 0,
+    cacheReadTokens: 0,
+    cacheWriteTokens: 0,
+    editTurns: 0,
+    oneShotTurns: 0,
+    models: {},
+    categories: {},
+    providers,
+    ...overrides,
+  }
+}
+
+function daysAgoStr(n: number): string {
+  const d = new Date()
+  d.setDate(d.getDate() - n)
+  return `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, '0')}-${String(d.getDate()).padStart(2, '0')}`
+}
+
+const noSessions = async (): Promise<ProjectSummary[]> => []
+
+/// The day whose session files are long gone: it exists in the daily cache and
+/// nowhere else, so every path below must still hand it back untouched.
+const VANISHED = day(daysAgoStr(40), { claude: slice(399.70, 1572) }, { carried: true })
+
+async function seed(overrides: Partial<DailyCache> = {}): Promise<void> {
+  await saveDailyCache({
+    version: DAILY_CACHE_VERSION,
+    savingsConfigHash: 'cfg-A',
+    tzKey: currentTzKey(),
+    lastComputedDate: daysAgoStr(4),
+    days: [VANISHED, day(daysAgoStr(4), { claude: slice(120, 900) })],
+    complete: true,
+    ...overrides,
+  })
+}
+
+/** The vanished-sources day is still there, with its original accounting. */
+function expectPreserved(cache: DailyCache): void {
+  const kept = cache.days.find(d => d.date === VANISHED.date)
+  expect(kept).toMatchObject({ cost: 399.70, calls: 1572 })
+  expect(kept!.providers['claude']!.cost).toBe(399.70)
+}
+
+describe('daily cache: a degraded session parse never finalizes history', () => {
+  it('does not publish complete, and does not advance the watermark past what it covered', async () => {
+    await seed()
+    const out = await ensureCacheHydrated(noSessions, () => [], 'cfg-A', () => false)
+    // The parse covered nothing it can vouch for, so the watermark stays put:
+    // advancing it to yesterday would put the missed days behind gapStart
+    // (lastComputedDate + 1) forever.
+    expect(out.lastComputedDate).toBe(daysAgoStr(4))
+    expect(out.complete).toBe(false)
+    expectPreserved(out)
+  })
+
+  it('does not advance the watermark on the full re-derive path either', async () => {
+    await seed({ complete: false })
+    const out = await ensureCacheHydrated(noSessions, () => [], 'cfg-A', () => false)
+    expect(out.lastComputedDate).toBe(daysAgoStr(4))
+    expect(out.complete).toBe(false)
+    expectPreserved(out)
+  })
+
+  it('a later healthy run rebuilds the days the degraded run missed', async () => {
+    await seed()
+    await ensureCacheHydrated(noSessions, () => [], 'cfg-A', () => false)
+    const missed = [1, 2, 3].map(n => day(daysAgoStr(n), { claude: slice(n * 10, n * 100) }))
+    const healed = await ensureCacheHydrated(noSessions, () => missed, 'cfg-A', () => true)
+    expect(healed.days.map(d => d.date)).toEqual([
+      daysAgoStr(40), daysAgoStr(4), daysAgoStr(3), daysAgoStr(2), daysAgoStr(1),
+    ])
+    expect(healed.lastComputedDate).toBe(daysAgoStr(1))
+    expect(healed.complete).toBe(true)
+    expectPreserved(healed)
+  })
+})
+
+describe('daily cache: a complete cache that outruns its own data is not trusted', () => {
+  it('re-derives the days between the newest entry and the watermark', async () => {
+    // The field artifact: complete: true, lastComputedDate yesterday, entries
+    // stopping four days earlier — written by a run that finalized off a parse
+    // which never covered those days.
+    await seed({ lastComputedDate: daysAgoStr(1) })
+    const ranges: DateRange[] = []
+    const missed = [1, 2, 3].map(n => day(daysAgoStr(n), { claude: slice(n * 10, n * 100) }))
+    const out = await ensureCacheHydrated(
+      async (range) => { ranges.push(range); return [] },
+      () => missed,
+      'cfg-A',
+      () => true,
+    )
+    expect(ranges).toHaveLength(1)
+    expect(out.days.map(d => d.date)).toEqual([
+      daysAgoStr(40), daysAgoStr(4), daysAgoStr(3), daysAgoStr(2), daysAgoStr(1),
+    ])
+    expect(out.days.find(d => d.date === daysAgoStr(2))!.cost).toBe(20)
+    expect(out.complete).toBe(true)
+    expectPreserved(out)
+  })
+
+  it('trusts a stamped watermark over an idle tail — no re-derive treadmill', async () => {
+    // Same shape as the corrupt case above (watermark past the newest populated
+    // day), but stamped by a COMPLETE parse: the recent days are genuinely
+    // empty, not a frozen hole. A degraded parse can no longer produce this
+    // state, so the stamp means the watermark is trustworthy and re-deriving the
+    // empty tail on every launch (the perf regression) must not happen.
+    await seed({ lastComputedDate: daysAgoStr(1), watermarkTrusted: true })
+    let parses = 0
+    const out = await ensureCacheHydrated(
+      async () => { parses += 1; return [] },
+      () => [],
+      'cfg-A',
+      () => true,
+    )
+    expect(parses).toBe(0)
+    expect(out.lastComputedDate).toBe(daysAgoStr(1))
+    expect(out.complete).toBe(true)
+    expectPreserved(out)
+  })
+
+  it('a degraded re-derivation of those days still keeps every carried day', async () => {
+    await seed({ lastComputedDate: daysAgoStr(1) })
+    const out = await ensureCacheHydrated(noSessions, () => [], 'cfg-A', () => false)
+    expect(out.complete).toBe(false)
+    expect(out.days.map(d => d.date)).toEqual([daysAgoStr(40), daysAgoStr(4)])
+    expectPreserved(out)
+  })
+
+  it('an empty cache still finalizes — no re-parse treadmill on a machine with no history', async () => {
+    await seed({ days: [], lastComputedDate: daysAgoStr(1) })
+    let parses = 0
+    const out = await ensureCacheHydrated(
+      async () => { parses += 1; return [] },
+      () => [],
+      'cfg-A',
+      () => true,
+    )
+    expect(parses).toBe(0)
+    expect(out.lastComputedDate).toBe(daysAgoStr(1))
+    expect(out.complete).toBe(true)
+  })
+})