From 628eada21c1bcf3d986b088cfd4a3a3cb750167f Mon Sep 17 00:00:00 2001 From: DavertMik Date: Sat, 11 Jul 2026 14:31:01 +0300 Subject: [PATCH 1/2] fix: honor --incognito by disabling the experience tracker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `explorbot --incognito` ("Run without recording experiences") flowed CLI -> ExplorBot -> Explorer, and Explorer called `new StateManager({ incognito })` — but StateManager's constructor took no parameters, so the flag was silently dropped and the ExperienceTracker was always enabled. Incognito runs read and wrote ./experience like normal runs. Thread `incognito` into the StateManager constructor and pass `{ disabled: options.incognito }` to ExperienceTracker (which already supports `disabled`). Argumentless constructions keep today's enabled behavior via `options.disabled ?? false`. Co-Authored-By: Claude Opus 4.8 --- src/state-manager.ts | 4 ++-- tests/unit/state-manager.test.ts | 9 +++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/state-manager.ts b/src/state-manager.ts index 26983a8..053a367 100644 --- a/src/state-manager.ts +++ b/src/state-manager.ts @@ -88,8 +88,8 @@ export class StateManager { private knowledgeDir: string; private nextStateId = 1; - constructor() { - this.experienceTracker = new ExperienceTracker(); + constructor(options: { incognito?: boolean } = {}) { + this.experienceTracker = new ExperienceTracker({ disabled: options.incognito }); const configParser = ConfigParser.getInstance(); const config = configParser.getConfig(); const configPath = configParser.getConfigPath(); diff --git a/tests/unit/state-manager.test.ts b/tests/unit/state-manager.test.ts index fd3bce7..48cdd80 100644 --- a/tests/unit/state-manager.test.ts +++ b/tests/unit/state-manager.test.ts @@ -40,6 +40,15 @@ describe('StateManager', () => { it('should have zero listeners initially', () => { expect(stateManager.getListenerCount()).toBe(0); }); + + it('disables the experience tracker in incognito mode', () => { + const incognito = new StateManager({ incognito: true }); + expect((incognito.getExperienceTracker() as any).disabled).toBe(true); + }); + + it('leaves the experience tracker enabled by default', () => { + expect((stateManager.getExperienceTracker() as any).disabled).toBe(false); + }); }); describe('updateState', () => { From f2a83c3ac3226c873adf9f61fedeb76ecf302bbf Mon Sep 17 00:00:00 2001 From: Michael Bodnarchuk Date: Sun, 12 Jul 2026 02:54:35 +0300 Subject: [PATCH 2/2] refactor: delegate StateManager knowledge loading to KnowledgeTracker (plan 010) (#79) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * refactor: delegate StateManager knowledge loading to KnowledgeTracker StateManager contained a complete second implementation of knowledge-file loading (scanKnowledgeFiles + its own cache + dir resolution) that had diverged from KnowledgeTracker.loadKnowledgeFiles — most importantly, the StateManager copy did NOT run ${env.*}/${config.*} interpolation. Researcher (the only production consumer of the StateManager copy) therefore saw raw ${env.PASSWORD} text while Navigator, TaskAgent, ExperienceTracker and commands saw interpolated content for the same page. Delete the copy and delegate to KnowledgeTracker (the CLAUDE.md single owner of the knowledge dir), sharing one tracker instance from Explorer. Make the tracker scan recursively (superset of both implementations), invalidate its isLoaded cache on addKnowledge (so a same-instance /learn read sees the write), and unify the two divergent Knowledge types onto one exported interface. Intended behavior deltas: the 30s rescan is gone (in-app writes picked up via addKnowledge invalidation), recursive scan now applies to every consumer, and Researcher now receives interpolated knowledge. Co-Authored-By: Claude Opus 4.8 * fix: give KnowledgeTracker a 30s refresh TTL (mid-session knowledge staleness) Deleting StateManager's knowledge scanner (which rescanned every 30s) left the shared KnowledgeTracker caching forever; addKnowledge's isLoaded invalidation is per-instance, so /learn (which builds its own tracker) and hand-edited files were never picked up by Explorer's shared instance until restart — and CLAUDE.md's documented "30-second refresh" existed nowhere. Restore the refresh in the single owner: loadKnowledgeFiles re-scans when the cache is older than 30s (KNOWLEDGE_REFRESH_MS), so on-disk changes from any source reach every consumer. Same-instance addKnowledge still forces an immediate reload via isLoaded=false. Co-Authored-By: Claude Opus 4.8 --------- Co-authored-by: DavertMik Co-authored-by: Claude Opus 4.8 --- src/explorer.ts | 2 +- src/knowledge-tracker.ts | 16 +++-- src/state-manager.ts | 88 +++------------------------- tests/unit/knowledge-tracker.test.ts | 44 +++++++++++++- tests/unit/state-manager.test.ts | 31 ++++++++++ 5 files changed, 95 insertions(+), 86 deletions(-) diff --git a/src/explorer.ts b/src/explorer.ts index 3dece6e..f9bc01e 100644 --- a/src/explorer.ts +++ b/src/explorer.ts @@ -82,8 +82,8 @@ class Explorer { this.aiProvider = aiProvider; this.options = options; this.initializeContainer(); - this.stateManager = new StateManager({ incognito: this.options?.incognito }); this.knowledgeTracker = new KnowledgeTracker(); + this.stateManager = new StateManager({ knowledgeTracker: this.knowledgeTracker, incognito: this.options?.incognito }); this.reporter = new Reporter(config.reporter, this.stateManager); } diff --git a/src/knowledge-tracker.ts b/src/knowledge-tracker.ts index d37ed9c..7385fd2 100644 --- a/src/knowledge-tracker.ts +++ b/src/knowledge-tracker.ts @@ -8,17 +8,20 @@ import { createDebug } from './utils/logger.js'; const debugLog = createDebug('explorbot:knowledge-tracker'); -interface Knowledge { +export interface Knowledge { filePath: string; url: string; content: string; [key: string]: any; } +const KNOWLEDGE_REFRESH_MS = 30000; + export class KnowledgeTracker { private knowledgeDir: string; private knowledgeFiles: Knowledge[] = []; private isLoaded = false; + private lastLoadedAt = 0; constructor() { const configParser = ConfigParser.getInstance(); @@ -38,7 +41,7 @@ export class KnowledgeTracker { } private loadKnowledgeFiles(): void { - if (this.isLoaded) return; + if (this.isLoaded && Date.now() - this.lastLoadedAt < KNOWLEDGE_REFRESH_MS) return; this.knowledgeFiles = []; @@ -46,9 +49,9 @@ export class KnowledgeTracker { return; } - const files = readdirSync(this.knowledgeDir) - .filter((file) => file.endsWith('.md')) - .map((file) => join(this.knowledgeDir, file)); + const files = readdirSync(this.knowledgeDir, { recursive: true }) + .filter((file) => typeof file === 'string' && file.endsWith('.md')) + .map((file) => join(this.knowledgeDir, file as string)); for (const filePath of files) { try { @@ -68,6 +71,7 @@ export class KnowledgeTracker { } this.isLoaded = true; + this.lastLoadedAt = Date.now(); } getRelevantKnowledge(state: ActionResult): Knowledge[] { @@ -127,6 +131,8 @@ export class KnowledgeTracker { writeFileSync(filePath, fileContent, 'utf8'); } + this.isLoaded = false; + return { filename, filePath, isNewFile }; } diff --git a/src/state-manager.ts b/src/state-manager.ts index 053a367..d118f7e 100644 --- a/src/state-manager.ts +++ b/src/state-manager.ts @@ -1,9 +1,9 @@ -import { existsSync, readFileSync, readdirSync } from 'node:fs'; -import { dirname, join } from 'node:path'; -import matter from 'gray-matter'; +import { existsSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; import { ActionResult } from './action-result.js'; -import { ConfigParser, outputPath } from './config.js'; +import { outputPath } from './config.js'; import { ExperienceTracker } from './experience-tracker.js'; +import { KnowledgeTracker, type Knowledge } from './knowledge-tracker.js'; import { detectFocusArea } from './utils/aria.js'; import { htmlTextSnapshot } from './utils/html.js'; import { createDebug, tag } from './utils/logger.js'; @@ -70,37 +70,20 @@ export interface StateTransition { export type StateChangeListener = (event: StateTransition) => void; -export interface Knowledge extends WebPageState { - /** File path */ - filePath: string; - /** Markdown content */ - content: string; -} +export type { Knowledge }; export class StateManager { private currentState: WebPageState | null = null; private stateHistory: StateTransition[] = []; private allVisitedUrls: Set = new Set(); - private knowledgeCache: Knowledge[] = []; - private lastKnowledgeScan: Date | null = null; private stateChangeListeners: StateChangeListener[] = []; private experienceTracker!: ExperienceTracker; - private knowledgeDir: string; + private knowledgeTracker: KnowledgeTracker; private nextStateId = 1; - constructor(options: { incognito?: boolean } = {}) { + constructor(options: { knowledgeTracker?: KnowledgeTracker; incognito?: boolean } = {}) { this.experienceTracker = new ExperienceTracker({ disabled: options.incognito }); - const configParser = ConfigParser.getInstance(); - const config = configParser.getConfig(); - const configPath = configParser.getConfigPath(); - - // Resolve knowledge directory relative to the config file location (project root) - if (configPath) { - const projectRoot = dirname(configPath); - this.knowledgeDir = join(projectRoot, config.dirs?.knowledge || 'knowledge'); - } else { - this.knowledgeDir = config.dirs?.knowledge || 'knowledge'; - } + this.knowledgeTracker = options.knowledgeTracker || new KnowledgeTracker(); } getExperienceTracker(): ExperienceTracker { @@ -335,65 +318,14 @@ export class StateManager { return this.stateHistory.slice(-count); } - /** - * Scan knowledge directory for .md files and cache them - */ - private scanKnowledgeFiles(): void { - const now = new Date(); - - // Only rescan every 30 seconds to avoid excessive file I/O - if (this.lastKnowledgeScan && now.getTime() - this.lastKnowledgeScan.getTime() < 30000) { - return; - } - - this.knowledgeCache = []; - - if (!existsSync(this.knowledgeDir)) { - debugLog(`Knowledge directory not found: ${this.knowledgeDir}`); - return; - } - - try { - const files = readdirSync(this.knowledgeDir, { recursive: true }) - .filter((file) => typeof file === 'string' && file.endsWith('.md')) - .map((file) => join(this.knowledgeDir, file as string)); - - for (const filePath of files) { - try { - const fileContent = readFileSync(filePath, 'utf8'); - const parsed = matter(fileContent); - - const urlPattern = parsed.data.url || parsed.data.path || '*'; - - this.knowledgeCache.push({ - filePath, - url: urlPattern, - ...parsed.data, - content: parsed.content, - }); - - debugLog(`Loaded knowledge file: ${filePath} (pattern: ${urlPattern})`); - } catch (error) { - debugLog(`Failed to load knowledge file ${filePath}:`, error); - } - } - - this.lastKnowledgeScan = now; - debugLog(`Scanned ${this.knowledgeCache.length} knowledge files`); - } catch (error) { - debugLog('Failed to scan knowledge directory:', error); - } - } /** * Get relevant knowledge files for current state */ getRelevantKnowledge(): Knowledge[] { if (!this.currentState) return []; - this.scanKnowledgeFiles(); - const actionResult = ActionResult.fromState(this.currentState); - return this.knowledgeCache.filter((knowledge) => actionResult.isMatchedBy(knowledge)); + return this.knowledgeTracker.getRelevantKnowledge(actionResult); } /** @@ -535,8 +467,6 @@ export class StateManager { this.stateHistory = []; this.allVisitedUrls.clear(); this.stateChangeListeners = []; - this.knowledgeCache = []; - this.lastKnowledgeScan = null; this.nextStateId = 1; // Clean up experience tracker if it has cleanup method diff --git a/tests/unit/knowledge-tracker.test.ts b/tests/unit/knowledge-tracker.test.ts index 6626096..27c2da1 100644 --- a/tests/unit/knowledge-tracker.test.ts +++ b/tests/unit/knowledge-tracker.test.ts @@ -1,4 +1,4 @@ -import { afterEach, beforeEach, describe, expect, it } from 'bun:test'; +import { afterEach, beforeEach, describe, expect, it, setSystemTime } from 'bun:test'; import { existsSync, rmSync } from 'node:fs'; import { mkdirSync, writeFileSync } from 'node:fs'; import matter from 'gray-matter'; @@ -125,4 +125,46 @@ describe('KnowledgeTracker', () => { expect(content[0]).not.toContain('${config.'); }); }); + + describe('recursive scan', () => { + it('loads knowledge from nested subdirectories', () => { + mkdirSync(`${knowledgeDir}/subdir`, { recursive: true }); + writeFileSync(`${knowledgeDir}/subdir/nested.md`, matter.stringify('Nested note', { url: '/nested-page' }), 'utf8'); + + const tracker = new KnowledgeTracker(); + const content = tracker.getKnowledgeForUrl('/nested-page'); + + expect(content[0]).toContain('Nested note'); + }); + }); + + describe('addKnowledge cache invalidation', () => { + it('sees knowledge added via addKnowledge on the same instance', () => { + const tracker = new KnowledgeTracker(); + expect(tracker.getMatchingKnowledge('/login')).toHaveLength(0); + + tracker.addKnowledge('/login', 'Use admin credentials'); + + const matched = tracker.getMatchingKnowledge('/login'); + expect(matched.length).toBeGreaterThan(0); + expect(matched[0].content).toContain('Use admin credentials'); + }); + + it('refreshes files written to disk by another instance after the TTL elapses', () => { + const tracker = new KnowledgeTracker(); + expect(tracker.getMatchingKnowledge('/ttl-page')).toHaveLength(0); + + writeFileSync(`${knowledgeDir}/ttl.md`, matter.stringify('Fresh note', { url: '/ttl-page' }), 'utf8'); + expect(tracker.getMatchingKnowledge('/ttl-page')).toHaveLength(0); + + setSystemTime(new Date(Date.now() + 31000)); + try { + const matched = tracker.getMatchingKnowledge('/ttl-page'); + expect(matched.length).toBeGreaterThan(0); + expect(matched[0].content).toContain('Fresh note'); + } finally { + setSystemTime(); + } + }); + }); }); diff --git a/tests/unit/state-manager.test.ts b/tests/unit/state-manager.test.ts index 48cdd80..9b89808 100644 --- a/tests/unit/state-manager.test.ts +++ b/tests/unit/state-manager.test.ts @@ -1,4 +1,6 @@ import { afterEach, beforeEach, describe, expect, it } from 'bun:test'; +import { mkdirSync, rmSync, writeFileSync } from 'node:fs'; +import matter from 'gray-matter'; import { ActionResult } from '../../src/action-result'; import { ConfigParser } from '../../src/config'; import { StateManager, type StateTransition, type WebPageState } from '../../src/state-manager'; @@ -51,6 +53,35 @@ describe('StateManager', () => { }); }); + describe('getRelevantKnowledge', () => { + const smKnowledgeDir = '/tmp/explorbot-sm-knowledge'; + const savedVar = process.env.SM_TEST_VAR; + + afterEach(() => { + process.env.SM_TEST_VAR = savedVar; + rmSync(smKnowledgeDir, { recursive: true, force: true }); + }); + + it('interpolates ${env.*} in knowledge via the delegated KnowledgeTracker', () => { + rmSync(smKnowledgeDir, { recursive: true, force: true }); + mkdirSync(smKnowledgeDir, { recursive: true }); + + const configParser = ConfigParser.getInstance(); + (configParser as any).config = { playwright: { browser: 'chromium' }, ai: { model: 'test' }, dirs: { knowledge: 'explorbot-sm-knowledge' } }; + (configParser as any).configPath = '/tmp/config.js'; + process.env.SM_TEST_VAR = 'interpolated-value'; + + writeFileSync(`${smKnowledgeDir}/page.md`, matter.stringify('Secret: ${env.SM_TEST_VAR}', { url: '/sm-page' }), 'utf8'); + + const sm = new StateManager(); + sm.updateState(new ActionResult({ url: '/sm-page', html: 'x' })); + + const knowledge = sm.getRelevantKnowledge(); + expect(knowledge[0].content).toContain('interpolated-value'); + expect(knowledge[0].content).not.toContain('${env.'); + }); + }); + describe('updateState', () => { it('should update current state from ActionResult', () => { const actionResult = new ActionResult({