From 107176b196e35e8e0985cc06c4394cb87dd7a377 Mon Sep 17 00:00:00 2001 From: DavertMik Date: Sat, 11 Jul 2026 14:39:32 +0300 Subject: [PATCH 1/2] refactor: delegate StateManager knowledge loading to KnowledgeTracker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit StateManager contained a complete second implementation of knowledge-file loading (scanKnowledgeFiles + its own cache + dir resolution) that had diverged from KnowledgeTracker.loadKnowledgeFiles — most importantly, the StateManager copy did NOT run ${env.*}/${config.*} interpolation. Researcher (the only production consumer of the StateManager copy) therefore saw raw ${env.PASSWORD} text while Navigator, TaskAgent, ExperienceTracker and commands saw interpolated content for the same page. Delete the copy and delegate to KnowledgeTracker (the CLAUDE.md single owner of the knowledge dir), sharing one tracker instance from Explorer. Make the tracker scan recursively (superset of both implementations), invalidate its isLoaded cache on addKnowledge (so a same-instance /learn read sees the write), and unify the two divergent Knowledge types onto one exported interface. Intended behavior deltas: the 30s rescan is gone (in-app writes picked up via addKnowledge invalidation), recursive scan now applies to every consumer, and Researcher now receives interpolated knowledge. Co-Authored-By: Claude Opus 4.8 --- src/explorer.ts | 2 +- src/knowledge-tracker.ts | 10 ++-- src/state-manager.ts | 88 +++------------------------- tests/unit/knowledge-tracker.test.ts | 25 ++++++++ tests/unit/state-manager.test.ts | 31 ++++++++++ 5 files changed, 72 insertions(+), 84 deletions(-) diff --git a/src/explorer.ts b/src/explorer.ts index 3dece6e..f9bc01e 100644 --- a/src/explorer.ts +++ b/src/explorer.ts @@ -82,8 +82,8 @@ class Explorer { this.aiProvider = aiProvider; this.options = options; this.initializeContainer(); - this.stateManager = new StateManager({ incognito: this.options?.incognito }); this.knowledgeTracker = new KnowledgeTracker(); + this.stateManager = new StateManager({ knowledgeTracker: this.knowledgeTracker, incognito: this.options?.incognito }); this.reporter = new Reporter(config.reporter, this.stateManager); } diff --git a/src/knowledge-tracker.ts b/src/knowledge-tracker.ts index d37ed9c..02a37c7 100644 --- a/src/knowledge-tracker.ts +++ b/src/knowledge-tracker.ts @@ -8,7 +8,7 @@ import { createDebug } from './utils/logger.js'; const debugLog = createDebug('explorbot:knowledge-tracker'); -interface Knowledge { +export interface Knowledge { filePath: string; url: string; content: string; @@ -46,9 +46,9 @@ export class KnowledgeTracker { return; } - const files = readdirSync(this.knowledgeDir) - .filter((file) => file.endsWith('.md')) - .map((file) => join(this.knowledgeDir, file)); + const files = readdirSync(this.knowledgeDir, { recursive: true }) + .filter((file) => typeof file === 'string' && file.endsWith('.md')) + .map((file) => join(this.knowledgeDir, file as string)); for (const filePath of files) { try { @@ -127,6 +127,8 @@ export class KnowledgeTracker { writeFileSync(filePath, fileContent, 'utf8'); } + this.isLoaded = false; + return { filename, filePath, isNewFile }; } diff --git a/src/state-manager.ts b/src/state-manager.ts index 053a367..d118f7e 100644 --- a/src/state-manager.ts +++ b/src/state-manager.ts @@ -1,9 +1,9 @@ -import { existsSync, readFileSync, readdirSync } from 'node:fs'; -import { dirname, join } from 'node:path'; -import matter from 'gray-matter'; +import { existsSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; import { ActionResult } from './action-result.js'; -import { ConfigParser, outputPath } from './config.js'; +import { outputPath } from './config.js'; import { ExperienceTracker } from './experience-tracker.js'; +import { KnowledgeTracker, type Knowledge } from './knowledge-tracker.js'; import { detectFocusArea } from './utils/aria.js'; import { htmlTextSnapshot } from './utils/html.js'; import { createDebug, tag } from './utils/logger.js'; @@ -70,37 +70,20 @@ export interface StateTransition { export type StateChangeListener = (event: StateTransition) => void; -export interface Knowledge extends WebPageState { - /** File path */ - filePath: string; - /** Markdown content */ - content: string; -} +export type { Knowledge }; export class StateManager { private currentState: WebPageState | null = null; private stateHistory: StateTransition[] = []; private allVisitedUrls: Set = new Set(); - private knowledgeCache: Knowledge[] = []; - private lastKnowledgeScan: Date | null = null; private stateChangeListeners: StateChangeListener[] = []; private experienceTracker!: ExperienceTracker; - private knowledgeDir: string; + private knowledgeTracker: KnowledgeTracker; private nextStateId = 1; - constructor(options: { incognito?: boolean } = {}) { + constructor(options: { knowledgeTracker?: KnowledgeTracker; incognito?: boolean } = {}) { this.experienceTracker = new ExperienceTracker({ disabled: options.incognito }); - const configParser = ConfigParser.getInstance(); - const config = configParser.getConfig(); - const configPath = configParser.getConfigPath(); - - // Resolve knowledge directory relative to the config file location (project root) - if (configPath) { - const projectRoot = dirname(configPath); - this.knowledgeDir = join(projectRoot, config.dirs?.knowledge || 'knowledge'); - } else { - this.knowledgeDir = config.dirs?.knowledge || 'knowledge'; - } + this.knowledgeTracker = options.knowledgeTracker || new KnowledgeTracker(); } getExperienceTracker(): ExperienceTracker { @@ -335,65 +318,14 @@ export class StateManager { return this.stateHistory.slice(-count); } - /** - * Scan knowledge directory for .md files and cache them - */ - private scanKnowledgeFiles(): void { - const now = new Date(); - - // Only rescan every 30 seconds to avoid excessive file I/O - if (this.lastKnowledgeScan && now.getTime() - this.lastKnowledgeScan.getTime() < 30000) { - return; - } - - this.knowledgeCache = []; - - if (!existsSync(this.knowledgeDir)) { - debugLog(`Knowledge directory not found: ${this.knowledgeDir}`); - return; - } - - try { - const files = readdirSync(this.knowledgeDir, { recursive: true }) - .filter((file) => typeof file === 'string' && file.endsWith('.md')) - .map((file) => join(this.knowledgeDir, file as string)); - - for (const filePath of files) { - try { - const fileContent = readFileSync(filePath, 'utf8'); - const parsed = matter(fileContent); - - const urlPattern = parsed.data.url || parsed.data.path || '*'; - - this.knowledgeCache.push({ - filePath, - url: urlPattern, - ...parsed.data, - content: parsed.content, - }); - - debugLog(`Loaded knowledge file: ${filePath} (pattern: ${urlPattern})`); - } catch (error) { - debugLog(`Failed to load knowledge file ${filePath}:`, error); - } - } - - this.lastKnowledgeScan = now; - debugLog(`Scanned ${this.knowledgeCache.length} knowledge files`); - } catch (error) { - debugLog('Failed to scan knowledge directory:', error); - } - } /** * Get relevant knowledge files for current state */ getRelevantKnowledge(): Knowledge[] { if (!this.currentState) return []; - this.scanKnowledgeFiles(); - const actionResult = ActionResult.fromState(this.currentState); - return this.knowledgeCache.filter((knowledge) => actionResult.isMatchedBy(knowledge)); + return this.knowledgeTracker.getRelevantKnowledge(actionResult); } /** @@ -535,8 +467,6 @@ export class StateManager { this.stateHistory = []; this.allVisitedUrls.clear(); this.stateChangeListeners = []; - this.knowledgeCache = []; - this.lastKnowledgeScan = null; this.nextStateId = 1; // Clean up experience tracker if it has cleanup method diff --git a/tests/unit/knowledge-tracker.test.ts b/tests/unit/knowledge-tracker.test.ts index 6626096..4c0e43e 100644 --- a/tests/unit/knowledge-tracker.test.ts +++ b/tests/unit/knowledge-tracker.test.ts @@ -125,4 +125,29 @@ describe('KnowledgeTracker', () => { expect(content[0]).not.toContain('${config.'); }); }); + + describe('recursive scan', () => { + it('loads knowledge from nested subdirectories', () => { + mkdirSync(`${knowledgeDir}/subdir`, { recursive: true }); + writeFileSync(`${knowledgeDir}/subdir/nested.md`, matter.stringify('Nested note', { url: '/nested-page' }), 'utf8'); + + const tracker = new KnowledgeTracker(); + const content = tracker.getKnowledgeForUrl('/nested-page'); + + expect(content[0]).toContain('Nested note'); + }); + }); + + describe('addKnowledge cache invalidation', () => { + it('sees knowledge added via addKnowledge on the same instance', () => { + const tracker = new KnowledgeTracker(); + expect(tracker.getMatchingKnowledge('/login')).toHaveLength(0); + + tracker.addKnowledge('/login', 'Use admin credentials'); + + const matched = tracker.getMatchingKnowledge('/login'); + expect(matched.length).toBeGreaterThan(0); + expect(matched[0].content).toContain('Use admin credentials'); + }); + }); }); diff --git a/tests/unit/state-manager.test.ts b/tests/unit/state-manager.test.ts index 48cdd80..9b89808 100644 --- a/tests/unit/state-manager.test.ts +++ b/tests/unit/state-manager.test.ts @@ -1,4 +1,6 @@ import { afterEach, beforeEach, describe, expect, it } from 'bun:test'; +import { mkdirSync, rmSync, writeFileSync } from 'node:fs'; +import matter from 'gray-matter'; import { ActionResult } from '../../src/action-result'; import { ConfigParser } from '../../src/config'; import { StateManager, type StateTransition, type WebPageState } from '../../src/state-manager'; @@ -51,6 +53,35 @@ describe('StateManager', () => { }); }); + describe('getRelevantKnowledge', () => { + const smKnowledgeDir = '/tmp/explorbot-sm-knowledge'; + const savedVar = process.env.SM_TEST_VAR; + + afterEach(() => { + process.env.SM_TEST_VAR = savedVar; + rmSync(smKnowledgeDir, { recursive: true, force: true }); + }); + + it('interpolates ${env.*} in knowledge via the delegated KnowledgeTracker', () => { + rmSync(smKnowledgeDir, { recursive: true, force: true }); + mkdirSync(smKnowledgeDir, { recursive: true }); + + const configParser = ConfigParser.getInstance(); + (configParser as any).config = { playwright: { browser: 'chromium' }, ai: { model: 'test' }, dirs: { knowledge: 'explorbot-sm-knowledge' } }; + (configParser as any).configPath = '/tmp/config.js'; + process.env.SM_TEST_VAR = 'interpolated-value'; + + writeFileSync(`${smKnowledgeDir}/page.md`, matter.stringify('Secret: ${env.SM_TEST_VAR}', { url: '/sm-page' }), 'utf8'); + + const sm = new StateManager(); + sm.updateState(new ActionResult({ url: '/sm-page', html: 'x' })); + + const knowledge = sm.getRelevantKnowledge(); + expect(knowledge[0].content).toContain('interpolated-value'); + expect(knowledge[0].content).not.toContain('${env.'); + }); + }); + describe('updateState', () => { it('should update current state from ActionResult', () => { const actionResult = new ActionResult({ From b4fc1abe317ed2fb617ae25c70096b6e14605a1b Mon Sep 17 00:00:00 2001 From: DavertMik Date: Sat, 11 Jul 2026 16:59:14 +0300 Subject: [PATCH 2/2] fix: give KnowledgeTracker a 30s refresh TTL (mid-session knowledge staleness) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deleting StateManager's knowledge scanner (which rescanned every 30s) left the shared KnowledgeTracker caching forever; addKnowledge's isLoaded invalidation is per-instance, so /learn (which builds its own tracker) and hand-edited files were never picked up by Explorer's shared instance until restart — and CLAUDE.md's documented "30-second refresh" existed nowhere. Restore the refresh in the single owner: loadKnowledgeFiles re-scans when the cache is older than 30s (KNOWLEDGE_REFRESH_MS), so on-disk changes from any source reach every consumer. Same-instance addKnowledge still forces an immediate reload via isLoaded=false. Co-Authored-By: Claude Opus 4.8 --- src/knowledge-tracker.ts | 6 +++++- tests/unit/knowledge-tracker.test.ts | 19 ++++++++++++++++++- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/knowledge-tracker.ts b/src/knowledge-tracker.ts index 02a37c7..7385fd2 100644 --- a/src/knowledge-tracker.ts +++ b/src/knowledge-tracker.ts @@ -15,10 +15,13 @@ export interface Knowledge { [key: string]: any; } +const KNOWLEDGE_REFRESH_MS = 30000; + export class KnowledgeTracker { private knowledgeDir: string; private knowledgeFiles: Knowledge[] = []; private isLoaded = false; + private lastLoadedAt = 0; constructor() { const configParser = ConfigParser.getInstance(); @@ -38,7 +41,7 @@ export class KnowledgeTracker { } private loadKnowledgeFiles(): void { - if (this.isLoaded) return; + if (this.isLoaded && Date.now() - this.lastLoadedAt < KNOWLEDGE_REFRESH_MS) return; this.knowledgeFiles = []; @@ -68,6 +71,7 @@ export class KnowledgeTracker { } this.isLoaded = true; + this.lastLoadedAt = Date.now(); } getRelevantKnowledge(state: ActionResult): Knowledge[] { diff --git a/tests/unit/knowledge-tracker.test.ts b/tests/unit/knowledge-tracker.test.ts index 4c0e43e..27c2da1 100644 --- a/tests/unit/knowledge-tracker.test.ts +++ b/tests/unit/knowledge-tracker.test.ts @@ -1,4 +1,4 @@ -import { afterEach, beforeEach, describe, expect, it } from 'bun:test'; +import { afterEach, beforeEach, describe, expect, it, setSystemTime } from 'bun:test'; import { existsSync, rmSync } from 'node:fs'; import { mkdirSync, writeFileSync } from 'node:fs'; import matter from 'gray-matter'; @@ -149,5 +149,22 @@ describe('KnowledgeTracker', () => { expect(matched.length).toBeGreaterThan(0); expect(matched[0].content).toContain('Use admin credentials'); }); + + it('refreshes files written to disk by another instance after the TTL elapses', () => { + const tracker = new KnowledgeTracker(); + expect(tracker.getMatchingKnowledge('/ttl-page')).toHaveLength(0); + + writeFileSync(`${knowledgeDir}/ttl.md`, matter.stringify('Fresh note', { url: '/ttl-page' }), 'utf8'); + expect(tracker.getMatchingKnowledge('/ttl-page')).toHaveLength(0); + + setSystemTime(new Date(Date.now() + 31000)); + try { + const matched = tracker.getMatchingKnowledge('/ttl-page'); + expect(matched.length).toBeGreaterThan(0); + expect(matched[0].content).toContain('Fresh note'); + } finally { + setSystemTime(); + } + }); }); });