diff --git a/src/explorer.ts b/src/explorer.ts index 3dece6e..f9bc01e 100644 --- a/src/explorer.ts +++ b/src/explorer.ts @@ -82,8 +82,8 @@ class Explorer { this.aiProvider = aiProvider; this.options = options; this.initializeContainer(); - this.stateManager = new StateManager({ incognito: this.options?.incognito }); this.knowledgeTracker = new KnowledgeTracker(); + this.stateManager = new StateManager({ knowledgeTracker: this.knowledgeTracker, incognito: this.options?.incognito }); this.reporter = new Reporter(config.reporter, this.stateManager); } diff --git a/src/knowledge-tracker.ts b/src/knowledge-tracker.ts index d37ed9c..7385fd2 100644 --- a/src/knowledge-tracker.ts +++ b/src/knowledge-tracker.ts @@ -8,17 +8,20 @@ import { createDebug } from './utils/logger.js'; const debugLog = createDebug('explorbot:knowledge-tracker'); -interface Knowledge { +export interface Knowledge { filePath: string; url: string; content: string; [key: string]: any; } +const KNOWLEDGE_REFRESH_MS = 30000; + export class KnowledgeTracker { private knowledgeDir: string; private knowledgeFiles: Knowledge[] = []; private isLoaded = false; + private lastLoadedAt = 0; constructor() { const configParser = ConfigParser.getInstance(); @@ -38,7 +41,7 @@ export class KnowledgeTracker { } private loadKnowledgeFiles(): void { - if (this.isLoaded) return; + if (this.isLoaded && Date.now() - this.lastLoadedAt < KNOWLEDGE_REFRESH_MS) return; this.knowledgeFiles = []; @@ -46,9 +49,9 @@ export class KnowledgeTracker { return; } - const files = readdirSync(this.knowledgeDir) - .filter((file) => file.endsWith('.md')) - .map((file) => join(this.knowledgeDir, file)); + const files = readdirSync(this.knowledgeDir, { recursive: true }) + .filter((file) => typeof file === 'string' && file.endsWith('.md')) + .map((file) => join(this.knowledgeDir, file as string)); for (const filePath of files) { try { @@ -68,6 +71,7 @@ export class KnowledgeTracker { } this.isLoaded = true; + this.lastLoadedAt = Date.now(); } getRelevantKnowledge(state: ActionResult): Knowledge[] { @@ -127,6 +131,8 @@ export class KnowledgeTracker { writeFileSync(filePath, fileContent, 'utf8'); } + this.isLoaded = false; + return { filename, filePath, isNewFile }; } diff --git a/src/state-manager.ts b/src/state-manager.ts index 053a367..d118f7e 100644 --- a/src/state-manager.ts +++ b/src/state-manager.ts @@ -1,9 +1,9 @@ -import { existsSync, readFileSync, readdirSync } from 'node:fs'; -import { dirname, join } from 'node:path'; -import matter from 'gray-matter'; +import { existsSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; import { ActionResult } from './action-result.js'; -import { ConfigParser, outputPath } from './config.js'; +import { outputPath } from './config.js'; import { ExperienceTracker } from './experience-tracker.js'; +import { KnowledgeTracker, type Knowledge } from './knowledge-tracker.js'; import { detectFocusArea } from './utils/aria.js'; import { htmlTextSnapshot } from './utils/html.js'; import { createDebug, tag } from './utils/logger.js'; @@ -70,37 +70,20 @@ export interface StateTransition { export type StateChangeListener = (event: StateTransition) => void; -export interface Knowledge extends WebPageState { - /** File path */ - filePath: string; - /** Markdown content */ - content: string; -} +export type { Knowledge }; export class StateManager { private currentState: WebPageState | null = null; private stateHistory: StateTransition[] = []; private allVisitedUrls: Set = new Set(); - private knowledgeCache: Knowledge[] = []; - private lastKnowledgeScan: Date | null = null; private stateChangeListeners: StateChangeListener[] = []; private experienceTracker!: ExperienceTracker; - private knowledgeDir: string; + private knowledgeTracker: KnowledgeTracker; private nextStateId = 1; - constructor(options: { incognito?: boolean } = {}) { + constructor(options: { knowledgeTracker?: KnowledgeTracker; incognito?: boolean } = {}) { this.experienceTracker = new ExperienceTracker({ disabled: options.incognito }); - const configParser = ConfigParser.getInstance(); - const config = configParser.getConfig(); - const configPath = configParser.getConfigPath(); - - // Resolve knowledge directory relative to the config file location (project root) - if (configPath) { - const projectRoot = dirname(configPath); - this.knowledgeDir = join(projectRoot, config.dirs?.knowledge || 'knowledge'); - } else { - this.knowledgeDir = config.dirs?.knowledge || 'knowledge'; - } + this.knowledgeTracker = options.knowledgeTracker || new KnowledgeTracker(); } getExperienceTracker(): ExperienceTracker { @@ -335,65 +318,14 @@ export class StateManager { return this.stateHistory.slice(-count); } - /** - * Scan knowledge directory for .md files and cache them - */ - private scanKnowledgeFiles(): void { - const now = new Date(); - - // Only rescan every 30 seconds to avoid excessive file I/O - if (this.lastKnowledgeScan && now.getTime() - this.lastKnowledgeScan.getTime() < 30000) { - return; - } - - this.knowledgeCache = []; - - if (!existsSync(this.knowledgeDir)) { - debugLog(`Knowledge directory not found: ${this.knowledgeDir}`); - return; - } - - try { - const files = readdirSync(this.knowledgeDir, { recursive: true }) - .filter((file) => typeof file === 'string' && file.endsWith('.md')) - .map((file) => join(this.knowledgeDir, file as string)); - - for (const filePath of files) { - try { - const fileContent = readFileSync(filePath, 'utf8'); - const parsed = matter(fileContent); - - const urlPattern = parsed.data.url || parsed.data.path || '*'; - - this.knowledgeCache.push({ - filePath, - url: urlPattern, - ...parsed.data, - content: parsed.content, - }); - - debugLog(`Loaded knowledge file: ${filePath} (pattern: ${urlPattern})`); - } catch (error) { - debugLog(`Failed to load knowledge file ${filePath}:`, error); - } - } - - this.lastKnowledgeScan = now; - debugLog(`Scanned ${this.knowledgeCache.length} knowledge files`); - } catch (error) { - debugLog('Failed to scan knowledge directory:', error); - } - } /** * Get relevant knowledge files for current state */ getRelevantKnowledge(): Knowledge[] { if (!this.currentState) return []; - this.scanKnowledgeFiles(); - const actionResult = ActionResult.fromState(this.currentState); - return this.knowledgeCache.filter((knowledge) => actionResult.isMatchedBy(knowledge)); + return this.knowledgeTracker.getRelevantKnowledge(actionResult); } /** @@ -535,8 +467,6 @@ export class StateManager { this.stateHistory = []; this.allVisitedUrls.clear(); this.stateChangeListeners = []; - this.knowledgeCache = []; - this.lastKnowledgeScan = null; this.nextStateId = 1; // Clean up experience tracker if it has cleanup method diff --git a/tests/unit/knowledge-tracker.test.ts b/tests/unit/knowledge-tracker.test.ts index 6626096..27c2da1 100644 --- a/tests/unit/knowledge-tracker.test.ts +++ b/tests/unit/knowledge-tracker.test.ts @@ -1,4 +1,4 @@ -import { afterEach, beforeEach, describe, expect, it } from 'bun:test'; +import { afterEach, beforeEach, describe, expect, it, setSystemTime } from 'bun:test'; import { existsSync, rmSync } from 'node:fs'; import { mkdirSync, writeFileSync } from 'node:fs'; import matter from 'gray-matter'; @@ -125,4 +125,46 @@ describe('KnowledgeTracker', () => { expect(content[0]).not.toContain('${config.'); }); }); + + describe('recursive scan', () => { + it('loads knowledge from nested subdirectories', () => { + mkdirSync(`${knowledgeDir}/subdir`, { recursive: true }); + writeFileSync(`${knowledgeDir}/subdir/nested.md`, matter.stringify('Nested note', { url: '/nested-page' }), 'utf8'); + + const tracker = new KnowledgeTracker(); + const content = tracker.getKnowledgeForUrl('/nested-page'); + + expect(content[0]).toContain('Nested note'); + }); + }); + + describe('addKnowledge cache invalidation', () => { + it('sees knowledge added via addKnowledge on the same instance', () => { + const tracker = new KnowledgeTracker(); + expect(tracker.getMatchingKnowledge('/login')).toHaveLength(0); + + tracker.addKnowledge('/login', 'Use admin credentials'); + + const matched = tracker.getMatchingKnowledge('/login'); + expect(matched.length).toBeGreaterThan(0); + expect(matched[0].content).toContain('Use admin credentials'); + }); + + it('refreshes files written to disk by another instance after the TTL elapses', () => { + const tracker = new KnowledgeTracker(); + expect(tracker.getMatchingKnowledge('/ttl-page')).toHaveLength(0); + + writeFileSync(`${knowledgeDir}/ttl.md`, matter.stringify('Fresh note', { url: '/ttl-page' }), 'utf8'); + expect(tracker.getMatchingKnowledge('/ttl-page')).toHaveLength(0); + + setSystemTime(new Date(Date.now() + 31000)); + try { + const matched = tracker.getMatchingKnowledge('/ttl-page'); + expect(matched.length).toBeGreaterThan(0); + expect(matched[0].content).toContain('Fresh note'); + } finally { + setSystemTime(); + } + }); + }); }); diff --git a/tests/unit/state-manager.test.ts b/tests/unit/state-manager.test.ts index 48cdd80..9b89808 100644 --- a/tests/unit/state-manager.test.ts +++ b/tests/unit/state-manager.test.ts @@ -1,4 +1,6 @@ import { afterEach, beforeEach, describe, expect, it } from 'bun:test'; +import { mkdirSync, rmSync, writeFileSync } from 'node:fs'; +import matter from 'gray-matter'; import { ActionResult } from '../../src/action-result'; import { ConfigParser } from '../../src/config'; import { StateManager, type StateTransition, type WebPageState } from '../../src/state-manager'; @@ -51,6 +53,35 @@ describe('StateManager', () => { }); }); + describe('getRelevantKnowledge', () => { + const smKnowledgeDir = '/tmp/explorbot-sm-knowledge'; + const savedVar = process.env.SM_TEST_VAR; + + afterEach(() => { + process.env.SM_TEST_VAR = savedVar; + rmSync(smKnowledgeDir, { recursive: true, force: true }); + }); + + it('interpolates ${env.*} in knowledge via the delegated KnowledgeTracker', () => { + rmSync(smKnowledgeDir, { recursive: true, force: true }); + mkdirSync(smKnowledgeDir, { recursive: true }); + + const configParser = ConfigParser.getInstance(); + (configParser as any).config = { playwright: { browser: 'chromium' }, ai: { model: 'test' }, dirs: { knowledge: 'explorbot-sm-knowledge' } }; + (configParser as any).configPath = '/tmp/config.js'; + process.env.SM_TEST_VAR = 'interpolated-value'; + + writeFileSync(`${smKnowledgeDir}/page.md`, matter.stringify('Secret: ${env.SM_TEST_VAR}', { url: '/sm-page' }), 'utf8'); + + const sm = new StateManager(); + sm.updateState(new ActionResult({ url: '/sm-page', html: 'x' })); + + const knowledge = sm.getRelevantKnowledge(); + expect(knowledge[0].content).toContain('interpolated-value'); + expect(knowledge[0].content).not.toContain('${env.'); + }); + }); + describe('updateState', () => { it('should update current state from ActionResult', () => { const actionResult = new ActionResult({