fix: honor --incognito by disabling the experience tracker (plan 009)

src/explorer.ts

@@ -82,8 +82,8 @@ class Explorer {
     this.aiProvider = aiProvider;
     this.options = options;
     this.initializeContainer();
-    this.stateManager = new StateManager({ incognito: this.options?.incognito });
     this.knowledgeTracker = new KnowledgeTracker();
+    this.stateManager = new StateManager({ knowledgeTracker: this.knowledgeTracker, incognito: this.options?.incognito });
     this.reporter = new Reporter(config.reporter, this.stateManager);
   }
 

src/knowledge-tracker.ts

@@ -8,17 +8,20 @@ import { createDebug } from './utils/logger.js';
 
 const debugLog = createDebug('explorbot:knowledge-tracker');
 
-interface Knowledge {
+export interface Knowledge {
   filePath: string;
   url: string;
   content: string;
   [key: string]: any;
 }
 
+const KNOWLEDGE_REFRESH_MS = 30000;
+
 export class KnowledgeTracker {
   private knowledgeDir: string;
   private knowledgeFiles: Knowledge[] = [];
   private isLoaded = false;
+  private lastLoadedAt = 0;
 
   constructor() {
     const configParser = ConfigParser.getInstance();
@@ -38,17 +41,17 @@ export class KnowledgeTracker {
   }
 
   private loadKnowledgeFiles(): void {
-    if (this.isLoaded) return;
+    if (this.isLoaded && Date.now() - this.lastLoadedAt < KNOWLEDGE_REFRESH_MS) return;
 
     this.knowledgeFiles = [];
 
     if (!existsSync(this.knowledgeDir)) {
       return;
     }
 
-    const files = readdirSync(this.knowledgeDir)
-      .filter((file) => file.endsWith('.md'))
-      .map((file) => join(this.knowledgeDir, file));
+    const files = readdirSync(this.knowledgeDir, { recursive: true })
+      .filter((file) => typeof file === 'string' && file.endsWith('.md'))
+      .map((file) => join(this.knowledgeDir, file as string));
 
     for (const filePath of files) {
       try {
@@ -68,6 +71,7 @@ export class KnowledgeTracker {
     }
 
     this.isLoaded = true;
+    this.lastLoadedAt = Date.now();
   }
 
   getRelevantKnowledge(state: ActionResult): Knowledge[] {
@@ -127,6 +131,8 @@ export class KnowledgeTracker {
       writeFileSync(filePath, fileContent, 'utf8');
     }
 
+    this.isLoaded = false;
+
     return { filename, filePath, isNewFile };
   }
 

src/state-manager.ts

@@ -1,9 +1,9 @@
-import { existsSync, readFileSync, readdirSync } from 'node:fs';
-import { dirname, join } from 'node:path';
-import matter from 'gray-matter';
+import { existsSync, readFileSync } from 'node:fs';
+import { join } from 'node:path';
 import { ActionResult } from './action-result.js';
-import { ConfigParser, outputPath } from './config.js';
+import { outputPath } from './config.js';
 import { ExperienceTracker } from './experience-tracker.js';
+import { KnowledgeTracker, type Knowledge } from './knowledge-tracker.js';
 import { detectFocusArea } from './utils/aria.js';
 import { htmlTextSnapshot } from './utils/html.js';
 import { createDebug, tag } from './utils/logger.js';
@@ -70,37 +70,20 @@ export interface StateTransition {
 
 export type StateChangeListener = (event: StateTransition) => void;
 
-export interface Knowledge extends WebPageState {
-  /** File path */
-  filePath: string;
-  /** Markdown content */
-  content: string;
-}
+export type { Knowledge };
 
 export class StateManager {
   private currentState: WebPageState | null = null;
   private stateHistory: StateTransition[] = [];
   private allVisitedUrls: Set<string> = new Set();
-  private knowledgeCache: Knowledge[] = [];
-  private lastKnowledgeScan: Date | null = null;
   private stateChangeListeners: StateChangeListener[] = [];
   private experienceTracker!: ExperienceTracker;
-  private knowledgeDir: string;
+  private knowledgeTracker: KnowledgeTracker;
   private nextStateId = 1;
 
-  constructor() {
-    this.experienceTracker = new ExperienceTracker();
-    const configParser = ConfigParser.getInstance();
-    const config = configParser.getConfig();
-    const configPath = configParser.getConfigPath();
-
-    // Resolve knowledge directory relative to the config file location (project root)
-    if (configPath) {
-      const projectRoot = dirname(configPath);
-      this.knowledgeDir = join(projectRoot, config.dirs?.knowledge || 'knowledge');
-    } else {
-      this.knowledgeDir = config.dirs?.knowledge || 'knowledge';
-    }
+  constructor(options: { knowledgeTracker?: KnowledgeTracker; incognito?: boolean } = {}) {
+    this.experienceTracker = new ExperienceTracker({ disabled: options.incognito });
+    this.knowledgeTracker = options.knowledgeTracker || new KnowledgeTracker();
   }
 
   getExperienceTracker(): ExperienceTracker {
@@ -335,65 +318,14 @@ export class StateManager {
     return this.stateHistory.slice(-count);
   }
 
-  /**
-   * Scan knowledge directory for .md files and cache them
-   */
-  private scanKnowledgeFiles(): void {
-    const now = new Date();
-
-    // Only rescan every 30 seconds to avoid excessive file I/O
-    if (this.lastKnowledgeScan && now.getTime() - this.lastKnowledgeScan.getTime() < 30000) {
-      return;
-    }
-
-    this.knowledgeCache = [];
-
-    if (!existsSync(this.knowledgeDir)) {
-      debugLog(`Knowledge directory not found: ${this.knowledgeDir}`);
-      return;
-    }
-
-    try {
-      const files = readdirSync(this.knowledgeDir, { recursive: true })
-        .filter((file) => typeof file === 'string' && file.endsWith('.md'))
-        .map((file) => join(this.knowledgeDir, file as string));
-
-      for (const filePath of files) {
-        try {
-          const fileContent = readFileSync(filePath, 'utf8');
-          const parsed = matter(fileContent);
-
-          const urlPattern = parsed.data.url || parsed.data.path || '*';
-
-          this.knowledgeCache.push({
-            filePath,
-            url: urlPattern,
-            ...parsed.data,
-            content: parsed.content,
-          });
-
-          debugLog(`Loaded knowledge file: ${filePath} (pattern: ${urlPattern})`);
-        } catch (error) {
-          debugLog(`Failed to load knowledge file ${filePath}:`, error);
-        }
-      }
-
-      this.lastKnowledgeScan = now;
-      debugLog(`Scanned ${this.knowledgeCache.length} knowledge files`);
-    } catch (error) {
-      debugLog('Failed to scan knowledge directory:', error);
-    }
-  }
   /**
    * Get relevant knowledge files for current state
    */
   getRelevantKnowledge(): Knowledge[] {
     if (!this.currentState) return [];
 
-    this.scanKnowledgeFiles();
-
     const actionResult = ActionResult.fromState(this.currentState);
-    return this.knowledgeCache.filter((knowledge) => actionResult.isMatchedBy(knowledge));
+    return this.knowledgeTracker.getRelevantKnowledge(actionResult);
   }
 
   /**
@@ -535,8 +467,6 @@ export class StateManager {
     this.stateHistory = [];
     this.allVisitedUrls.clear();
     this.stateChangeListeners = [];
-    this.knowledgeCache = [];
-    this.lastKnowledgeScan = null;
     this.nextStateId = 1;
 
     // Clean up experience tracker if it has cleanup method

tests/unit/knowledge-tracker.test.ts

@@ -1,4 +1,4 @@
-import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
+import { afterEach, beforeEach, describe, expect, it, setSystemTime } from 'bun:test';
 import { existsSync, rmSync } from 'node:fs';
 import { mkdirSync, writeFileSync } from 'node:fs';
 import matter from 'gray-matter';
@@ -125,4 +125,46 @@ describe('KnowledgeTracker', () => {
       expect(content[0]).not.toContain('${config.');
     });
   });
+
+  describe('recursive scan', () => {
+    it('loads knowledge from nested subdirectories', () => {
+      mkdirSync(`${knowledgeDir}/subdir`, { recursive: true });
+      writeFileSync(`${knowledgeDir}/subdir/nested.md`, matter.stringify('Nested note', { url: '/nested-page' }), 'utf8');
+
+      const tracker = new KnowledgeTracker();
+      const content = tracker.getKnowledgeForUrl('/nested-page');
+
+      expect(content[0]).toContain('Nested note');
+    });
+  });
+
+  describe('addKnowledge cache invalidation', () => {
+    it('sees knowledge added via addKnowledge on the same instance', () => {
+      const tracker = new KnowledgeTracker();
+      expect(tracker.getMatchingKnowledge('/login')).toHaveLength(0);
+
+      tracker.addKnowledge('/login', 'Use admin credentials');
+
+      const matched = tracker.getMatchingKnowledge('/login');
+      expect(matched.length).toBeGreaterThan(0);
+      expect(matched[0].content).toContain('Use admin credentials');
+    });
+
+    it('refreshes files written to disk by another instance after the TTL elapses', () => {
+      const tracker = new KnowledgeTracker();
+      expect(tracker.getMatchingKnowledge('/ttl-page')).toHaveLength(0);
+
+      writeFileSync(`${knowledgeDir}/ttl.md`, matter.stringify('Fresh note', { url: '/ttl-page' }), 'utf8');
+      expect(tracker.getMatchingKnowledge('/ttl-page')).toHaveLength(0);
+
+      setSystemTime(new Date(Date.now() + 31000));
+      try {
+        const matched = tracker.getMatchingKnowledge('/ttl-page');
+        expect(matched.length).toBeGreaterThan(0);
+        expect(matched[0].content).toContain('Fresh note');
+      } finally {
+        setSystemTime();
+      }
+    });
+  });
 });

tests/unit/state-manager.test.ts

@@ -1,4 +1,6 @@
 import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
+import { mkdirSync, rmSync, writeFileSync } from 'node:fs';
+import matter from 'gray-matter';
 import { ActionResult } from '../../src/action-result';
 import { ConfigParser } from '../../src/config';
 import { StateManager, type StateTransition, type WebPageState } from '../../src/state-manager';
@@ -40,6 +42,44 @@ describe('StateManager', () => {
     it('should have zero listeners initially', () => {
       expect(stateManager.getListenerCount()).toBe(0);
     });
+
+    it('disables the experience tracker in incognito mode', () => {
+      const incognito = new StateManager({ incognito: true });
+      expect((incognito.getExperienceTracker() as any).disabled).toBe(true);
+    });
+
+    it('leaves the experience tracker enabled by default', () => {
+      expect((stateManager.getExperienceTracker() as any).disabled).toBe(false);
+    });
+  });
+
+  describe('getRelevantKnowledge', () => {
+    const smKnowledgeDir = '/tmp/explorbot-sm-knowledge';
+    const savedVar = process.env.SM_TEST_VAR;
+
+    afterEach(() => {
+      process.env.SM_TEST_VAR = savedVar;
+      rmSync(smKnowledgeDir, { recursive: true, force: true });
+    });
+
+    it('interpolates ${env.*} in knowledge via the delegated KnowledgeTracker', () => {
+      rmSync(smKnowledgeDir, { recursive: true, force: true });
+      mkdirSync(smKnowledgeDir, { recursive: true });
+
+      const configParser = ConfigParser.getInstance();
+      (configParser as any).config = { playwright: { browser: 'chromium' }, ai: { model: 'test' }, dirs: { knowledge: 'explorbot-sm-knowledge' } };
+      (configParser as any).configPath = '/tmp/config.js';
+      process.env.SM_TEST_VAR = 'interpolated-value';
+
+      writeFileSync(`${smKnowledgeDir}/page.md`, matter.stringify('Secret: ${env.SM_TEST_VAR}', { url: '/sm-page' }), 'utf8');
+
+      const sm = new StateManager();
+      sm.updateState(new ActionResult({ url: '/sm-page', html: '<html><body>x</body></html>' }));
+
+      const knowledge = sm.getRelevantKnowledge();
+      expect(knowledge[0].content).toContain('interpolated-value');
+      expect(knowledge[0].content).not.toContain('${env.');
+    });
   });
 
   describe('updateState', () => {