Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions boat/doc-collector/src/ai/tools.ts
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,7 @@ async function attemptInteraction(explorer: Explorer, candidate: InteractionCand
const action = explorer.createAction();

for (const command of buildClickCommands(candidate.element, candidate.container)) {
const success = await action.attempt(command, buildPurpose(candidate), false);
const success = await action.attempt(command, buildPurpose(candidate));
if (success) {
return true;
}
Expand All @@ -155,15 +155,15 @@ async function attemptInteraction(explorer: Explorer, candidate: InteractionCand
async function restoreInteractionState(explorer: Explorer, restoreUrl: string, primaryCommand?: string | null): Promise<void> {
if (primaryCommand) {
const action = explorer.createAction();
const restored = await action.attempt(primaryCommand, `Restore initial state on ${restoreUrl}`, false);
const restored = await action.attempt(primaryCommand, `Restore initial state on ${restoreUrl}`);
if (restored) {
await wait(TAB_WAIT_MS);
return;
}
}

const action = explorer.createAction();
await action.attempt(`I.amOnPage(${JSON.stringify(restoreUrl)})`, `Restore page ${restoreUrl}`, false);
await action.attempt(`I.amOnPage(${JSON.stringify(restoreUrl)})`, `Restore page ${restoreUrl}`);
}

function buildTransition(candidate: InteractionCandidate, beforeState: WebPageState, afterState: WebPageState, changes: InteractionChanges): DocStateTransition {
Expand Down
66 changes: 0 additions & 66 deletions src/action-result.ts
Original file line number Diff line number Diff line change
Expand Up @@ -166,16 +166,6 @@ export class ActionResult implements ActionResultData {
return undefined;
}

get browserLogsContent(): any[] {
if (this._browserLogs !== undefined) {
return this._browserLogs;
}
if (this.logFile) {
return ActionResult.loadBrowserLogsFromFile(this.logFile);
}
return [];
}

get ariaSnapshot(): string | null {
if (this._ariaSnapshot !== undefined) {
return this._ariaSnapshot;
Expand Down Expand Up @@ -525,58 +515,6 @@ export class ActionResult implements ActionResultData {
return stateString;
}

private saveBrowserLogs(): void {
const logs = this.browserLogsContent;
if (!logs || logs.length === 0) return;

try {
const outputDir = 'output';
const stateHash = this.getStateHash();
const filename = `${stateHash}.log`;
const filePath = join(outputDir, filename);

// Ensure output directory exists
if (!fs.existsSync(outputDir)) {
fs.mkdirSync(outputDir, { recursive: true });
}

// Format logs for saving
const formattedLogs = logs.map((log: any) => {
const timestamp = new Date().toISOString();
const level = (log.type || log.level || 'LOG').toUpperCase();
const message = log.text || log.message || String(log);
return `[${timestamp}] ${level}: ${message}`;
});

// Save log content to file
const logContent = `${formattedLogs.join('\n')}\n`;
fs.writeFileSync(filePath, logContent, 'utf8');
} catch (error) {
// Silently fail to avoid breaking the main flow
console.error('Failed to save browser logs:', error);
}
}

private saveHtmlOutput(): void {
try {
const outputDir = 'output';
const stateHash = this.getStateHash();
const filename = `${stateHash}.html`;
const filePath = join(outputDir, filename);

// Ensure output directory exists
if (!fs.existsSync(outputDir)) {
fs.mkdirSync(outputDir, { recursive: true });
}

// Save HTML content to file
fs.writeFileSync(filePath, this.html, 'utf8');
} catch (error) {
// Silently fail to avoid breaking the main flow
console.error('Failed to save HTML output:', error);
}
}

async diff(previousState: ActionResult | null): Promise<Diff> {
return new Diff(this, previousState);
}
Expand Down Expand Up @@ -717,10 +655,6 @@ export class Diff {
return this._ariaDiffResult;
}

get ariaRemoved(): string | null {
return this._ariaDiffResult;
}

get htmlDiff(): HtmlDiffResult | null {
return this._htmlDiffResult;
}
Expand Down
88 changes: 1 addition & 87 deletions src/action.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,23 +5,17 @@ import { container, recorder } from 'codeceptjs';
import * as codeceptjs from 'codeceptjs';
import { ActionResult } from './action-result.js';
import { clearActivity, setActivity } from './activity.ts';
import { ExperienceCompactor } from './ai/experience-compactor.js';
import { Navigator } from './ai/navigator.js';
import type { Provider } from './ai/provider.js';
import { ConfigParser, outputPath } from './config.js';
import type { ExplorbotConfig } from './config.js';
import type { UserResolveFunction } from './explorbot.ts';
import { Observability } from './observability.ts';
import type { PlaywrightRecorder } from './playwright-recorder.ts';
import type { StateManager } from './state-manager.js';
import { isFatalBrowserError, isNavigationTransitionError } from './utils/browser-errors.ts';
import { extractCodeBlocks } from './utils/code-extractor.js';
import { captureHtmlForSnapshot, htmlCombinedSnapshot, minifyHtml } from './utils/html.js';
import { createDebug, setStepSpanParent, tag } from './utils/logger.js';
import { waitForPageReadiness } from './utils/page-readiness.ts';
import { codeceptJSSandbox, hasPlaywrightCommands, playwrightSandbox, sanitizeCodeBlock } from './utils/web-sandbox.ts';
import { safeFilename } from './utils/strings.ts';
import { throttle } from './utils/throttle.ts';

const debugLog = createDebug('explorbot:action');
const CAPTURE_NAVIGATION_TRANSITION_ATTEMPTS = 3;
Expand All @@ -34,7 +28,6 @@ class Action {

// action info
private action: string | null = null;
private expectation: string | null = null;
public lastError: Error | null = null;
public playwrightHelper: any;
public playwrightGroupId: string | null = null;
Expand All @@ -50,10 +43,6 @@ class Action {
this.recorder = recorder;
}

async caputrePageWithScreenshot(): Promise<ActionResult> {
return this.capturePageState({ includeScreenshot: true });
}

async saveScreenshot(): Promise<string | undefined> {
const currentState = this.stateManager.getCurrentState();
if (currentState?.screenshotFile) return currentState.screenshotFile;
Expand Down Expand Up @@ -339,84 +328,17 @@ class Action {
return this;
}

async expect(codeOrFunction: string | ((I: CodeceptJS.I) => void)): Promise<Action> {
const codeString = typeof codeOrFunction === 'string' ? codeOrFunction : codeOrFunction.toString();
this.expectation = codeString.toString();
const expectationPreview = sanitizeCodeBlock(codeString)
.split('\n')
.map((line) => line.trim())
.filter(Boolean)
.slice(0, 2)
.join(' ');
tag('step').log(`Expecting: ${expectationPreview || 'assertion'}`);
try {
debugLog('Executing expectation:', codeString);

if (typeof codeOrFunction === 'function') {
codeceptJSSandbox(this.actor, codeOrFunction);
} else {
const sanitizedCode = sanitizeCodeBlock(codeString);
if (!sanitizedCode) {
throw new Error('No valid I.* commands found in code block');
}
codeceptJSSandbox(this.actor, sanitizedCode);
}
await recorder.promise();
debugLog('Expectation executed successfully');

// Get current state from state manager
const currentState = this.stateManager.getCurrentState();
if (currentState) {
// Create ActionResult from current state for compatibility
this.actionResult = new ActionResult({
url: currentState.fullUrl || '',
title: currentState.title,
timestamp: currentState.timestamp,
html: '', // Empty HTML for expectation state
});
}

return this;
} catch (err) {
tag('error').log('Expectation failed:', errorToString(err));
this.lastError = err as Error;
await recorder.reset();
await recorder.start();
debugLog('Expectation failed:', errorToString(err));
} finally {
clearActivity();
}

return this;
}

public async waitForInteraction(): Promise<Action> {
// start with basic approach
await this.actor.wait(0.5);
return this;
}

public async attempt(codeBlock: string, originalMessage?: string, experience = true): Promise<boolean> {
public async attempt(codeBlock: string, originalMessage?: string): Promise<boolean> {
try {
debugLog('Resolution attempt...');
setActivity('🦾 Acting in browser...', 'action');

if (!this.actionResult) {
this.actionResult = ActionResult.fromState(this.stateManager.getCurrentState()!);
}
const prevActionResult = this.actionResult;
this.lastError = null;
await this.execute(codeBlock);

if (!this.expectation && originalMessage) {
this.expectation = originalMessage;
}

if (!this.expectation) {
return true;
}

debugLog('Resolved Expectation:', this.expectation);
return true;
} catch (error) {
this.lastError = error as Error;
Expand All @@ -430,14 +352,6 @@ class Action {
return this.actor;
}

setActor(actor: CodeceptJS.I): void {
this.actor = actor;
}

getCurrentState(): ActionResult | null {
return this.actionResult;
}

getActionResult(): ActionResult | null {
return this.actionResult;
}
Expand Down
20 changes: 0 additions & 20 deletions src/ai/conversation.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,14 +21,12 @@ const AUTO_COMPACT_ARIA_CHANGES_CUTOFF = 500;
const AUTO_COMPACT_TARGETED_HTML_CUTOFF = 500;

export class Conversation {
id: string;
messages: ModelMessage[];
model: any;
protectedPrefixCount = 0;
private autoTrimRules: Map<string, number>;

constructor(messages: ModelMessage[] = [], model?: any) {
this.id = this.generateId();
this.messages = messages;
this.model = model || '';
this.autoTrimRules = new Map();
Expand All @@ -45,20 +43,6 @@ export class Conversation {
});
}

addUserImage(image: string): void {
if (!image || image.trim() === '') {
console.warn('Warning: Attempting to add empty image to conversation');
return;
}

const imageData = image.startsWith('data:') ? image : `data:image/png;base64,${image}`;

this.messages.push({
role: 'user',
content: [{ type: 'file', mediaType: 'image/png', data: imageData }],
});
}

addAssistantText(text: string): void {
// Skip empty or whitespace-only messages
if (!text || text.trim() === '') {
Expand Down Expand Up @@ -226,10 +210,6 @@ export class Conversation {
return result;
}

private generateId(): string {
return `conv_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
}

getToolExecutions(): ToolExecution[] {
const toolCalls = new Map<string, any>();
for (const message of this.messages) {
Expand Down
4 changes: 2 additions & 2 deletions src/ai/driller.ts
Original file line number Diff line number Diff line change
Expand Up @@ -606,11 +606,11 @@ export class Driller extends TaskAgent implements Agent {
const action = this.explorer.createAction();

if (currentState?.url !== originalState.url) {
await action.attempt(`I.amOnPage(${JSON.stringify(targetUrl)})`, `${reason} (restore URL)`, false);
await action.attempt(`I.amOnPage(${JSON.stringify(targetUrl)})`, `${reason} (restore URL)`);
return;
}

await action.attempt('I.pressKey("Escape")', `${reason} (restore state)`, false);
await action.attempt('I.pressKey("Escape")', `${reason} (restore state)`);
}

private async saveToExperience(state: ActionResult, results: InteractionResult[]): Promise<void> {
Expand Down
9 changes: 3 additions & 6 deletions src/ai/navigator.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import type Action from '../action.ts';
import { ExperienceTracker, renderExperienceToc } from '../experience-tracker.js';
import Explorer from '../explorer.ts';
import { KnowledgeTracker } from '../knowledge-tracker.js';
import { type WebPageState, normalizeUrl } from '../state-manager.js';
import { normalizeUrl } from '../state-manager.js';
import { extractCodeBlocks } from '../utils/code-extractor.js';
import { HooksRunner } from '../utils/hooks-runner.ts';
import { createDebug, pluralize, tag } from '../utils/logger.js';
Expand All @@ -15,7 +15,6 @@ import { RulesLoader } from '../utils/rules-loader.ts';
import { extractStatePath } from '../utils/url-matcher.js';
import type { Agent } from './agent.js';
import type { Conversation } from './conversation.js';
import { ExperienceCompactor } from './experience-compactor.js';
import type { Provider } from './provider.js';
import { Researcher } from './researcher.ts';
import { actionRule, locatorRule, unexpectedPopupRule } from './rules.js';
Expand All @@ -27,7 +26,6 @@ const debugLog = createDebug('explorbot:navigator');
class Navigator implements Agent {
emoji = '🧭';
private provider: Provider;
private experienceCompactor: ExperienceCompactor;
private knowledgeTracker: KnowledgeTracker;
private experienceTracker: ExperienceTracker;
private hooksRunner: HooksRunner;
Expand Down Expand Up @@ -73,10 +71,9 @@ class Navigator implements Agent {
`;
private explorer: Explorer;

constructor(explorer: Explorer, provider: Provider, experienceCompactor: ExperienceCompactor, experienceTracker?: ExperienceTracker) {
constructor(explorer: Explorer, provider: Provider, experienceTracker?: ExperienceTracker) {
this.provider = provider;
this.explorer = explorer;
this.experienceCompactor = experienceCompactor;
this.knowledgeTracker = new KnowledgeTracker();
this.experienceTracker = experienceTracker || new ExperienceTracker();
this.hooksRunner = new HooksRunner(explorer, explorer.getConfig());
Expand Down Expand Up @@ -764,7 +761,7 @@ class Navigator implements Agent {

await this.explorer.switchToMainFrame();

const verified = await action.attempt(codeBlock, message, false);
const verified = await action.attempt(codeBlock, message);

if (verified) {
tag('success').log('Verification passed');
Expand Down
2 changes: 1 addition & 1 deletion src/ai/planner.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ import type { Agent } from './agent.js';
import { Conversation } from './conversation.ts';
import type { Fisherman } from './fisherman.ts';
import { WithSessionDedup } from './planner/session-dedup.ts';
import { getActiveStyle, getStyles } from './planner/styles.ts';
import { getActiveStyle } from './planner/styles.ts';
import { WithSubPages, getPlannedByStateHash, getRegisteredPlan, registerPlan } from './planner/subpages.ts';
import type { Provider } from './provider.js';
import { POSSIBLE_SECTIONS, type Researcher } from './researcher.ts';
Expand Down
Loading
Loading