diff --git a/Cargo.lock b/Cargo.lock index 0e0e03f3..b5c55b89 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11370,6 +11370,7 @@ dependencies = [ "anyhow", "async-trait", "axum", + "base64 0.22.1", "chromiumoxide", "futures", "htmd", diff --git a/crates/code_assistant_core/src/tools/impls/browser.rs b/crates/code_assistant_core/src/tools/impls/browser.rs index 71f9f74b..09765b65 100644 --- a/crates/code_assistant_core/src/tools/impls/browser.rs +++ b/crates/code_assistant_core/src/tools/impls/browser.rs @@ -112,12 +112,19 @@ impl BrowserOutput { /// Observe + screenshot the session into a success output. Any capture /// error is folded into `error` rather than failing the whole tool call. - async fn capture(profile: &str, session: &BrowserSession, full_page: bool) -> Self { + /// `include_text` false suppresses the (often large, step-to-step + /// redundant) page-text dump, keeping the screenshot and element list. + async fn capture( + profile: &str, + session: &BrowserSession, + full_page: bool, + include_text: bool, + ) -> Self { // Let a navigation triggered by the preceding action settle first, so // the text (`observe`) and the screenshot show the same page rather // than racing a mid-transition document. session.settle().await; - let observation = session.observe().await.ok(); + let observation = session.observe_with(include_text).await.ok(); let screenshot_base64 = match session.screenshot(full_page).await { Ok(png) => Some(base64::engine::general_purpose::STANDARD.encode(png)), Err(_) => None, @@ -303,7 +310,7 @@ impl Tool for BrowserNavigateTool { format!("Navigation failed: {e}"), )); } - Ok(BrowserOutput::capture(&profile, &session, false).await) + Ok(BrowserOutput::capture(&profile, &session, false, true).await) } } @@ -318,6 +325,11 @@ pub struct BrowserReadInput { /// Capture the entire scrollable page instead of just the viewport. #[serde(default, skip_serializing_if = "std::ops::Not::not")] pub full_page: bool, + /// Omit the page's full text dump, returning only the screenshot and the + /// interactive-element list. Cuts token cost on long forms where the text + /// barely changes between steps. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub no_text: bool, } pub struct BrowserReadTool; @@ -342,7 +354,8 @@ impl Tool for BrowserReadTool { "type": "object", "properties": { "profile": {"type": "string", "description": "Profile to read; omit for the throwaway browser"}, - "full_page": {"type": "boolean", "description": "Capture the whole scrollable page instead of just the viewport"} + "full_page": {"type": "boolean", "description": "Capture the whole scrollable page instead of just the viewport"}, + "no_text": {"type": "boolean", "description": "Omit the page-text dump; return only the screenshot and the interactive-element list (saves tokens on long forms)"} } }), annotations: Some(json!({"readOnlyHint": true})), @@ -367,7 +380,12 @@ impl Tool for BrowserReadTool { return Ok(BrowserOutput::unavailable(&profile)); }; match manager.get_by_label(&profile) { - Some(session) => Ok(BrowserOutput::capture(&profile, &session, input.full_page).await), + Some(session) => { + Ok( + BrowserOutput::capture(&profile, &session, input.full_page, !input.no_text) + .await, + ) + } None => Ok(BrowserOutput::failure( &profile, "No open browser for this profile. Use browser_navigate first.", @@ -386,12 +404,19 @@ impl Tool for BrowserReadTool { pub enum BrowserAction { /// Click the first element matching the CSS selector. Click { selector: String }, - /// Focus a field and type text into it. Not for credentials — use - /// `browser_login` for those. + /// Focus a field and type text into it, appending to any existing content. + /// Not for credentials — use `browser_login` for those. Prefer `fill` to + /// replace a prefilled field. Type { selector: String, text: String }, - /// Press a key (e.g. "Enter") on the element matching the selector. The - /// element is focused first. Omit `selector` to send the key to whatever is - /// currently focused (e.g. arrow keys for a focused game canvas). + /// Replace a field's content: clear it, then type `text`. The editing verb + /// for prefilled inputs (no manual clearing needed). + Fill { selector: String, text: String }, + /// Empty a field's current content. + Clear { selector: String }, + /// Press a key or chord (e.g. "Enter", "Meta+A", "Control+a", "Shift+Tab") + /// on the element matching the selector. The element is focused first. Omit + /// `selector` to send the key to whatever is currently focused (e.g. arrow + /// keys for a focused game canvas, or a chord for select-all). Press { #[serde(default)] selector: Option, @@ -403,8 +428,10 @@ pub enum BrowserAction { #[serde(default)] timeout_ms: Option, }, - /// Scroll the page: with `selector`, scroll that element into view; - /// otherwise scroll by `(dx, dy)` pixels (positive `dy` scrolls down). + /// Scroll: with `selector` and no delta, scroll that element into view; + /// with `selector` **and** a `(dx, dy)` delta, scroll *inside* that element + /// (for a modal/dialog with its own scroll container); with no selector, + /// scroll the page by `(dx, dy)` pixels (positive `dy` scrolls down). Scroll { #[serde(default)] selector: Option, @@ -475,6 +502,11 @@ pub struct BrowserActInput { pub actions: Vec, #[serde(default, skip_serializing_if = "Option::is_none")] pub profile: Option, + /// Omit the page's full text dump from the result, returning only the + /// screenshot and interactive-element list. Cuts token cost when stepping + /// through a long form whose text barely changes. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub no_text: bool, } pub struct BrowserActTool; @@ -484,6 +516,8 @@ impl BrowserActTool { match action { BrowserAction::Click { selector } => session.click(selector).await, BrowserAction::Type { selector, text } => session.type_text(selector, text).await, + BrowserAction::Fill { selector, text } => session.fill(selector, text).await, + BrowserAction::Clear { selector } => session.clear(selector).await, BrowserAction::Press { selector, key } => match selector { Some(sel) => session.press_key(sel, key).await, None => session.press_key_global(key).await, @@ -531,24 +565,37 @@ impl Tool for BrowserActTool { name: "browser_act".into(), description: concat!( "Interact with the current page of a browser profile: a sequence of ", - "click / type / press / wait_for / scroll / click_at / move_at steps, executed ", - "in order. Returns a screenshot and text of the resulting page.\n", + "click / type / fill / clear / press / wait_for / scroll / click_at / move_at ", + "steps, executed in order. Returns a screenshot and text of the resulting page.\n", "Each action is an object with one key: {\"click\": {\"selector\": \"#go\"}}, ", - "{\"type\": {\"selector\": \"#user\", \"text\": \"hello\"}}, ", - "{\"press\": {\"selector\": \"#user\", \"key\": \"Enter\"}} (omit selector to send ", - "the key to whatever is focused, e.g. arrow keys for a game canvas), ", + "{\"type\": {\"selector\": \"#user\", \"text\": \"hello\"}} (appends), ", + "{\"fill\": {\"selector\": \"#user\", \"text\": \"hello\"}} (REPLACES the field's ", + "content — use this to edit a prefilled field), ", + "{\"clear\": {\"selector\": \"#user\"}} (empty a field), ", + "{\"press\": {\"selector\": \"#user\", \"key\": \"Enter\"}} — `key` may be a chord ", + "like \"Meta+A\", \"Control+a\" or \"Shift+Tab\"; omit selector to send the key to ", + "whatever is focused (e.g. arrow keys for a game canvas), ", "{\"wait_for\": {\"selector\": \"#result\", \"timeout_ms\": 5000}}, ", - "{\"scroll\": {\"dy\": 600}} (scroll down 600px) / {\"scroll\": {\"selector\": \"#footer\"}} ", - "(scroll an element into view), ", + "{\"scroll\": {\"dy\": 600}} (scroll the page down 600px), ", + "{\"scroll\": {\"selector\": \"#footer\"}} (scroll an element into view), ", + "{\"scroll\": {\"selector\": \"#dialog\", \"dy\": 400}} (scroll INSIDE a modal/dialog ", + "with its own scroll container), ", "{\"click_at\": {\"x\": \"40vw\", \"y\": \"30vh\"}} (click at a coordinate, for ", "canvas/WebGL surfaces without selectors), or {\"move_at\": {\"x\": \"40vw\", ", "\"y\": \"30vh\"}} (move the mouse for hover/pointer-move).\n", + "Selectors are CSS by default, but you can also target by visible text/role, ", + "which survives a re-render that changes hashed ids: \"text=Speichern\" (visible ", + "text/label, substring, case-insensitive), \"role=button\" or ", + "\"role=button[name=Save]\", \"aria=Save\" (aria-label). Elements are scrolled into ", + "view automatically before acting.\n", "Coordinate values carry a CSS unit — think about which unit you mean: \"40vw\"/", "\"30vh\" or \"50%\" express a fraction of the viewport axis (robust — use these ", "when eyeballing from the screenshot); \"640px\" is exact CSS pixels (only when you ", "know the size — the read output shows the Viewport dimensions). rem/em are not ", "accepted. Prefer selectors when available (see the interactive-element list from a ", "read); use coordinates only for canvas/game surfaces. ", + "Pass \"no_text\": true to omit the page-text dump from the result (screenshot and ", + "element list only) to save tokens on long forms. ", "Do not type passwords or 2FA codes here — use browser_login." ) .into(), @@ -563,7 +610,9 @@ impl Tool for BrowserActTool { "properties": { "click": {"type": "object", "properties": {"selector": {"type": "string"}}, "required": ["selector"]}, "type": {"type": "object", "properties": {"selector": {"type": "string"}, "text": {"type": "string"}}, "required": ["selector", "text"]}, - "press": {"type": "object", "properties": {"selector": {"type": "string"}, "key": {"type": "string"}}, "required": ["key"]}, + "fill": {"type": "object", "properties": {"selector": {"type": "string"}, "text": {"type": "string"}}, "required": ["selector", "text"]}, + "clear": {"type": "object", "properties": {"selector": {"type": "string"}}, "required": ["selector"]}, + "press": {"type": "object", "properties": {"selector": {"type": "string"}, "key": {"type": "string", "description": "A key or chord, e.g. \"Enter\", \"Meta+A\", \"Control+a\", \"Shift+Tab\""}}, "required": ["key"]}, "wait_for": {"type": "object", "properties": {"selector": {"type": "string"}, "timeout_ms": {"type": "integer"}}, "required": ["selector"]}, "scroll": {"type": "object", "properties": {"selector": {"type": "string"}, "dx": {"type": "number"}, "dy": {"type": "number"}}}, "click_at": {"type": "object", "properties": {"x": {"type": "string", "description": "x coordinate with CSS unit: vw/% (of width) or px"}, "y": {"type": "string", "description": "y coordinate with CSS unit: vh/% (of height) or px"}}, "required": ["x", "y"]}, @@ -571,7 +620,8 @@ impl Tool for BrowserActTool { } } }, - "profile": {"type": "string", "description": "Profile to act on; omit for the throwaway browser"} + "profile": {"type": "string", "description": "Profile to act on; omit for the throwaway browser"}, + "no_text": {"type": "boolean", "description": "Omit the page-text dump from the result (screenshot and element list only)"} }, "required": ["actions"] }), @@ -607,12 +657,13 @@ impl Tool for BrowserActTool { if let Err(e) = Self::run_action(&session, action).await { // Capture the page as it stands so the model can see where the // sequence stopped, but report the failing step. - let mut out = BrowserOutput::capture(&profile, &session, false).await; + let mut out = + BrowserOutput::capture(&profile, &session, false, !input.no_text).await; out.error = Some(format!("Action {} failed: {e}", i + 1)); return Ok(out); } } - Ok(BrowserOutput::capture(&profile, &session, false).await) + Ok(BrowserOutput::capture(&profile, &session, false, !input.no_text).await) } } @@ -767,7 +818,7 @@ async fn login_handoff( // Note the login so a later session can discover it via // browser_profiles instead of asking the user to log in again. record_login_in(&profiles_root(), profile, url); - Ok(BrowserOutput::capture(profile, &headless, false).await) + Ok(BrowserOutput::capture(profile, &headless, false, true).await) } Err(e) => Ok(BrowserOutput::failure( profile, @@ -1294,6 +1345,7 @@ mod tests { }, ], profile: None, + no_text: false, }; let out = BrowserActTool.execute(&mut context, &mut act).await?; assert!(out.error.is_none(), "act error: {:?}", out.error); @@ -1302,6 +1354,7 @@ mod tests { let mut read = BrowserReadInput { profile: None, full_page: false, + no_text: false, }; let out = BrowserReadTool.execute(&mut context, &mut read).await?; let obs = out.observation.as_ref().expect("observation"); @@ -1326,6 +1379,140 @@ mod tests { Ok(()) } + #[tokio::test] + async fn fill_clear_and_no_text_work_through_the_act_tool() -> Result<()> { + // A page with a prefilled input; fill should replace, clear should empty. + let html = concat!( + "Form", + "

Editing

", + "", + "" + ); + let url = format!( + "data:text/html;base64,{}", + base64::engine::general_purpose::STANDARD.encode(html) + ); + + let mut fixture = ToolTestFixture::new().with_browser_sessions(); + { + let mut context = fixture.context(); + let mut nav = BrowserNavigateInput { url, profile: None }; + BrowserNavigateTool.execute(&mut context, &mut nav).await?; + + // Fill replaces the prefilled value, and no_text suppresses the dump + // while still returning the element list. + let mut act = BrowserActInput { + actions: vec![BrowserAction::Fill { + selector: "#name".into(), + text: "replaced".into(), + }], + profile: None, + no_text: true, + }; + let out = BrowserActTool.execute(&mut context, &mut act).await?; + assert!(out.error.is_none(), "fill error: {:?}", out.error); + let obs = out.observation.as_ref().expect("observation"); + assert!(obs.text.is_empty(), "no_text should suppress the text dump"); + assert!( + obs.elements.iter().any(|e| e.selector == "#name"), + "elements should still be present with no_text" + ); + } + + let session = fixture + .browser_sessions() + .unwrap() + .get_by_label("default") + .unwrap(); + let value = session + .eval("document.getElementById('name').value") + .await?; + assert_eq!( + value.as_str().unwrap_or_default(), + "replaced", + "fill should replace the prefilled value" + ); + + // Clear empties the field. + { + let mut context = fixture.context(); + let mut act = BrowserActInput { + actions: vec![BrowserAction::Clear { + selector: "#name".into(), + }], + profile: None, + no_text: false, + }; + let out = BrowserActTool.execute(&mut context, &mut act).await?; + assert!(out.error.is_none(), "clear error: {:?}", out.error); + } + let value = session + .eval("document.getElementById('name').value") + .await?; + assert_eq!(value.as_str().unwrap_or_default(), "", "clear should empty"); + + Ok(()) + } + + #[tokio::test] + async fn act_resolves_a_text_selector_and_a_chord() -> Result<()> { + // A button targeted by visible text, and a chord recorded on keydown. + let html = concat!( + "t", + "", + "", + "", + "" + ); + let url = format!( + "data:text/html;base64,{}", + base64::engine::general_purpose::STANDARD.encode(html) + ); + + let mut fixture = ToolTestFixture::new().with_browser_sessions(); + { + let mut context = fixture.context(); + let mut nav = BrowserNavigateInput { url, profile: None }; + BrowserNavigateTool.execute(&mut context, &mut nav).await?; + + let mut act = BrowserActInput { + actions: vec![ + BrowserAction::Press { + selector: Some("#f".into()), + key: "Control+a".into(), + }, + BrowserAction::Click { + selector: "text=Speichern".into(), + }, + ], + profile: None, + no_text: false, + }; + let out = BrowserActTool.execute(&mut context, &mut act).await?; + assert!(out.error.is_none(), "act error: {:?}", out.error); + } + + let session = fixture + .browser_sessions() + .unwrap() + .get_by_label("default") + .unwrap(); + let title = session.eval("document.title").await?; + assert_eq!( + title.as_str().unwrap_or_default(), + "hit", + "text= selector should have clicked the button" + ); + let log = session + .eval("document.getElementById('log').textContent") + .await?; + assert!( + log.as_str().unwrap_or_default().contains("/true"), + "Control+a chord should set ctrlKey, got: {log}" + ); + Ok(()) + } + #[tokio::test] async fn scroll_moves_the_viewport_and_full_page_capture_works() -> Result<()> { // A page much taller than the viewport. @@ -1352,6 +1539,7 @@ mod tests { dy: Some(1200.0), }], profile: None, + no_text: false, }; let out = BrowserActTool.execute(&mut context, &mut act).await?; assert!(out.error.is_none(), "scroll error: {:?}", out.error); @@ -1418,6 +1606,7 @@ mod tests { y: "80px".into(), }], profile: None, + no_text: false, }; let out = BrowserActTool.execute(&mut context, &mut act).await?; assert!(out.error.is_none(), "click_at error: {:?}", out.error); @@ -1450,6 +1639,7 @@ mod tests { key: "a".into(), }], profile: None, + no_text: false, }; let out = BrowserActTool.execute(&mut context, &mut act).await?; assert!(out.error.is_none(), "global press error: {:?}", out.error); @@ -1551,6 +1741,7 @@ mod tests { let mut read = BrowserReadInput { profile: None, full_page: false, + no_text: false, }; let out = BrowserReadTool.execute(&mut context, &mut read).await?; assert!(out.error.unwrap().contains("browser_navigate")); diff --git a/crates/web/Cargo.toml b/crates/web/Cargo.toml index d03623ec..920d5265 100644 --- a/crates/web/Cargo.toml +++ b/crates/web/Cargo.toml @@ -22,3 +22,4 @@ url = "2.5" [dev-dependencies] axum = "0.7" +base64 = "0.22" diff --git a/crates/web/src/browser_session.rs b/crates/web/src/browser_session.rs index 2573c185..a8b442ee 100644 --- a/crates/web/src/browser_session.rs +++ b/crates/web/src/browser_session.rs @@ -13,9 +13,11 @@ use crate::browser::LaunchedBrowser; use anyhow::Result; +use chromiumoxide::cdp::browser_protocol::input::{DispatchKeyEventParams, DispatchKeyEventType}; use chromiumoxide::cdp::browser_protocol::network::{CookieParam, CookieSameSite, TimeSinceEpoch}; use chromiumoxide::cdp::browser_protocol::page::CaptureScreenshotFormat; use chromiumoxide::element::Element; +use chromiumoxide::keys::get_key_definition; use chromiumoxide::layout::Point; use chromiumoxide::page::{Page, ScreenshotParams}; use std::collections::HashMap; @@ -27,10 +29,14 @@ use tokio::sync::Mutex as AsyncMutex; /// array of `{selector, role, label}`. Best-effort: it prefers `#id` selectors, /// falls back to an `:nth-of-type` path, skips hidden/disabled elements, and is /// bounded so a huge page can't blow up the observation. +/// +/// It also descends into open shadow roots and puts elements that live inside a +/// modal/dialog (or a fixed high-z-index overlay) first, so a dialog's buttons +/// are never dropped by the cap when the page behind it is long. const DISCOVER_ELEMENTS_JS: &str = r#" (() => { - const MAX = 40; - const SEL = 'a,button,input,textarea,select,summary,[role=button],[role=link],[role=checkbox],[role=tab],[role=menuitem],[onclick],[tabindex]'; + const MAX = 200; + const SEL = 'a,button,input,textarea,select,summary,[role=button],[role=link],[role=checkbox],[role=tab],[role=menuitem],[role=menuitemcheckbox],[role=menuitemradio],[role=switch],[role=option],[contenteditable=true],[onclick],[tabindex]'; const seen = new Set(); const out = []; @@ -53,14 +59,14 @@ const DISCOVER_ELEMENTS_JS: &str = r#" let sel = node.tagName.toLowerCase(); if (node.id) { parts.unshift('#' + CSS.escape(node.id)); break; } const parent = node.parentNode; - if (parent) { + if (parent && parent.children) { const sameTag = Array.from(parent.children).filter(c => c.tagName === node.tagName); if (sameTag.length > 1) { sel += ':nth-of-type(' + (sameTag.indexOf(node) + 1) + ')'; } } parts.unshift(sel); - node = node.parentNode; + node = node.parentNode && node.parentNode.host ? node.parentNode.host : node.parentNode; } return parts.join(' > '); }; @@ -85,7 +91,42 @@ const DISCOVER_ELEMENTS_JS: &str = r#" return l.slice(0, 80); }; - for (const el of document.querySelectorAll(SEL)) { + // Rank: elements inside a dialog / high-z fixed overlay first, so the + // topmost interactive surface is never truncated away. + const inDialog = (el) => { + let node = el; + while (node && node.nodeType === 1) { + const role = node.getAttribute && node.getAttribute('role'); + if (node.tagName === 'DIALOG' || role === 'dialog' || role === 'alertdialog' || node.getAttribute && node.getAttribute('aria-modal') === 'true') { + return true; + } + if (node.parentNode && node.parentNode.host) { node = node.parentNode.host; continue; } + node = node.parentNode; + } + return false; + }; + + // Gather across the main document and any open shadow roots. + const collect = (root, acc) => { + let nodes = []; + try { nodes = Array.from(root.querySelectorAll(SEL)); } catch (e) {} + for (const el of nodes) acc.push(el); + let all = []; + try { all = Array.from(root.querySelectorAll('*')); } catch (e) {} + for (const el of all) { + if (el.shadowRoot) collect(el.shadowRoot, acc); + } + }; + + const candidates = []; + collect(document, candidates); + + // Stable sort: dialog elements first, keeping document order otherwise. + const ranked = candidates + .map((el, i) => ({ el, i, dlg: inDialog(el) ? 0 : 1 })) + .sort((a, b) => (a.dlg - b.dlg) || (a.i - b.i)); + + for (const { el } of ranked) { if (out.length >= MAX) break; if (!visible(el)) continue; const selector = cssPath(el); @@ -191,20 +232,44 @@ impl BrowserSession { Ok(self.page.screenshot(params).await?) } - /// Scroll the page. With a `selector`, scroll that element into view; - /// otherwise scroll by `(dx, dy)` pixels relative to the current position - /// (positive `dy` scrolls down). The selector is JSON-encoded into the - /// script, so it cannot break out of the string. + /// Scroll the page or an element. With a `selector` and no delta, scroll + /// that element into view. With a `selector` **and** a non-zero `(dx, dy)`, + /// scroll *inside* that element — the fix for modal/dialog content that + /// lives in its own scroll container (a plain `window.scrollBy` moves the + /// page behind it, not the dialog). With no selector, scroll the page by + /// `(dx, dy)` (positive `dy` scrolls down). The selector is JSON-encoded + /// into the script, so it cannot break out of the string. pub async fn scroll(&self, selector: Option<&str>, dx: f64, dy: f64) -> Result<()> { match selector { Some(sel) => { let sel_json = serde_json::to_string(sel)?; - let js = format!( - "(() => {{ const e = document.querySelector({sel_json}); \ - if (!e) return false; \ - e.scrollIntoView({{block: 'center', inline: 'center'}}); \ - return true; }})()" - ); + let js = if dx == 0.0 && dy == 0.0 { + format!( + "(() => {{ const e = document.querySelector({sel_json}); \ + if (!e) return false; \ + e.scrollIntoView({{block: 'center', inline: 'center'}}); \ + return true; }})()" + ) + } else { + // Scroll within the element's own scroll container (or the + // nearest scrollable ancestor if the element itself does not + // scroll), so a dialog's inner content moves. + format!( + "(() => {{ let e = document.querySelector({sel_json}); \ + if (!e) return false; \ + const scrollable = (n) => {{ \ + while (n && n !== document.body) {{ \ + const s = getComputedStyle(n); \ + if (/(auto|scroll)/.test(s.overflowY + s.overflow) && n.scrollHeight > n.clientHeight) return n; \ + n = n.parentElement; \ + }} \ + return e; \ + }}; \ + const target = (e.scrollHeight > e.clientHeight) ? e : scrollable(e); \ + target.scrollBy({dx}, {dy}); \ + return true; }})()" + ) + }; let found = self .page .evaluate(js) @@ -227,14 +292,25 @@ impl BrowserSession { /// Read the current location, title, visible text, and the actionable /// elements on the page. pub async fn observe(&self) -> Result { + self.observe_with(true).await + } + + /// Like [`observe`](Self::observe), but `include_text` can suppress the + /// (often large and redundant) `innerText` dump — the model keeps the + /// screenshot plus the interactive-element list, and avoids re-reading a + /// long form's text on every step. + pub async fn observe_with(&self, include_text: bool) -> Result { let url = self.page.url().await?.unwrap_or_default(); let title = self.page.get_title().await?.unwrap_or_default(); - let text = self - .page - .evaluate("document.body ? document.body.innerText : ''") - .await? - .into_value::() - .unwrap_or_default(); + let text = if include_text { + self.page + .evaluate("document.body ? document.body.innerText : ''") + .await? + .into_value::() + .unwrap_or_default() + } else { + String::new() + }; // Element discovery is best-effort: a failure (e.g. mid-navigation) // just yields an empty list rather than failing the observation. let elements = match self.page.evaluate(DISCOVER_ELEMENTS_JS).await { @@ -267,18 +343,119 @@ impl BrowserSession { Ok(dims) } - /// Find an element, turning chromiumoxide's opaque CDP miss ("Could not - /// find node with given id") into a message that names the selector. + /// Find an element by a selector, turning chromiumoxide's opaque CDP miss + /// ("Could not find node with given id") into a message that names the + /// selector. + /// + /// Besides plain CSS, this understands robust prefixes that survive a site + /// re-rendering with fresh hashed ids (a real problem on portals like + /// ELSTER): + /// - `text=Foo` — the first visible element whose trimmed text / + /// aria-label / value contains `Foo` (case-insensitive). + /// - `role=button` — the first element with that ARIA role (or, for a bare + /// tag, that tag). `role=button[name=Save]` also matches on text. + /// - `aria=Save` — the first element whose aria-label matches. + /// + /// These are resolved to a concrete node in the page, so a fragile hashed + /// `#id` is never needed. async fn find(&self, selector: &str) -> Result { + if let Some(css) = self.resolve_semantic_selector(selector).await? { + return self + .page + .find_element(&css) + .await + .map_err(|_| anyhow::anyhow!("no element matches selector '{selector}'")); + } self.page .find_element(selector) .await .map_err(|_| anyhow::anyhow!("no element matches selector '{selector}'")) } - /// Click the first element matching a CSS selector. + /// If `selector` uses a semantic prefix (`text=`, `role=`, `aria=`), locate + /// the matching element in the page and stamp it with a unique data + /// attribute, returning a concrete CSS selector for it. Returns `Ok(None)` + /// for a plain CSS selector (handled directly by the caller). + async fn resolve_semantic_selector(&self, selector: &str) -> Result> { + let sel = selector.trim(); + let (kind, query) = if let Some(q) = sel.strip_prefix("text=") { + ("text", q) + } else if let Some(q) = sel.strip_prefix("role=") { + ("role", q) + } else if let Some(q) = sel.strip_prefix("aria=") { + ("aria", q) + } else { + return Ok(None); + }; + let kind_json = serde_json::to_string(kind)?; + let query_json = serde_json::to_string(query.trim())?; + // Tag the match with a unique attribute so we can hand back a stable CSS + // selector even on a page that mints fresh ids every render. + let js = format!( + r#"(() => {{ + const kind = {kind_json}; + const q = {query_json}; + const SEL = 'a,button,input,textarea,select,summary,[role],[onclick],[tabindex],label'; + const norm = (s) => (s || '').replace(/\s+/g, ' ').trim().toLowerCase(); + const visible = (el) => {{ + const r = el.getClientRects(); + if (!r.length) return false; + const st = getComputedStyle(el); + return st.visibility !== 'hidden' && st.display !== 'none'; + }}; + const labelText = (el) => norm(el.getAttribute('aria-label')) || norm(el.textContent) || norm(el.value) || norm(el.getAttribute('placeholder')) || norm(el.title); + const roleOf = (el) => (el.getAttribute('role') || el.tagName.toLowerCase()); + // role=button[name=Save] → role + optional name filter. + let wantRole = q, wantName = null; + const m = q.match(/^([^\[]+)\[name=(.+)\]$/); + if (kind === 'role' && m) {{ wantRole = m[1].trim(); wantName = norm(m[2]); }} + const needle = norm(q); + const collect = (root, acc) => {{ + let nodes = []; + try {{ nodes = Array.from(root.querySelectorAll(SEL)); }} catch (e) {{}} + for (const el of nodes) acc.push(el); + let all = []; + try {{ all = Array.from(root.querySelectorAll('*')); }} catch (e) {{}} + for (const el of all) if (el.shadowRoot) collect(el.shadowRoot, acc); + }}; + const cands = []; + collect(document, cands); + const match = cands.find((el) => {{ + if (!visible(el)) return false; + if (kind === 'text') return labelText(el).includes(needle); + if (kind === 'aria') return norm(el.getAttribute('aria-label')).includes(needle); + if (kind === 'role') {{ + if (norm(roleOf(el)) !== norm(wantRole)) return false; + return wantName ? labelText(el).includes(wantName) : true; + }} + return false; + }}); + if (!match) return null; + const token = 'ca-sel-' + Math.random().toString(36).slice(2); + match.setAttribute('data-ca-sel', token); + return '[data-ca-sel="' + token + '"]'; +}})()"# + ); + let resolved = self + .page + .evaluate(js) + .await? + .into_value::>() + .unwrap_or(None); + match resolved { + Some(css) => Ok(Some(css)), + None => Err(anyhow::anyhow!("no element matches selector '{selector}'")), + } + } + + /// Click the first element matching a selector. The element is scrolled + /// into view first (via `scrollIntoView`, which handles nested scroll + /// containers), so an off-screen or collapsed target no longer fails with + /// "Node is either not visible or not an HTMLElement". pub async fn click(&self, selector: &str) -> Result<()> { - self.find(selector).await?.click().await?; + let element = self.find(selector).await?; + let _ = element.scroll_into_view().await; + element.click().await?; Ok(()) } @@ -296,31 +473,141 @@ impl BrowserSession { Ok(()) } - /// Focus a field and type text into it. Never used for credentials — those - /// go through the human-in-the-loop login handoff. + /// Focus a field and type text into it, appending to any existing value. + /// The element is scrolled into view first. Never used for credentials — + /// those go through the human-in-the-loop login handoff. Prefer + /// [`fill`](Self::fill) to replace a prefilled field. pub async fn type_text(&self, selector: &str, text: &str) -> Result<()> { let element = self.find(selector).await?; + let _ = element.scroll_into_view().await; element.focus().await?; element.type_str(text).await?; Ok(()) } - /// Press a key (e.g. `"Enter"`) on the element matching a selector. The - /// element is focused first so the key event lands on it — CDP dispatches - /// key events to whatever currently has focus, not to a named node. - pub async fn press_key(&self, selector: &str, key: &str) -> Result<()> { + /// Clear a field, then type `text` — the replace semantics editing a + /// prefilled input needs (the old workflow required End + repeated + /// Backspace). Works for ``/`