web_search: fetch a specific URL directly via Venice's scrape endpoint

docs/dev/tools.md

@@ -93,12 +93,20 @@ Notable members (the full ordered list is `alwaysOnToolbox` in
   `docs/dev/` for "how would I add feature X" planning questions.
   The edge implementation reads a build-time bundle, not the
   filesystem - see Interactions, "Help / user docs."
-- `web_search` - runs a one-shot Venice sub-completion with
+- `web_search` - two retrieval modes behind one name. With `query`,
+  runs a one-shot Venice sub-completion with
   `enable_web_search: 'on'` + `enable_web_citations: true` and
-  returns `{answer, citations}`. Always-on because time-sensitive
-  questions (news, prices, today's weather) are the canonical case
-  for search and we don't want the model to refuse or hedge while
-  waiting for a toolbox flip. Read-only (no DB writes).
+  returns `{answer, citations}`. With `url`, posts the link to
+  Venice's `/augment/scrape` endpoint (`veniceScrapeUrl` in
+  `_shared/venice.ts`) and returns `{url, content}` - the page as
+  markdown, capped at 32k chars with a `truncated` flag, plus a
+  single self-citation so the page shows in the reply's sources
+  panel. The split exists because the search pipeline is built
+  around queries and searches FOR a bare URL instead of reading
+  it. Always-on because time-sensitive questions (news, prices,
+  today's weather) are the canonical case for search and we don't
+  want the model to refuse or hedge while waiting for a toolbox
+  flip. Read-only (no DB writes).
   Deliberately absent from every agent toolbox - background agents
   have no reason to reach for live web data, and giving them the
   tool would burn search quota and pollute memories with scraped

docs/user/chat.md

@@ -109,6 +109,13 @@ cited sources) expands the same panel on demand. Each row shows
 the title, date where provided, and a short snippet; click the
 title to open the original page in a new tab.
 
+The same tool also handles direct links: paste a URL and ask about
+it, and instead of searching, Nak fetches that one page and reads
+its content directly. Very long pages are truncated, and the model
+is told when that happened. Some sites block automated fetching
+(X/Twitter and Reddit among them); those come back as a tool error
+the model will explain rather than guess around.
+
 There is no on/off toggle for web search - the model decides per
 turn. Questions that don't need current facts won't trigger the
 tool, so they never pay the latency or quota cost of a search.

src/lib/tools/web_search.schema.ts

@@ -1,34 +1,48 @@
 /**
- * Schema-only export for web_search. Impl lives in `./web_search`.
+ * Schema-only export for web_search. Impl lives in
+ * `supabase/functions/venice/tools/web_search.ts`.
  */
 export const webSearchSchema = {
   name: 'web_search',
   description:
-    'Search the live web and return a synthesized answer with source ' +
-    'citations. Use whenever the question benefits from current facts ' +
-    "(news, prices, scores, releases past your training cutoff, " +
-    "today's weather, anything time-sensitive). Phrase query like a " +
-    'search-engine query, not a sentence to the user. Optional ' +
-    'context_hint (1-2 sentences) keeps the sub-search on task. ' +
-    'Returns {answer, citations}; citations surface automatically on ' +
+    'Reach the live web, two ways. (1) Search: pass `query` and get a ' +
+    'synthesized answer with source citations - use whenever the ' +
+    'question benefits from current facts (news, prices, scores, ' +
+    "releases past your training cutoff, today's weather, anything " +
+    'time-sensitive). Phrase query like a search-engine query, not a ' +
+    'sentence to the user. Optional context_hint (1-2 sentences) keeps ' +
+    'the sub-search on task. (2) Fetch a page: pass `url` when the user ' +
+    'gives you a specific link or you already know the exact page - the ' +
+    'page content comes back directly as markdown (truncated: true ' +
+    'flags a cut-off tail). Never put a URL in `query`; the search ' +
+    'backend searches FOR the URL instead of reading it. Provide query ' +
+    'or url, not both. Returns {answer, citations} for a search and ' +
+    '{url, content} for a fetch; citations surface automatically on ' +
     'your next reply (you may reference them inline as ^N^).',
-  shortDescription: 'search the live web for current facts',
+  shortDescription: 'search the live web or fetch a specific URL',
   parameters: {
     type: 'object',
     properties: {
       query: {
         type: 'string',
         description:
-          'Search-engine-style query (keywords, not a full sentence).',
+          'Search-engine-style query (keywords, not a full sentence). ' +
+          'Omit when passing url.',
       },
       context_hint: {
         type: 'string',
         description:
           'Optional 1-2 sentences of caller context for the ' +
-          'sub-search.',
+          'sub-search. Query mode only.',
+      },
+      url: {
+        type: 'string',
+        description:
+          'Exact http(s) page to fetch directly instead of searching. ' +
+          'Omit when passing query.',
       },
     },
-    required: ['query'],
+    required: [],
     additionalProperties: false,
   },
 } as const;

supabase/functions/_shared/venice.ts

@@ -563,6 +563,78 @@ export async function veniceExtractText(opts: VeniceExtractTextOptions): Promise
   );
 }
 
+export interface VeniceScrapeOptions {
+  apiKey: string;
+  /** The page to fetch. Venice rejects some hosts (X/Twitter, Reddit) with a 400. */
+  url: string;
+  baseUrl?: string;
+  fetchImpl?: typeof fetch;
+  signal?: AbortSignal;
+}
+
+/**
+ * POST /augment/scrape against Venice: fetch one page and return its
+ * content converted to markdown. This is the direct-retrieval
+ * complement to enable_web_search - the search pipeline is built
+ * around queries and does poorly when handed a bare URL, so the
+ * web_search tool routes single-page requests here instead.
+ *
+ * Venice's response is `{ url, content, format: "markdown" }`; only
+ * `content` is returned - the caller already knows the URL it asked
+ * for. An empty content string is surfaced as a parse error rather
+ * than returned: a blank page body gives the model nothing to work
+ * with and reads as success, which is worse than a loud failure.
+ * Error mapping mirrors the sibling helpers: 429 -> rate_limit,
+ * other non-OK -> http (a blocked-host 400's body rides along so the
+ * model can see why), connection failure -> network.
+ */
+export async function veniceScrapeUrl(opts: VeniceScrapeOptions): Promise<string> {
+  const baseUrl = (opts.baseUrl ?? DEFAULT_BASE_URL).replace(/\/$/, '');
+  const fetchImpl = opts.fetchImpl ?? fetch;
+
+  let res: Response;
+  try {
+    res = await fetchImpl(`${baseUrl}/augment/scrape`, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        Authorization: `Bearer ${opts.apiKey}`,
+      },
+      body: JSON.stringify({ url: opts.url }),
+      signal: opts.signal,
+    });
+  } catch (err) {
+    throw new VeniceError(
+      `Network error contacting Venice: ${(err as Error).message}`,
+      'network'
+    );
+  }
+
+  if (!res.ok) {
+    const body = await res.text().catch(() => '');
+    throw new VeniceError(
+      `Venice augment/scrape ${res.status}: ${body.slice(0, 600)}`,
+      res.status === 429 ? 'rate_limit' : 'http',
+      res.status
+    );
+  }
+
+  let payload: unknown;
+  try {
+    payload = await res.json();
+  } catch {
+    throw new VeniceError('Failed to parse Venice scrape response.', 'parse');
+  }
+  const content = (payload as { content?: unknown }).content;
+  if (typeof content !== 'string' || content.length === 0) {
+    throw new VeniceError(
+      'Venice scrape response contained no page content.',
+      'parse'
+    );
+  }
+  return content;
+}
+
 export interface VeniceUsageAnalyticsOptions {
   apiKey: string;
   params: UsageAnalyticsParams;

supabase/functions/tests/venice-scrape.test.ts

@@ -0,0 +1,92 @@
+// Offline unit tests for the Venice /augment/scrape wire-shape. Pure: a
+// fake fetch is injected, so these run under `deno test` with zero network
+// and no Supabase. The web_search tool's url-mode glue (arg validation,
+// truncation, self-citation) sits above this helper in
+// ../venice/tools/web_search.ts.
+import { assertEquals, assertRejects } from '@std/assert';
+import { veniceScrapeUrl, VeniceError } from '../_shared/venice.ts';
+
+Deno.test('veniceScrapeUrl posts the url as JSON and returns the content field', async () => {
+  let captured: { url: string; init: RequestInit } | null = null;
+  const fakeFetch = ((url: string | URL | Request, init?: RequestInit) => {
+    captured = { url: String(url), init: init ?? {} };
+    return Promise.resolve(
+      new Response(
+        JSON.stringify({
+          url: 'https://example.com/post',
+          content: '# A page\n\nbody text',
+          format: 'markdown',
+        }),
+        { status: 200 }
+      )
+    );
+  }) as typeof fetch;
+
+  const out = await veniceScrapeUrl({
+    apiKey: 'test-key',
+    url: 'https://example.com/post',
+    fetchImpl: fakeFetch,
+  });
+
+  assertEquals(out, '# A page\n\nbody text');
+  assertEquals(captured!.url, 'https://api.venice.ai/api/v1/augment/scrape');
+  const headers = captured!.init.headers as Record<string, string>;
+  assertEquals(headers.Authorization, 'Bearer test-key');
+  assertEquals(headers['Content-Type'], 'application/json');
+  assertEquals(
+    JSON.parse(captured!.init.body as string),
+    { url: 'https://example.com/post' }
+  );
+});
+
+Deno.test('veniceScrapeUrl maps a 429 to a rate_limit VeniceError', async () => {
+  const fakeFetch = (() =>
+    Promise.resolve(new Response('slow down', { status: 429 }))) as typeof fetch;
+  const err = await assertRejects(
+    () => veniceScrapeUrl({ apiKey: 'k', url: 'https://a.example', fetchImpl: fakeFetch }),
+    VeniceError
+  );
+  assertEquals(err.kind, 'rate_limit');
+  assertEquals(err.status, 429);
+});
+
+Deno.test('veniceScrapeUrl surfaces a blocked-host 400 body in the http error', async () => {
+  // Venice rejects some hosts (X/Twitter, Reddit) with a 400 whose body
+  // says so; the message must carry it so the model can tell the user
+  // why the fetch failed rather than retrying blindly.
+  const fakeFetch = (() =>
+    Promise.resolve(new Response('this site cannot be scraped', { status: 400 }))) as typeof fetch;
+  const err = await assertRejects(
+    () => veniceScrapeUrl({ apiKey: 'k', url: 'https://x.com/some/post', fetchImpl: fakeFetch }),
+    VeniceError
+  );
+  assertEquals(err.kind, 'http');
+  assertEquals(err.status, 400);
+  assertEquals(err.message.includes('this site cannot be scraped'), true);
+});
+
+Deno.test('veniceScrapeUrl treats an empty content field as a parse error', async () => {
+  // A blank page body reads as success to the caller and gives the model
+  // nothing; fail loud instead.
+  const fakeFetch = (() =>
+    Promise.resolve(
+      new Response(JSON.stringify({ url: 'https://a.example', content: '', format: 'markdown' }), {
+        status: 200,
+      })
+    )) as typeof fetch;
+  const err = await assertRejects(
+    () => veniceScrapeUrl({ apiKey: 'k', url: 'https://a.example', fetchImpl: fakeFetch }),
+    VeniceError
+  );
+  assertEquals(err.kind, 'parse');
+});
+
+Deno.test('veniceScrapeUrl treats a missing content field as a parse error', async () => {
+  const fakeFetch = (() =>
+    Promise.resolve(new Response(JSON.stringify({ ok: true }), { status: 200 }))) as typeof fetch;
+  const err = await assertRejects(
+    () => veniceScrapeUrl({ apiKey: 'k', url: 'https://a.example', fetchImpl: fakeFetch }),
+    VeniceError
+  );
+  assertEquals(err.kind, 'parse');
+});

supabase/functions/venice/tools/web_search.ts

@@ -1,9 +1,18 @@
 // web_search (function-side port)
 //
-// Calls Venice with enable_web_search='on' through a one-shot
-// non-streaming completion against a dedicated search model, harvests
-// the synthesized answer plus web_search_citations, and returns them
-// to the model. Wire schema lives in src/lib/tools/web_search.schema.ts.
+// Two retrieval modes behind one tool name:
+//
+// - query mode: calls Venice with enable_web_search='on' through a
+//   one-shot non-streaming completion against a dedicated search
+//   model, harvests the synthesized answer plus web_search_citations,
+//   and returns them to the model.
+// - url mode: posts the URL to Venice's /augment/scrape endpoint and
+//   returns the page content as markdown, verbatim. The search
+//   pipeline is built around queries and does poorly when handed a
+//   bare URL - it searches FOR the URL instead of reading it - so
+//   direct links skip the search model entirely.
+//
+// Wire schema lives in src/lib/tools/web_search.schema.ts.
 //
 // The search model is mistral-small, a non-reasoning instruct model -
 // the right class here because faithfulness is the priority (a
@@ -19,6 +28,15 @@
 import { registerTool, type ToolContext, type ToolDef } from '../performToolCall.ts';
 import { readVeniceKey } from './_venice_key.ts';
 import { toolComplete } from './_venice_complete.ts';
+import { veniceScrapeUrl } from '../../_shared/venice.ts';
+
+// Ceiling on scraped-page content returned to the model, in
+// characters (~8k tokens - the same order as the query mode's
+// 8196-token answer cap). Scraped pages are unbounded; an article
+// index or a docs page can run hundreds of KB of markdown, which
+// would blow the chat context in one tool round. Truncation is
+// flagged in the result so the model knows the tail is missing.
+const SCRAPE_MAX_CHARS = 32_000;
 
 // Mirror of agentModel('webSearch') in src/lib/models/index.ts. Same
 // same-PR sync discipline as the other browser-mirror constants.
@@ -60,9 +78,20 @@ export const webSearch: ToolDef = {
   name: 'web_search',
   async execute(args: Record<string, unknown>, ctx: ToolContext) {
     const query = typeof args.query === 'string' ? args.query.trim() : '';
-    if (query.length === 0) {
-      throw new Error('web_search requires a non-empty `query` argument');
+    const url = typeof args.url === 'string' ? args.url.trim() : '';
+    if (query.length === 0 && url.length === 0) {
+      throw new Error(
+        'web_search requires either a `query` (to search) or a `url` (to fetch one page)'
+      );
+    }
+
+    if (url.length > 0) {
+      // A URL wins over a query when both arrive: the model naming a
+      // specific page is the stronger signal of intent, and scraping
+      // it answers the query-shaped framing anyway.
+      return await executeScrape(url, ctx);
     }
+
     const contextHint =
       typeof args.context_hint === 'string' ? args.context_hint.trim() : '';
 
@@ -108,4 +137,36 @@ export const webSearch: ToolDef = {
   },
 };
 
+// url mode: fetch one page via Venice's scrape endpoint and hand its
+// markdown to the model raw - no sub-completion, no synthesis. The
+// single self-citation makes the fetched page surface in the reply's
+// sources panel through the same tool-citation harvest the query
+// mode's citations ride (getStreamingResponse.ts).
+async function executeScrape(url: string, ctx: ToolContext) {
+  let parsed: URL;
+  try {
+    parsed = new URL(url);
+  } catch {
+    throw new Error(`web_search: \`url\` is not a valid URL: ${url}`);
+  }
+  if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
+    throw new Error(
+      `web_search: \`url\` must be http(s), got ${parsed.protocol}//`
+    );
+  }
+
+  const apiKey = await readVeniceKey(ctx.adminClient);
+  if (!apiKey) throw new Error('no Venice key configured (app_config unseeded)');
+
+  const content = await veniceScrapeUrl({ apiKey, url, signal: ctx.signal });
+
+  const truncated = content.length > SCRAPE_MAX_CHARS;
+  return {
+    url,
+    content: truncated ? content.slice(0, SCRAPE_MAX_CHARS) : content,
+    ...(truncated ? { truncated: true } : {}),
+    citations: [{ title: url, url, content: null, date: null, index: 1 }],
+  };
+}
+
 registerTool(webSearch);