From d5987a12ccb743a557689e218ac420883fdba6a1 Mon Sep 17 00:00:00 2001 From: Julien Nioche Date: Tue, 21 Jul 2026 17:31:03 +0100 Subject: [PATCH] Resolve relative outlinks against in JSoupParserBolt Relative links were resolved against the document URL, ignoring any element. On a page served from a sub-path with a base pointing at the domain root this doubled the path segment. Resolve against jsoupDoc.baseUri() instead, which honours and falls back to the document URL when absent. --- .../stormcrawler/bolt/JSoupParserBolt.java | 11 ++++++- .../bolt/JSoupParserBoltTest.java | 33 +++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/stormcrawler/bolt/JSoupParserBolt.java b/core/src/main/java/org/apache/stormcrawler/bolt/JSoupParserBolt.java index 3d63fd6cb..577d828fe 100644 --- a/core/src/main/java/org/apache/stormcrawler/bolt/JSoupParserBolt.java +++ b/core/src/main/java/org/apache/stormcrawler/bolt/JSoupParserBolt.java @@ -338,7 +338,16 @@ public void execute(Tuple tuple) { } else { final Elements links = jsoupDoc.select("a[href]"); slinks = new HashMap<>(links.size()); - final URL baseUrl = URLUtil.toURL(url); + // Resolve relative links against the document's base URI + // rather than its URL: when the page contains a + // element jsoup updates baseUri() accordingly, which is what + // browsers use to resolve relative hrefs. Falls back to the + // document URL when no (valid) is present. + String baseUri = jsoupDoc.baseUri(); + if (StringUtils.isBlank(baseUri)) { + baseUri = url; + } + final URL baseUrl = URLUtil.toURL(baseUri); for (Element link : links) { // nofollow String[] relkeywords = link.attr("rel").split(" "); diff --git a/core/src/test/java/org/apache/stormcrawler/bolt/JSoupParserBoltTest.java b/core/src/test/java/org/apache/stormcrawler/bolt/JSoupParserBoltTest.java index 100344d4c..05ae595cc 100644 --- a/core/src/test/java/org/apache/stormcrawler/bolt/JSoupParserBoltTest.java +++ b/core/src/test/java/org/apache/stormcrawler/bolt/JSoupParserBoltTest.java @@ -304,4 +304,37 @@ void testExecuteWithJavascriptLink() throws IOException { Assertions.assertEquals( "http://www.javascriptlinks.com/mylink", statusTuples.get(0).get(0)); } + + /** + * Relative outlinks must be resolved against the document's {@code } element when + * one is present, matching browser behaviour, rather than against the document URL. Otherwise a + * page served from a sub-path with a base pointing at the domain root doubles the path segment. + */ + @Test + void testBaseHrefOutlinkResolution() throws IOException { + bolt.prepare(stormConf, TestUtil.getMockedTopologyContext(), new OutputCollector(output)); + + String html = + "" + + "" + + "" + + "Stockage" + + ""; + + Metadata metadata = new Metadata(); + metadata.setValue("Content-Type", "text/html; charset=UTF-8"); + parse( + "https://www.example.be/fr/rayons/actualites-reportages/216943", + html.getBytes(StandardCharsets.UTF_8), + metadata); + + List> statusTuples = output.getEmitted(Constants.StatusStreamName); + Assertions.assertEquals(1, statusTuples.size()); + Assertions.assertEquals(Status.DISCOVERED, statusTuples.get(0).get(2)); + // resolved against the (domain root), NOT against the document path, + // which would have produced .../actualites-reportages/fr/rayons/stockage-... + Assertions.assertEquals( + "https://www.example.be/fr/rayons/stockage-archivage-de-donnees/218725", + statusTuples.get(0).get(0)); + } }