From cd2a690ce9ec6706d3741f39da3f564228593a15 Mon Sep 17 00:00:00 2001 From: HarshDevelops Date: Sun, 19 Jul 2026 08:32:26 +0530 Subject: [PATCH] #108 Normalize misspelled HTTP headers --- .../org/apache/stormcrawler/Metadata.java | 86 +++++++++++++++++++ .../org/apache/stormcrawler/MetadataTest.java | 19 ++++ 2 files changed, 105 insertions(+) diff --git a/core/src/main/java/org/apache/stormcrawler/Metadata.java b/core/src/main/java/org/apache/stormcrawler/Metadata.java index 00de145ed..6bf673eda 100644 --- a/core/src/main/java/org/apache/stormcrawler/Metadata.java +++ b/core/src/main/java/org/apache/stormcrawler/Metadata.java @@ -20,6 +20,8 @@ import com.esotericsoftware.kryo.serializers.DefaultArraySerializers.StringArraySerializer; import com.esotericsoftware.kryo.serializers.DefaultSerializers.StringSerializer; import com.esotericsoftware.kryo.serializers.MapSerializer.BindMap; +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; import java.util.Collection; import java.util.Collections; import java.util.ConcurrentModificationException; @@ -30,10 +32,17 @@ import java.util.Set; import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; +import org.apache.http.HttpHeaders; /** Wrapper around Map <String,String[]>. * */ public class Metadata { + private static final int HTTP_HEADER_SPELLCHECK_THRESHOLD_DIVIDER = 3; + private static final Map HTTP_HEADERS_BY_NORMALIZED_NAME = + httpHeadersByNormalizedName(); + private static final String[] NORMALIZED_HTTP_HEADER_NAMES = + HTTP_HEADERS_BY_NORMALIZED_NAME.keySet().toArray(new String[0]); + // customize the behaviour of Kryo via annotations @BindMap( valueSerializer = StringArraySerializer.class, @@ -309,9 +318,86 @@ public Metadata unlock() { } private static String normalizeKey(String key) { + String normalizedHttpHeader = normalizeHttpHeader(key); + if (normalizedHttpHeader != null) { + return normalizedHttpHeader.toLowerCase(Locale.ROOT); + } return key.toLowerCase(Locale.ROOT); } + private static String normalizeHttpHeader(String key) { + String normalizedKey = normalizeHttpHeaderName(key); + String header = HTTP_HEADERS_BY_NORMALIZED_NAME.get(normalizedKey); + + if (header == null) { + int threshold = + Math.min(3, normalizedKey.length() / HTTP_HEADER_SPELLCHECK_THRESHOLD_DIVIDER); + if (threshold == 0) { + return null; + } + for (String normalizedHeader : NORMALIZED_HTTP_HEADER_NAMES) { + if (levenshteinDistance(normalizedKey, normalizedHeader) < threshold) { + header = HTTP_HEADERS_BY_NORMALIZED_NAME.get(normalizedHeader); + break; + } + } + } + return header; + } + + private static int levenshteinDistance(String source, String target) { + int[] previous = new int[target.length() + 1]; + int[] current = new int[target.length() + 1]; + for (int j = 0; j <= target.length(); j++) { + previous[j] = j; + } + for (int i = 1; i <= source.length(); i++) { + current[0] = i; + for (int j = 1; j <= target.length(); j++) { + int substitution = source.charAt(i - 1) == target.charAt(j - 1) ? 0 : 1; + current[j] = + Math.min( + Math.min(current[j - 1] + 1, previous[j] + 1), + previous[j - 1] + substitution); + } + int[] tmp = previous; + previous = current; + current = tmp; + } + return previous[target.length()]; + } + + private static String normalizeHttpHeaderName(String key) { + StringBuilder normalized = new StringBuilder(key.length()); + for (int i = 0; i < key.length(); i++) { + char c = key.charAt(i); + if (Character.isLetter(c)) { + normalized.append(Character.toLowerCase(c)); + } + } + return normalized.toString(); + } + + private static Map httpHeadersByNormalizedName() { + Map headers = new HashMap<>(); + for (Field field : HttpHeaders.class.getFields()) { + int modifiers = field.getModifiers(); + if (!Modifier.isFinal(modifiers) + || !Modifier.isPublic(modifiers) + || !Modifier.isStatic(modifiers) + || !field.getType().equals(String.class)) { + continue; + } + try { + String value = (String) field.get(null); + headers.put(normalizeHttpHeaderName(value), value); + } catch (IllegalAccessException e) { + // Ignore inaccessible constants. + } + } + return Collections.unmodifiableMap(headers); + } + /** * @since 1.16 */ diff --git a/core/src/test/java/org/apache/stormcrawler/MetadataTest.java b/core/src/test/java/org/apache/stormcrawler/MetadataTest.java index 033b88469..f613495cb 100644 --- a/core/src/test/java/org/apache/stormcrawler/MetadataTest.java +++ b/core/src/test/java/org/apache/stormcrawler/MetadataTest.java @@ -52,6 +52,25 @@ void testAddValueIsCaseInsensitive() { Assertions.assertEquals(1, metadata.size()); } + @Test + void testHttpHeadersAreSpellChecked() { + Metadata metadata = new Metadata(); + metadata.addValue("ContentType", "text/html"); + metadata.addValue("ConTnTtYpe", "application/xhtml+xml"); + + Assertions.assertEquals(2, metadata.getValues("Content-Type").length); + Assertions.assertTrue(metadata.containsKey("content-type")); + } + + @Test + void testNonHttpHeadersAreNotSpellChecked() { + Metadata metadata = new Metadata(); + metadata.addValue("customContentType", "value"); + + Assertions.assertNull(metadata.getFirstValue("Content-Type")); + Assertions.assertEquals("value", metadata.getFirstValue("customContentType")); + } + @Test void testRemoveIsCaseInsensitive() { Metadata metadata = new Metadata();