#108 Normalize misspelled HTTP headers

core/src/main/java/org/apache/stormcrawler/Metadata.java

@@ -20,6 +20,8 @@
 import com.esotericsoftware.kryo.serializers.DefaultArraySerializers.StringArraySerializer;
 import com.esotericsoftware.kryo.serializers.DefaultSerializers.StringSerializer;
 import com.esotericsoftware.kryo.serializers.MapSerializer.BindMap;
+import java.lang.reflect.Field;
+import java.lang.reflect.Modifier;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.ConcurrentModificationException;
@@ -30,10 +32,17 @@
 import java.util.Set;
 import java.util.stream.Collectors;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.http.HttpHeaders;
 
 /** Wrapper around Map <String,String[]>. * */
 public class Metadata {
 
+    private static final int HTTP_HEADER_SPELLCHECK_THRESHOLD_DIVIDER = 3;
+    private static final Map<String, String> HTTP_HEADERS_BY_NORMALIZED_NAME =
+            httpHeadersByNormalizedName();
+    private static final String[] NORMALIZED_HTTP_HEADER_NAMES =
+            HTTP_HEADERS_BY_NORMALIZED_NAME.keySet().toArray(new String[0]);
+
     // customize the behaviour of Kryo via annotations
     @BindMap(
             valueSerializer = StringArraySerializer.class,
@@ -309,9 +318,86 @@ public Metadata unlock() {
     }
 
     private static String normalizeKey(String key) {
+        String normalizedHttpHeader = normalizeHttpHeader(key);
+        if (normalizedHttpHeader != null) {
+            return normalizedHttpHeader.toLowerCase(Locale.ROOT);
+        }
         return key.toLowerCase(Locale.ROOT);
     }
 
+    private static String normalizeHttpHeader(String key) {
+        String normalizedKey = normalizeHttpHeaderName(key);
+        String header = HTTP_HEADERS_BY_NORMALIZED_NAME.get(normalizedKey);
+
+        if (header == null) {
+            int threshold =
+                    Math.min(3, normalizedKey.length() / HTTP_HEADER_SPELLCHECK_THRESHOLD_DIVIDER);
+            if (threshold == 0) {
+                return null;
+            }
+            for (String normalizedHeader : NORMALIZED_HTTP_HEADER_NAMES) {
+                if (levenshteinDistance(normalizedKey, normalizedHeader) < threshold) {
+                    header = HTTP_HEADERS_BY_NORMALIZED_NAME.get(normalizedHeader);
+                    break;
+                }
+            }
+        }
+        return header;
+    }
+
+    private static int levenshteinDistance(String source, String target) {
+        int[] previous = new int[target.length() + 1];
+        int[] current = new int[target.length() + 1];
+        for (int j = 0; j <= target.length(); j++) {
+            previous[j] = j;
+        }
+        for (int i = 1; i <= source.length(); i++) {
+            current[0] = i;
+            for (int j = 1; j <= target.length(); j++) {
+                int substitution = source.charAt(i - 1) == target.charAt(j - 1) ? 0 : 1;
+                current[j] =
+                        Math.min(
+                                Math.min(current[j - 1] + 1, previous[j] + 1),
+                                previous[j - 1] + substitution);
+            }
+            int[] tmp = previous;
+            previous = current;
+            current = tmp;
+        }
+        return previous[target.length()];
+    }
+
+    private static String normalizeHttpHeaderName(String key) {
+        StringBuilder normalized = new StringBuilder(key.length());
+        for (int i = 0; i < key.length(); i++) {
+            char c = key.charAt(i);
+            if (Character.isLetter(c)) {
+                normalized.append(Character.toLowerCase(c));
+            }
+        }
+        return normalized.toString();
+    }
+
+    private static Map<String, String> httpHeadersByNormalizedName() {
+        Map<String, String> headers = new HashMap<>();
+        for (Field field : HttpHeaders.class.getFields()) {
+            int modifiers = field.getModifiers();
+            if (!Modifier.isFinal(modifiers)
+                    || !Modifier.isPublic(modifiers)
+                    || !Modifier.isStatic(modifiers)
+                    || !field.getType().equals(String.class)) {
+                continue;
+            }
+            try {
+                String value = (String) field.get(null);
+                headers.put(normalizeHttpHeaderName(value), value);
+            } catch (IllegalAccessException e) {
+                // Ignore inaccessible constants.
+            }
+        }
+        return Collections.unmodifiableMap(headers);
+    }
+
     /**
      * @since 1.16
      */

core/src/test/java/org/apache/stormcrawler/MetadataTest.java

@@ -52,6 +52,25 @@ void testAddValueIsCaseInsensitive() {
         Assertions.assertEquals(1, metadata.size());
     }
 
+    @Test
+    void testHttpHeadersAreSpellChecked() {
+        Metadata metadata = new Metadata();
+        metadata.addValue("ContentType", "text/html");
+        metadata.addValue("ConTnTtYpe", "application/xhtml+xml");
+
+        Assertions.assertEquals(2, metadata.getValues("Content-Type").length);
+        Assertions.assertTrue(metadata.containsKey("content-type"));
+    }
+
+    @Test
+    void testNonHttpHeadersAreNotSpellChecked() {
+        Metadata metadata = new Metadata();
+        metadata.addValue("customContentType", "value");
+
+        Assertions.assertNull(metadata.getFirstValue("Content-Type"));
+        Assertions.assertEquals("value", metadata.getFirstValue("customContentType"));
+    }
+
     @Test
     void testRemoveIsCaseInsensitive() {
         Metadata metadata = new Metadata();