Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 19 additions & 2 deletions src/parse/pdf_resources/page_font.h
Original file line number Diff line number Diff line change
Expand Up @@ -2218,18 +2218,35 @@ namespace pdflib
and cmap_numb_to_char.at(numb)!=replacement_char;
};

// Subset generators (FontForge, fontTools, mPDF, ...) name glyphs by
// bare index: /gid00043, /g43, /glyph43, /cid43, /index43. Such a name
// identifies the glyph inside the embedded font program but carries no
// reading text, so keeping it fabricates plausible-looking garbage
// ('gid00043gid00049...') that downstream quality gates cannot detect
// (docling-project/docling-parse#238).
std::regex re_gid(R"((gid|glyph|g|cid|index)\d+)", std::regex::icase);

// Last-resort for glyph-names that neither the /ToUnicode cmap nor
// any glyph-table could resolve (eg custom ligatures like /Th, /ft
// or /tt in a font without cmap): keep the glyph-name itself without
// the leading '/' and any '.suffix' as the reading text.
auto resolve_unknown_name = [](const std::string& name, const std::string& name_)
// the leading '/' and any '.suffix' as the reading text — unless the
// name is a pure glyph-index, which becomes a GLYPH marker so the
// unresolved glyph stays detectable.
auto resolve_unknown_name = [&re_gid](const std::string& name, const std::string& name_)
{
std::string result = name_;
if(result.empty())
{
result = (name.size()>0 and name[0]=='/')? name.substr(1) : name;
}

if(std::regex_match(result, re_gid))
{
LOG_S(WARNING) << "glyph-index name " << name
<< ": emitting marker 'GLYPH<name:" << result << ">'";
return "GLYPH<name:"+result+">";
}

LOG_S(WARNING) << "unknown glyph-name " << name
<< ": falling back to '" << result << "'";
return result;
Expand Down
120 changes: 120 additions & 0 deletions tests/test_gid_glyph_names.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
#!/usr/bin/env python
"""Glyph-index names in /Encoding/Differences must not leak as reading text.

Subset generators (FontForge, fontTools, mPDF, ...) name glyphs by bare
index: /gid00043, /g43, /glyph43. Such a name identifies the glyph inside
the embedded font program but carries no reading text. When neither a
/ToUnicode CMap nor the glyph-tables can resolve the code, keeping the name
fabricates plausible-looking garbage ('gid00043gid00049...') that downstream
quality gates cannot detect (docling-project/docling-parse#238). The parser
must emit a GLYPH marker instead.

Meaningful unknown names (custom ligatures like /Th) keep the existing
name-as-text fallback, and a valid /ToUnicode CMap stays authoritative.

These tests build the PDFs in memory: a Type1 font whose /Differences maps
codes 0x41-0x43 to gid-style names, drawn as the string (ABC).
"""

from io import BytesIO

from docling_parse.pdf_parser import DecodeConfig, DoclingPdfParser

_TOUNICODE = (
"/CIDInit /ProcSet findresource begin\n"
"12 dict begin\n"
"begincmap\n"
"/CIDSystemInfo << /Registry (Adobe) /Ordering (UCS) /Supplement 0 >> def\n"
"/CMapName /Adobe-Identity-UCS def\n"
"/CMapType 2 def\n"
"1 begincodespacerange\n"
"<00> <FF>\n"
"endcodespacerange\n"
"3 beginbfchar\n"
"<41> <0043>\n" # C
"<42> <0049>\n" # I
"<43> <0041>\n" # A
"endbfchar\n"
"endcmap\n"
"CMapName currentdict /CMap defineresource pop\n"
"end\n"
"end"
)


def _build_pdf(glyph_names: list, include_tounicode: bool) -> bytes:
differences = " ".join(f"/{name}" for name in glyph_names)
tounicode = " /ToUnicode 6 0 R" if include_tounicode else ""
content = "BT /F1 24 Tf 72 700 Td (ABC) Tj ET"
objects = [
"<< /Type /Catalog /Pages 2 0 R >>",
"<< /Type /Pages /Kids [3 0 R] /Count 1 >>",
"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] "
"/Resources << /Font << /F1 4 0 R >> >> /Contents 5 0 R >>",
"<< /Type /Font /Subtype /Type1 /BaseFont /XXXXXX+FakeSubset "
"/Encoding << /Type /Encoding "
f"/Differences [ 65 {differences} ] >>{tounicode} >>",
f"<< /Length {len(content)} >>\nstream\n{content}\nendstream",
]
if include_tounicode:
objects.append(f"<< /Length {len(_TOUNICODE)} >>\nstream\n{_TOUNICODE}\nendstream")

out = b"%PDF-1.4\n"
offsets = []
for index, obj in enumerate(objects, start=1):
offsets.append(len(out))
out += f"{index} 0 obj\n{obj}\nendobj\n".encode("latin-1")

startxref = len(out)
out += f"xref\n0 {len(objects) + 1}\n".encode("latin-1")
out += b"0000000000 65535 f \n"
for offset in offsets:
out += f"{offset:010d} 00000 n \n".encode("latin-1")
out += (
f"trailer\n<< /Size {len(objects) + 1} /Root 1 0 R >>\n"
f"startxref\n{startxref}\n%%EOF"
).encode("latin-1")
return out


def _extract_text(glyph_names: list, include_tounicode: bool) -> str:
parser = DoclingPdfParser(loglevel="fatal")
config = DecodeConfig(keep_glyphs=True)
doc = parser.load(
path_or_stream=BytesIO(_build_pdf(glyph_names, include_tounicode)),
decode_config=config,
)
_, page = next(doc.iterate_pages())
return "".join(cell.text for cell in page.textline_cells)


def test_gid_names_without_tounicode_yield_glyph_markers():
# No fabricated 'gid00043...' reading text: an unresolvable glyph-index
# name must surface as a GLYPH marker.
text = _extract_text(["gid00043", "gid00049", "gid00041"], include_tounicode=False)
assert "GLYPH<name:gid00043>" in text
assert "GLYPH<name:gid00049>" in text
assert "GLYPH<name:gid00041>" in text
assert "gid00043gid00049" not in text.replace("GLYPH<name:gid00043>", "")


def test_gid_name_variants_yield_glyph_markers():
text = _extract_text(["g43", "glyph49", "index41"], include_tounicode=False)
assert "GLYPH<name:g43>" in text
assert "GLYPH<name:glyph49>" in text
assert "GLYPH<name:index41>" in text


def test_tounicode_stays_authoritative_over_gid_names():
# PDF 32000-1 section 9.10.2: the /ToUnicode CMap is the first method;
# gid-style names never shadow it.
text = _extract_text(["gid00043", "gid00049", "gid00041"], include_tounicode=True)
assert text == "CIA"


def test_meaningful_unknown_names_keep_name_fallback():
# Custom ligature names like /Th carry reading text; the existing
# name-as-text fallback stays.
text = _extract_text(["Th", "ft", "tt"], include_tounicode=False)
assert text == "Thfttt"
assert "GLYPH" not in text
Loading