quote handling

CHANGELOG.md

@@ -130,4 +130,7 @@
 - oa-config update
 
 ## 0.5.1 
-- tighten version pinning for oa-config
+- tighten version pinning for oa-config
+
+## 0.5.2
+- changes to quote handling in tab delim files

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "orm-loader"
-version = "0.5.1"
+version = "0.5.2"
 description = "Generic base classes to handle ORM functionality for multiple downstream datamodels"
 readme = "README.md"
 authors = [
@@ -11,7 +11,7 @@ dependencies = [
     "chardet>=5.2.0",
     "oa-configurator==0.1.2",
     "pandas>=2.3.3",
-    "pyarrow>=23.0.0",
+    "pyarrow>=23.0.1",
     "sqlalchemy>=2.0.45",
 ]
 

src/orm_loader/loaders/loader_interface.py

@@ -1,13 +1,14 @@
 from __future__ import annotations
 from typing import Any
+import csv as _csv
 import pandas as pd
 import logging
 import pyarrow as pa
 import pyarrow.dataset as ds
 import pyarrow.compute as pc
 from functools import reduce
 from .data_classes import LoaderContext, TableCastingStats, LoaderInterface
-from .loading_helpers import infer_delim, infer_encoding, conservative_load_parquet, arrow_drop_duplicates
+from .loading_helpers import infer_delim, infer_encoding, conservative_load_parquet, arrow_drop_duplicates, resolve_quote_mode
 from .data import perform_cast, cast_arrow_column
 
 logger = logging.getLogger(__name__)
@@ -126,13 +127,22 @@ def orm_file_load(cls, ctx: LoaderContext) -> int:
         delimiter = infer_delim(ctx.path)
         encoding = infer_encoding(ctx.path)['encoding']
 
+        # Resolve to the same concrete mode the COPY fast-path would use, so a
+        # file that falls back here is parsed identically. Without this the
+        # fallback ignored quote_mode entirely and applied RFC-4180 quoting
+        # unconditionally — re-breaking on the stray quotes that pushed a
+        # literal-mode file off the fast-path.
+        quote_mode = resolve_quote_mode(ctx.quote_mode, ctx.path, delimiter, encoding or "utf-8")
+        quoting = _csv.QUOTE_NONE if quote_mode == "literal" else _csv.QUOTE_MINIMAL
+
         try:
             reader = pd.read_csv(
                 ctx.path,
                 delimiter=delimiter,
                 dtype=str,
                 chunksize=ctx.chunksize,
                 encoding=encoding,
+                quoting=quoting,
             )
         except pd.errors.EmptyDataError:
             logger.info(f"File {ctx.path.name} is empty — skipping load for {ctx.tableclass.__tablename__}")

src/orm_loader/loaders/loading_helpers.py

@@ -95,7 +95,7 @@ def infer_quote_mode(
     path: Path,
     delimiter: str,
     encoding: str = "utf-8",
-    sample_rows: int = 200,
+    sample_rows: int = 2000,
 ) -> str:
     """Return 'csv' or 'literal' by comparing column-count consistency under both
     quoting interpretations across a sample of rows.
@@ -150,6 +150,38 @@ def infer_quote_mode(
     # Prefer csv on a tie; only choose literal when it is strictly more consistent
     return "literal" if lit_ok > csv_ok else "csv"
 
+
+def resolve_quote_mode(
+    quote_mode: str,
+    path: Path,
+    delimiter: str,
+    encoding: str = "utf-8",
+) -> str:
+    """Resolve a requested quote_mode to a concrete ``"csv"`` or ``"literal"``.
+
+    Recognised inputs:
+
+    - ``"csv"`` / ``"literal"`` — returned unchanged (explicit override).
+    - ``"auto"`` — legacy content sniff via :func:`infer_quote_mode`. Compares
+      column-count consistency under both interpretations across a sample of
+      rows. Supports RFC-4180 quoting (embedded delimiters/newlines) but can
+      guess wrong when the discriminating rows fall outside the sample.
+    - ``"by_delimiter"`` — derive the mode from the delimiter alone, no content
+      scan: tab-delimited input never embeds the delimiter in a field so any
+      double-quote is literal data (``"literal"``); for non-tab delimiters this
+      resolves to ``"csv"`` where RFC-4180 quoting can be meaningful. Deterministic
+      and immune to the sampling gamble; use it when the source guarantees tab
+      fields never contain embedded tabs/newlines.
+    """
+    if quote_mode in ("csv", "literal"):
+        return quote_mode
+    if quote_mode == "by_delimiter":
+        return "literal" if delimiter == "\t" else "csv"
+    if quote_mode == "auto":
+        return infer_quote_mode(path, delimiter=delimiter, encoding=encoding)
+    raise ValueError(f"Unknown quote_mode: {quote_mode!r}")
+
+
 def arrow_drop_duplicates(
     table: pa.Table,
     pk_names: list[str],
@@ -244,9 +276,8 @@ def quick_load_pg(
     if not _SAFE_ENCODING.match(encoding):
         raise ValueError(f"Unsafe encoding value from chardet: {encoding!r}")
     delimiter = infer_delim(path)
-    if quote_mode == "auto":
-        quote_mode = infer_quote_mode(path, delimiter=delimiter, encoding=encoding)
-        logger.info(f"Auto-detected quote_mode={quote_mode!r} for {path.name}")
+    quote_mode = resolve_quote_mode(quote_mode, path, delimiter, encoding)
+    logger.info(f"Using quote_mode={quote_mode!r} for {path.name} (delimiter={delimiter!r})")
     if quote_mode == "csv":
         copy_options = f"""
             FORMAT csv,

tests/loaders/test_helpers.py

@@ -1,5 +1,14 @@
+from pathlib import Path
+
+import pytest
+
 from orm_loader.loaders.data_classes import ColumnCastingStats, TableCastingStats
-from orm_loader.loaders.loading_helpers import infer_delim, infer_encoding, infer_quote_mode
+from orm_loader.loaders.loading_helpers import (
+    infer_delim,
+    infer_encoding,
+    infer_quote_mode,
+    resolve_quote_mode,
+)
 
 def test_column_casting_stats_records_examples():
     stats = ColumnCastingStats()
@@ -67,3 +76,48 @@ def test_infer_quote_mode_unbalanced_quote_returns_literal(tmp_path):
     p = tmp_path / "x.csv"
     p.write_text('id\tname\n1\t"open\n2\t"open\n3\t"open\n')
     assert infer_quote_mode(p, delimiter="\t") == "literal"
+
+
+# ---------------------------------------------------------------------------
+# resolve_quote_mode
+# ---------------------------------------------------------------------------
+
+def test_resolve_quote_mode_explicit_csv_passthrough():
+    # Explicit modes are returned unchanged and never read the file
+    assert resolve_quote_mode("csv", Path("does-not-exist.csv"), ",") == "csv"
+
+
+def test_resolve_quote_mode_explicit_literal_passthrough():
+    assert resolve_quote_mode("literal", Path("does-not-exist.csv"), "\t") == "literal"
+
+
+def test_resolve_quote_mode_by_delimiter_tab_is_literal():
+    # Tab-delimited input never embeds the delimiter in a field, so any
+    # double-quote is literal data — derived from the delimiter, no file scan
+    assert resolve_quote_mode("by_delimiter", Path("does-not-exist.tsv"), "\t") == "literal"
+
+
+def test_resolve_quote_mode_by_delimiter_comma_is_csv():
+    # Comma-delimited input must wrap fields containing the delimiter in quotes,
+    # so quoting is meaningful
+    assert resolve_quote_mode("by_delimiter", Path("does-not-exist.csv"), ",") == "csv"
+
+
+def test_resolve_quote_mode_auto_delegates_to_infer(tmp_path):
+    # "auto" defers to the content sniff: an unbalanced leading quote is only
+    # consistent under literal parsing
+    p = tmp_path / "x.csv"
+    p.write_text('id\tname\n1\t"open\n2\t"open\n3\t"open\n')
+    assert resolve_quote_mode("auto", p, delimiter="\t") == "literal"
+
+
+def test_resolve_quote_mode_auto_clean_file_returns_csv(tmp_path):
+    # No quotes anywhere: the sniff falls back to the csv default
+    p = tmp_path / "x.csv"
+    p.write_text("id\tname\n1\tAlice\n2\tBob\n")
+    assert resolve_quote_mode("auto", p, delimiter="\t") == "csv"
+
+
+def test_resolve_quote_mode_unknown_raises(tmp_path):
+    with pytest.raises(ValueError, match="Unknown quote_mode"):
+        resolve_quote_mode("bogus", tmp_path / "x.csv", ",")

tests/loaders/test_pandas_loader.py

@@ -54,6 +54,110 @@ def test_pandas_loader_case_insensitive_headers(tmp_path, session):
     assert rows == [(1, "alpha"), (2, "beta")]
 
 
+def test_pandas_loader_literal_mode_preserves_quotes(tmp_path, session):
+    """
+    The pandas fallback must honour quote_mode. Under 'literal' it uses
+    QUOTE_NONE, so double-quotes are literal data and are preserved verbatim
+    rather than being stripped as RFC-4180 field wrappers.
+    """
+
+    csv = tmp_path / "quoted.csv"
+    csv.write_text(
+        'id,value\n'
+        '1,"hello"\n'
+        '2,"world"\n'
+    )
+
+    ctx = LoaderContext(
+        tableclass=PandasLoaderTable,
+        session=session,
+        path=csv,
+        staging_table=PandasLoaderTable.__table__,
+        chunksize=100,
+        normalise=True,
+        dedupe=True,
+        quote_mode="literal",
+    )
+
+    n = PandasLoader.orm_file_load(ctx)
+
+    rows = session.execute(
+        sa.text('SELECT id, value FROM test_pandas_loader ORDER BY id')
+    ).all()
+
+    assert n == 2
+    assert rows == [(1, '"hello"'), (2, '"world"')]
+
+
+def test_pandas_loader_csv_mode_strips_quotes(tmp_path, session):
+    """
+    Contrast with the literal case above: the same file under 'csv' mode uses
+    QUOTE_MINIMAL, so the wrapping quotes are consumed. Proves quote_mode
+    actually flows through the fallback rather than being ignored.
+    """
+
+    csv = tmp_path / "quoted.csv"
+    csv.write_text(
+        'id,value\n'
+        '1,"hello"\n'
+        '2,"world"\n'
+    )
+
+    ctx = LoaderContext(
+        tableclass=PandasLoaderTable,
+        session=session,
+        path=csv,
+        staging_table=PandasLoaderTable.__table__,
+        chunksize=100,
+        normalise=True,
+        dedupe=True,
+        quote_mode="csv",
+    )
+
+    n = PandasLoader.orm_file_load(ctx)
+
+    rows = session.execute(
+        sa.text('SELECT id, value FROM test_pandas_loader ORDER BY id')
+    ).all()
+
+    assert n == 2
+    assert rows == [(1, "hello"), (2, "world")]
+
+
+def test_pandas_loader_by_delimiter_tab_preserves_quotes(tmp_path, session):
+    """
+    'by_delimiter' derives the mode from the delimiter alone: a tab-delimited
+    file resolves to 'literal', so embedded double-quotes are kept as data.
+    """
+
+    tsv = tmp_path / "tab.tsv"
+    tsv.write_text(
+        'id\tvalue\n'
+        '1\t"hi"\n'
+        '2\tbye\n'
+    )
+
+    ctx = LoaderContext(
+        tableclass=PandasLoaderTable,
+        session=session,
+        path=tsv,
+        staging_table=PandasLoaderTable.__table__,
+        chunksize=100,
+        normalise=True,
+        dedupe=True,
+        quote_mode="by_delimiter",
+    )
+
+    n = PandasLoader.orm_file_load(ctx)
+
+    rows = session.execute(
+        sa.text('SELECT id, value FROM test_pandas_loader ORDER BY id')
+    ).all()
+
+    assert n == 2
+    assert rows == [(1, '"hi"'), (2, "bye")]
+
+
 def test_pandas_dedupe_internal(session, tmp_path):
     df = pd.DataFrame({
         "id": [1, 1, 2],