From a9391255100d9cf3efcd63c1d15c5bd0835c02e5 Mon Sep 17 00:00:00 2001 From: gkennos Date: Sun, 5 Jul 2026 23:02:00 +1000 Subject: [PATCH 1/4] quote handling --- src/orm_loader/loaders/loader_interface.py | 12 ++- src/orm_loader/loaders/loading_helpers.py | 39 +++++++- tests/loaders/test_helpers.py | 56 ++++++++++- tests/loaders/test_pandas_loader.py | 104 +++++++++++++++++++++ 4 files changed, 205 insertions(+), 6 deletions(-) diff --git a/src/orm_loader/loaders/loader_interface.py b/src/orm_loader/loaders/loader_interface.py index 8b500f1..606e8d6 100644 --- a/src/orm_loader/loaders/loader_interface.py +++ b/src/orm_loader/loaders/loader_interface.py @@ -1,5 +1,6 @@ from __future__ import annotations from typing import Any +import csv as _csv import pandas as pd import logging import pyarrow as pa @@ -7,7 +8,7 @@ import pyarrow.compute as pc from functools import reduce from .data_classes import LoaderContext, TableCastingStats, LoaderInterface -from .loading_helpers import infer_delim, infer_encoding, conservative_load_parquet, arrow_drop_duplicates +from .loading_helpers import infer_delim, infer_encoding, conservative_load_parquet, arrow_drop_duplicates, resolve_quote_mode from .data import perform_cast, cast_arrow_column logger = logging.getLogger(__name__) @@ -126,6 +127,14 @@ def orm_file_load(cls, ctx: LoaderContext) -> int: delimiter = infer_delim(ctx.path) encoding = infer_encoding(ctx.path)['encoding'] + # Resolve to the same concrete mode the COPY fast-path would use, so a + # file that falls back here is parsed identically. Without this the + # fallback ignored quote_mode entirely and applied RFC-4180 quoting + # unconditionally — re-breaking on the stray quotes that pushed a + # literal-mode file off the fast-path. + quote_mode = resolve_quote_mode(ctx.quote_mode, ctx.path, delimiter, encoding or "utf-8") + quoting = _csv.QUOTE_NONE if quote_mode == "literal" else _csv.QUOTE_MINIMAL + try: reader = pd.read_csv( ctx.path, @@ -133,6 +142,7 @@ def orm_file_load(cls, ctx: LoaderContext) -> int: dtype=str, chunksize=ctx.chunksize, encoding=encoding, + quoting=quoting, ) except pd.errors.EmptyDataError: logger.info(f"File {ctx.path.name} is empty — skipping load for {ctx.tableclass.__tablename__}") diff --git a/src/orm_loader/loaders/loading_helpers.py b/src/orm_loader/loaders/loading_helpers.py index 25fc949..6574aac 100644 --- a/src/orm_loader/loaders/loading_helpers.py +++ b/src/orm_loader/loaders/loading_helpers.py @@ -95,7 +95,7 @@ def infer_quote_mode( path: Path, delimiter: str, encoding: str = "utf-8", - sample_rows: int = 200, + sample_rows: int = 2000, ) -> str: """Return 'csv' or 'literal' by comparing column-count consistency under both quoting interpretations across a sample of rows. @@ -150,6 +150,38 @@ def infer_quote_mode( # Prefer csv on a tie; only choose literal when it is strictly more consistent return "literal" if lit_ok > csv_ok else "csv" + +def resolve_quote_mode( + quote_mode: str, + path: Path, + delimiter: str, + encoding: str = "utf-8", +) -> str: + """Resolve a requested quote_mode to a concrete ``"csv"`` or ``"literal"``. + + Recognised inputs: + + - ``"csv"`` / ``"literal"`` — returned unchanged (explicit override). + - ``"auto"`` — legacy content sniff via :func:`infer_quote_mode`. Compares + column-count consistency under both interpretations across a sample of + rows. Supports RFC-4180 quoting (embedded delimiters/newlines) but can + guess wrong when the discriminating rows fall outside the sample. + - ``"by_delimiter"`` — derive the mode from the delimiter alone, no content + scan: tab-delimited input never embeds the delimiter in a field so any + double-quote is literal data (``"literal"``); comma-delimited input must + wrap fields containing the delimiter in quotes so quoting is meaningful + (``"csv"``). Deterministic and immune to the sampling gamble; use it when + the source guarantees tab fields never contain embedded tabs/newlines. + """ + if quote_mode in ("csv", "literal"): + return quote_mode + if quote_mode == "by_delimiter": + return "literal" if delimiter == "\t" else "csv" + if quote_mode == "auto": + return infer_quote_mode(path, delimiter=delimiter, encoding=encoding) + raise ValueError(f"Unknown quote_mode: {quote_mode!r}") + + def arrow_drop_duplicates( table: pa.Table, pk_names: list[str], @@ -244,9 +276,8 @@ def quick_load_pg( if not _SAFE_ENCODING.match(encoding): raise ValueError(f"Unsafe encoding value from chardet: {encoding!r}") delimiter = infer_delim(path) - if quote_mode == "auto": - quote_mode = infer_quote_mode(path, delimiter=delimiter, encoding=encoding) - logger.info(f"Auto-detected quote_mode={quote_mode!r} for {path.name}") + quote_mode = resolve_quote_mode(quote_mode, path, delimiter, encoding) + logger.info(f"Using quote_mode={quote_mode!r} for {path.name} (delimiter={delimiter!r})") if quote_mode == "csv": copy_options = f""" FORMAT csv, diff --git a/tests/loaders/test_helpers.py b/tests/loaders/test_helpers.py index da907d1..faf70b1 100644 --- a/tests/loaders/test_helpers.py +++ b/tests/loaders/test_helpers.py @@ -1,5 +1,14 @@ +from pathlib import Path + +import pytest + from orm_loader.loaders.data_classes import ColumnCastingStats, TableCastingStats -from orm_loader.loaders.loading_helpers import infer_delim, infer_encoding, infer_quote_mode +from orm_loader.loaders.loading_helpers import ( + infer_delim, + infer_encoding, + infer_quote_mode, + resolve_quote_mode, +) def test_column_casting_stats_records_examples(): stats = ColumnCastingStats() @@ -67,3 +76,48 @@ def test_infer_quote_mode_unbalanced_quote_returns_literal(tmp_path): p = tmp_path / "x.csv" p.write_text('id\tname\n1\t"open\n2\t"open\n3\t"open\n') assert infer_quote_mode(p, delimiter="\t") == "literal" + + +# --------------------------------------------------------------------------- +# resolve_quote_mode +# --------------------------------------------------------------------------- + +def test_resolve_quote_mode_explicit_csv_passthrough(): + # Explicit modes are returned unchanged and never read the file + assert resolve_quote_mode("csv", Path("does-not-exist.csv"), ",") == "csv" + + +def test_resolve_quote_mode_explicit_literal_passthrough(): + assert resolve_quote_mode("literal", Path("does-not-exist.csv"), "\t") == "literal" + + +def test_resolve_quote_mode_by_delimiter_tab_is_literal(): + # Tab-delimited input never embeds the delimiter in a field, so any + # double-quote is literal data — derived from the delimiter, no file scan + assert resolve_quote_mode("by_delimiter", Path("does-not-exist.tsv"), "\t") == "literal" + + +def test_resolve_quote_mode_by_delimiter_comma_is_csv(): + # Comma-delimited input must wrap fields containing the delimiter in quotes, + # so quoting is meaningful + assert resolve_quote_mode("by_delimiter", Path("does-not-exist.csv"), ",") == "csv" + + +def test_resolve_quote_mode_auto_delegates_to_infer(tmp_path): + # "auto" defers to the content sniff: an unbalanced leading quote is only + # consistent under literal parsing + p = tmp_path / "x.csv" + p.write_text('id\tname\n1\t"open\n2\t"open\n3\t"open\n') + assert resolve_quote_mode("auto", p, delimiter="\t") == "literal" + + +def test_resolve_quote_mode_auto_clean_file_returns_csv(tmp_path): + # No quotes anywhere: the sniff falls back to the csv default + p = tmp_path / "x.csv" + p.write_text("id\tname\n1\tAlice\n2\tBob\n") + assert resolve_quote_mode("auto", p, delimiter="\t") == "csv" + + +def test_resolve_quote_mode_unknown_raises(tmp_path): + with pytest.raises(ValueError, match="Unknown quote_mode"): + resolve_quote_mode("bogus", tmp_path / "x.csv", ",") diff --git a/tests/loaders/test_pandas_loader.py b/tests/loaders/test_pandas_loader.py index aee46d0..3cc0e0a 100644 --- a/tests/loaders/test_pandas_loader.py +++ b/tests/loaders/test_pandas_loader.py @@ -54,6 +54,110 @@ def test_pandas_loader_case_insensitive_headers(tmp_path, session): assert rows == [(1, "alpha"), (2, "beta")] +def test_pandas_loader_literal_mode_preserves_quotes(tmp_path, session): + """ + The pandas fallback must honour quote_mode. Under 'literal' it uses + QUOTE_NONE, so double-quotes are literal data and are preserved verbatim + rather than being stripped as RFC-4180 field wrappers. + """ + + csv = tmp_path / "quoted.csv" + csv.write_text( + 'id,value\n' + '1,"hello"\n' + '2,"world"\n' + ) + + ctx = LoaderContext( + tableclass=PandasLoaderTable, + session=session, + path=csv, + staging_table=PandasLoaderTable.__table__, + chunksize=100, + normalise=True, + dedupe=True, + quote_mode="literal", + ) + + n = PandasLoader.orm_file_load(ctx) + + rows = session.execute( + sa.text('SELECT id, value FROM test_pandas_loader ORDER BY id') + ).all() + + assert n == 2 + assert rows == [(1, '"hello"'), (2, '"world"')] + + +def test_pandas_loader_csv_mode_strips_quotes(tmp_path, session): + """ + Contrast with the literal case above: the same file under 'csv' mode uses + QUOTE_MINIMAL, so the wrapping quotes are consumed. Proves quote_mode + actually flows through the fallback rather than being ignored. + """ + + csv = tmp_path / "quoted.csv" + csv.write_text( + 'id,value\n' + '1,"hello"\n' + '2,"world"\n' + ) + + ctx = LoaderContext( + tableclass=PandasLoaderTable, + session=session, + path=csv, + staging_table=PandasLoaderTable.__table__, + chunksize=100, + normalise=True, + dedupe=True, + quote_mode="csv", + ) + + n = PandasLoader.orm_file_load(ctx) + + rows = session.execute( + sa.text('SELECT id, value FROM test_pandas_loader ORDER BY id') + ).all() + + assert n == 2 + assert rows == [(1, "hello"), (2, "world")] + + +def test_pandas_loader_by_delimiter_tab_preserves_quotes(tmp_path, session): + """ + 'by_delimiter' derives the mode from the delimiter alone: a tab-delimited + file resolves to 'literal', so embedded double-quotes are kept as data. + """ + + tsv = tmp_path / "tab.tsv" + tsv.write_text( + 'id\tvalue\n' + '1\t"hi"\n' + '2\tbye\n' + ) + + ctx = LoaderContext( + tableclass=PandasLoaderTable, + session=session, + path=tsv, + staging_table=PandasLoaderTable.__table__, + chunksize=100, + normalise=True, + dedupe=True, + quote_mode="by_delimiter", + ) + + n = PandasLoader.orm_file_load(ctx) + + rows = session.execute( + sa.text('SELECT id, value FROM test_pandas_loader ORDER BY id') + ).all() + + assert n == 2 + assert rows == [(1, '"hi"'), (2, "bye")] + + def test_pandas_dedupe_internal(session, tmp_path): df = pd.DataFrame({ "id": [1, 1, 2], From dc153f197bfbfc49a488a6913c82cea040a77c19 Mon Sep 17 00:00:00 2001 From: gkennos Date: Sun, 5 Jul 2026 23:13:59 +1000 Subject: [PATCH 2/4] bump --- CHANGELOG.md | 5 ++++- pyproject.toml | 2 +- uv.lock | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cb79264..382ca4a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -130,4 +130,7 @@ - oa-config update ## 0.5.1 -- tighten version pinning for oa-config \ No newline at end of file +- tighten version pinning for oa-config + +## 0.5.2 +- changes to quote handling in tab delim files \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 058d8d2..0eeca50 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "orm-loader" -version = "0.5.1" +version = "0.5.2" description = "Generic base classes to handle ORM functionality for multiple downstream datamodels" readme = "README.md" authors = [ diff --git a/uv.lock b/uv.lock index da65207..7c8a3eb 100644 --- a/uv.lock +++ b/uv.lock @@ -679,7 +679,7 @@ postgres = [ [[package]] name = "orm-loader" -version = "0.5.1" +version = "0.5.2" source = { editable = "." } dependencies = [ { name = "chardet" }, From 52a9a406134beafd7fc525f4e4338091a35f636b Mon Sep 17 00:00:00 2001 From: gkennos Date: Sun, 5 Jul 2026 23:14:54 +1000 Subject: [PATCH 3/4] dependabot --- pyproject.toml | 2 +- uv.lock | 80 +++++++++++++++++++++++++------------------------- 2 files changed, 41 insertions(+), 41 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 0eeca50..a7209ce 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,7 @@ dependencies = [ "chardet>=5.2.0", "oa-configurator==0.1.2", "pandas>=2.3.3", - "pyarrow>=23.0.0", + "pyarrow>=23.0.1", "sqlalchemy>=2.0.45", ] diff --git a/uv.lock b/uv.lock index 7c8a3eb..ecad039 100644 --- a/uv.lock +++ b/uv.lock @@ -719,7 +719,7 @@ requires-dist = [ { name = "oa-configurator", extras = ["postgres"], marker = "extra == 'dev'", specifier = "==0.1.2" }, { name = "pandas", specifier = ">=2.3.3" }, { name = "psycopg", extras = ["binary"], marker = "extra == 'postgres'", specifier = ">=3.2" }, - { name = "pyarrow", specifier = ">=23.0.0" }, + { name = "pyarrow", specifier = ">=23.0.1" }, { name = "pygments", marker = "extra == 'dev'", specifier = ">=2.20.0" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=9.0.3" }, { name = "python-dotenv", marker = "extra == 'dev'" }, @@ -881,45 +881,45 @@ wheels = [ [[package]] name = "pyarrow" -version = "23.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/01/33/ffd9c3eb087fa41dd79c3cf20c4c0ae3cdb877c4f8e1107a446006344924/pyarrow-23.0.0.tar.gz", hash = "sha256:180e3150e7edfcd182d3d9afba72f7cf19839a497cc76555a8dce998a8f67615", size = 1167185, upload-time = "2026-01-18T16:19:42.218Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3d/bd/c861d020831ee57609b73ea721a617985ece817684dc82415b0bc3e03ac3/pyarrow-23.0.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:5961a9f646c232697c24f54d3419e69b4261ba8a8b66b0ac54a1851faffcbab8", size = 34189116, upload-time = "2026-01-18T16:15:28.054Z" }, - { url = "https://files.pythonhosted.org/packages/8c/23/7725ad6cdcbaf6346221391e7b3eecd113684c805b0a95f32014e6fa0736/pyarrow-23.0.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:632b3e7c3d232f41d64e1a4a043fb82d44f8a349f339a1188c6a0dd9d2d47d8a", size = 35803831, upload-time = "2026-01-18T16:15:33.798Z" }, - { url = "https://files.pythonhosted.org/packages/57/06/684a421543455cdc2944d6a0c2cc3425b028a4c6b90e34b35580c4899743/pyarrow-23.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:76242c846db1411f1d6c2cc3823be6b86b40567ee24493344f8226ba34a81333", size = 44436452, upload-time = "2026-01-18T16:15:41.598Z" }, - { url = "https://files.pythonhosted.org/packages/c6/6f/8f9eb40c2328d66e8b097777ddcf38494115ff9f1b5bc9754ba46991191e/pyarrow-23.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:b73519f8b52ae28127000986bf228fda781e81d3095cd2d3ece76eb5cf760e1b", size = 47557396, upload-time = "2026-01-18T16:15:51.252Z" }, - { url = "https://files.pythonhosted.org/packages/10/6e/f08075f1472e5159553501fde2cc7bc6700944bdabe49a03f8a035ee6ccd/pyarrow-23.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:068701f6823449b1b6469120f399a1239766b117d211c5d2519d4ed5861f75de", size = 48147129, upload-time = "2026-01-18T16:16:00.299Z" }, - { url = "https://files.pythonhosted.org/packages/7d/82/d5a680cd507deed62d141cc7f07f7944a6766fc51019f7f118e4d8ad0fb8/pyarrow-23.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:1801ba947015d10e23bca9dd6ef5d0e9064a81569a89b6e9a63b59224fd060df", size = 50596642, upload-time = "2026-01-18T16:16:08.502Z" }, - { url = "https://files.pythonhosted.org/packages/a9/26/4f29c61b3dce9fa7780303b86895ec6a0917c9af927101daaaf118fbe462/pyarrow-23.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:52265266201ec25b6839bf6bd4ea918ca6d50f31d13e1cf200b4261cd11dc25c", size = 27660628, upload-time = "2026-01-18T16:16:15.28Z" }, - { url = "https://files.pythonhosted.org/packages/66/34/564db447d083ec7ff93e0a883a597d2f214e552823bfc178a2d0b1f2c257/pyarrow-23.0.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:ad96a597547af7827342ffb3c503c8316e5043bb09b47a84885ce39394c96e00", size = 34184630, upload-time = "2026-01-18T16:16:22.141Z" }, - { url = "https://files.pythonhosted.org/packages/aa/3a/3999daebcb5e6119690c92a621c4d78eef2ffba7a0a1b56386d2875fcd77/pyarrow-23.0.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:b9edf990df77c2901e79608f08c13fbde60202334a4fcadb15c1f57bf7afee43", size = 35796820, upload-time = "2026-01-18T16:16:29.441Z" }, - { url = "https://files.pythonhosted.org/packages/ec/ee/39195233056c6a8d0976d7d1ac1cd4fe21fb0ec534eca76bc23ef3f60e11/pyarrow-23.0.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:36d1b5bc6ddcaff0083ceec7e2561ed61a51f49cce8be079ee8ed406acb6fdef", size = 44438735, upload-time = "2026-01-18T16:16:38.79Z" }, - { url = "https://files.pythonhosted.org/packages/2c/41/6a7328ee493527e7afc0c88d105ecca69a3580e29f2faaeac29308369fd7/pyarrow-23.0.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:4292b889cd224f403304ddda8b63a36e60f92911f89927ec8d98021845ea21be", size = 47557263, upload-time = "2026-01-18T16:16:46.248Z" }, - { url = "https://files.pythonhosted.org/packages/c6/ee/34e95b21ee84db494eae60083ddb4383477b31fb1fd19fd866d794881696/pyarrow-23.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:dfd9e133e60eaa847fd80530a1b89a052f09f695d0b9c34c235ea6b2e0924cf7", size = 48153529, upload-time = "2026-01-18T16:16:53.412Z" }, - { url = "https://files.pythonhosted.org/packages/52/88/8a8d83cea30f4563efa1b7bf51d241331ee5cd1b185a7e063f5634eca415/pyarrow-23.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:832141cc09fac6aab1cd3719951d23301396968de87080c57c9a7634e0ecd068", size = 50598851, upload-time = "2026-01-18T16:17:01.133Z" }, - { url = "https://files.pythonhosted.org/packages/c6/4c/2929c4be88723ba025e7b3453047dc67e491c9422965c141d24bab6b5962/pyarrow-23.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:7a7d067c9a88faca655c71bcc30ee2782038d59c802d57950826a07f60d83c4c", size = 27577747, upload-time = "2026-01-18T16:18:02.413Z" }, - { url = "https://files.pythonhosted.org/packages/64/52/564a61b0b82d72bd68ec3aef1adda1e3eba776f89134b9ebcb5af4b13cb6/pyarrow-23.0.0-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:ce9486e0535a843cf85d990e2ec5820a47918235183a5c7b8b97ed7e92c2d47d", size = 34446038, upload-time = "2026-01-18T16:17:07.861Z" }, - { url = "https://files.pythonhosted.org/packages/cc/c9/232d4f9855fd1de0067c8a7808a363230d223c83aeee75e0fe6eab851ba9/pyarrow-23.0.0-cp313-cp313t-macosx_12_0_x86_64.whl", hash = "sha256:075c29aeaa685fd1182992a9ed2499c66f084ee54eea47da3eb76e125e06064c", size = 35921142, upload-time = "2026-01-18T16:17:15.401Z" }, - { url = "https://files.pythonhosted.org/packages/96/f2/60af606a3748367b906bb82d41f0032e059f075444445d47e32a7ff1df62/pyarrow-23.0.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:799965a5379589510d888be3094c2296efd186a17ca1cef5b77703d4d5121f53", size = 44490374, upload-time = "2026-01-18T16:17:23.93Z" }, - { url = "https://files.pythonhosted.org/packages/ff/2d/7731543050a678ea3a413955a2d5d80d2a642f270aa57a3cb7d5a86e3f46/pyarrow-23.0.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:ef7cac8fe6fccd8b9e7617bfac785b0371a7fe26af59463074e4882747145d40", size = 47527896, upload-time = "2026-01-18T16:17:33.393Z" }, - { url = "https://files.pythonhosted.org/packages/5a/90/f3342553b7ac9879413aed46500f1637296f3c8222107523a43a1c08b42a/pyarrow-23.0.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:15a414f710dc927132dd67c361f78c194447479555af57317066ee5116b90e9e", size = 48210401, upload-time = "2026-01-18T16:17:42.012Z" }, - { url = "https://files.pythonhosted.org/packages/f3/da/9862ade205ecc46c172b6ce5038a74b5151c7401e36255f15975a45878b2/pyarrow-23.0.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:3e0d2e6915eca7d786be6a77bf227fbc06d825a75b5b5fe9bcbef121dec32685", size = 50579677, upload-time = "2026-01-18T16:17:50.241Z" }, - { url = "https://files.pythonhosted.org/packages/c2/4c/f11f371f5d4740a5dafc2e11c76bcf42d03dfdb2d68696da97de420b6963/pyarrow-23.0.0-cp313-cp313t-win_amd64.whl", hash = "sha256:4b317ea6e800b5704e5e5929acb6e2dc13e9276b708ea97a39eb8b345aa2658b", size = 27631889, upload-time = "2026-01-18T16:17:56.55Z" }, - { url = "https://files.pythonhosted.org/packages/97/bb/15aec78bcf43a0c004067bd33eb5352836a29a49db8581fc56f2b6ca88b7/pyarrow-23.0.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:20b187ed9550d233a872074159f765f52f9d92973191cd4b93f293a19efbe377", size = 34213265, upload-time = "2026-01-18T16:18:07.904Z" }, - { url = "https://files.pythonhosted.org/packages/f6/6c/deb2c594bbba41c37c5d9aa82f510376998352aa69dfcb886cb4b18ad80f/pyarrow-23.0.0-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:18ec84e839b493c3886b9b5e06861962ab4adfaeb79b81c76afbd8d84c7d5fda", size = 35819211, upload-time = "2026-01-18T16:18:13.94Z" }, - { url = "https://files.pythonhosted.org/packages/e0/e5/ee82af693cb7b5b2b74f6524cdfede0e6ace779d7720ebca24d68b57c36b/pyarrow-23.0.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:e438dd3f33894e34fd02b26bd12a32d30d006f5852315f611aa4add6c7fab4bc", size = 44502313, upload-time = "2026-01-18T16:18:20.367Z" }, - { url = "https://files.pythonhosted.org/packages/9c/86/95c61ad82236495f3c31987e85135926ba3ec7f3819296b70a68d8066b49/pyarrow-23.0.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:a244279f240c81f135631be91146d7fa0e9e840e1dfed2aba8483eba25cd98e6", size = 47585886, upload-time = "2026-01-18T16:18:27.544Z" }, - { url = "https://files.pythonhosted.org/packages/bb/6e/a72d901f305201802f016d015de1e05def7706fff68a1dedefef5dc7eff7/pyarrow-23.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c4692e83e42438dba512a570c6eaa42be2f8b6c0f492aea27dec54bdc495103a", size = 48207055, upload-time = "2026-01-18T16:18:35.425Z" }, - { url = "https://files.pythonhosted.org/packages/f9/e5/5de029c537630ca18828db45c30e2a78da03675a70ac6c3528203c416fe3/pyarrow-23.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ae7f30f898dfe44ea69654a35c93e8da4cef6606dc4c72394068fd95f8e9f54a", size = 50619812, upload-time = "2026-01-18T16:18:43.553Z" }, - { url = "https://files.pythonhosted.org/packages/59/8d/2af846cd2412e67a087f5bda4a8e23dfd4ebd570f777db2e8686615dafc1/pyarrow-23.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:5b86bb649e4112fb0614294b7d0a175c7513738876b89655605ebb87c804f861", size = 28263851, upload-time = "2026-01-18T16:19:38.567Z" }, - { url = "https://files.pythonhosted.org/packages/7b/7f/caab863e587041156f6786c52e64151b7386742c8c27140f637176e9230e/pyarrow-23.0.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:ebc017d765d71d80a3f8584ca0566b53e40464586585ac64176115baa0ada7d3", size = 34463240, upload-time = "2026-01-18T16:18:49.755Z" }, - { url = "https://files.pythonhosted.org/packages/c9/fa/3a5b8c86c958e83622b40865e11af0857c48ec763c11d472c87cd518283d/pyarrow-23.0.0-cp314-cp314t-macosx_12_0_x86_64.whl", hash = "sha256:0800cc58a6d17d159df823f87ad66cefebf105b982493d4bad03ee7fab84b993", size = 35935712, upload-time = "2026-01-18T16:18:55.626Z" }, - { url = "https://files.pythonhosted.org/packages/c5/08/17a62078fc1a53decb34a9aa79cf9009efc74d63d2422e5ade9fed2f99e3/pyarrow-23.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:3a7c68c722da9bb5b0f8c10e3eae71d9825a4b429b40b32709df5d1fa55beb3d", size = 44503523, upload-time = "2026-01-18T16:19:03.958Z" }, - { url = "https://files.pythonhosted.org/packages/cc/70/84d45c74341e798aae0323d33b7c39194e23b1abc439ceaf60a68a7a969a/pyarrow-23.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:bd5556c24622df90551063ea41f559b714aa63ca953db884cfb958559087a14e", size = 47542490, upload-time = "2026-01-18T16:19:11.208Z" }, - { url = "https://files.pythonhosted.org/packages/61/d9/d1274b0e6f19e235de17441e53224f4716574b2ca837022d55702f24d71d/pyarrow-23.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:54810f6e6afc4ffee7c2e0051b61722fbea9a4961b46192dcfae8ea12fa09059", size = 48233605, upload-time = "2026-01-18T16:19:19.544Z" }, - { url = "https://files.pythonhosted.org/packages/39/07/e4e2d568cb57543d84482f61e510732820cddb0f47c4bb7df629abfed852/pyarrow-23.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:14de7d48052cf4b0ed174533eafa3cfe0711b8076ad70bede32cf59f744f0d7c", size = 50603979, upload-time = "2026-01-18T16:19:26.717Z" }, - { url = "https://files.pythonhosted.org/packages/72/9c/47693463894b610f8439b2e970b82ef81e9599c757bf2049365e40ff963c/pyarrow-23.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:427deac1f535830a744a4f04a6ac183a64fcac4341b3f618e693c41b7b98d2b0", size = 28338905, upload-time = "2026-01-18T16:19:32.93Z" }, +version = "24.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/91/13/13e1069b351bdc3881266e11147ffccf687505dbb0ea74036237f5d454a5/pyarrow-24.0.0.tar.gz", hash = "sha256:85fe721a14dd823aca09127acbb06c3ca723efbd436c004f16bca601b04dcc83", size = 1180261, upload-time = "2026-04-21T10:51:25.837Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b4/a9/9686d9f07837f91f775e8932659192e02c74f9d8920524b480b85212cc68/pyarrow-24.0.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:6233c9ed9ab9d1db47de57d9753256d9dcffbf42db341576099f0fd9f6bf4810", size = 34981559, upload-time = "2026-04-21T10:47:22.17Z" }, + { url = "https://files.pythonhosted.org/packages/80/b6/0ddf0e9b6ead3474ab087ae598c76b031fc45532bf6a63f3a553440fb258/pyarrow-24.0.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:f7616236ec1bc2b15bfdec22a71ab38851c86f8f05ff64f379e1278cf20c634a", size = 36663654, upload-time = "2026-04-21T10:47:28.315Z" }, + { url = "https://files.pythonhosted.org/packages/7c/3b/926382efe8ce27ba729071d3566ade6dfb86bdf112f366000196b2f5780a/pyarrow-24.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:1617043b99bd33e5318ae18eb2919af09c71322ef1ca46566cdafc6e6712fb66", size = 45679394, upload-time = "2026-04-21T10:47:34.821Z" }, + { url = "https://files.pythonhosted.org/packages/b3/7a/829f7d9dfd37c207206081d6dad474d81dde29952401f07f2ba507814818/pyarrow-24.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:6165461f55ef6314f026de6638d661188e3455d3ec49834556a0ebbdbace18bb", size = 48863122, upload-time = "2026-04-21T10:47:42.056Z" }, + { url = "https://files.pythonhosted.org/packages/5f/e8/f88ce625fe8babaae64e8db2d417c7653adb3019b08aae85c5ed787dc816/pyarrow-24.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3b13dedfe76a0ad2d1d859b0811b53827a4e9d93a0bcb05cf59333ab4980cc7e", size = 49376032, upload-time = "2026-04-21T10:47:48.967Z" }, + { url = "https://files.pythonhosted.org/packages/36/7a/82c363caa145fff88fb475da50d3bf52bb024f61917be5424c3392eaf878/pyarrow-24.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:25ea65d868eb04015cd18e6df2fbe98f07e5bda2abefabcb88fce39a947716f6", size = 51929490, upload-time = "2026-04-21T10:47:55.981Z" }, + { url = "https://files.pythonhosted.org/packages/66/1c/e3e72c8014ad2743ca64a701652c733cc5cbcee15c0463a32a8c55518d9e/pyarrow-24.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:295f0a7f2e242dabd513737cf076007dc5b2d59237e3eca37b05c0c6446f3826", size = 27355660, upload-time = "2026-04-21T10:48:01.718Z" }, + { url = "https://files.pythonhosted.org/packages/6f/d3/a1abf004482026ddc17f4503db227787fa3cfe41ec5091ff20e4fea55e57/pyarrow-24.0.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:02b001b3ed4723caa44f6cd1af2d5c86aa2cf9971dacc2ffa55b21237713dfba", size = 34976759, upload-time = "2026-04-21T10:48:07.258Z" }, + { url = "https://files.pythonhosted.org/packages/4f/4a/34f0a36d28a2dd32225301b79daad44e243dc1a2bb77d43b60749be255c4/pyarrow-24.0.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:04920d6a71aabd08a0417709efce97d45ea8e6fb733d9ca9ecffb13c67839f68", size = 36658471, upload-time = "2026-04-21T10:48:13.347Z" }, + { url = "https://files.pythonhosted.org/packages/1f/78/543b94712ae8bb1a6023bcc1acf1a740fbff8286747c289cd9468fced2a5/pyarrow-24.0.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:a964266397740257f16f7bb2e4f08a0c81454004beab8ff59dd531b73610e9f2", size = 45675981, upload-time = "2026-04-21T10:48:20.201Z" }, + { url = "https://files.pythonhosted.org/packages/84/9f/8fb7c222b100d314137fa40ec050de56cd8c6d957d1cfff685ce72f15b17/pyarrow-24.0.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:6f066b179d68c413374294bc1735f68475457c933258df594443bb9d88ddc2a0", size = 48859172, upload-time = "2026-04-21T10:48:27.541Z" }, + { url = "https://files.pythonhosted.org/packages/a7/d3/1ea72538e6c8b3b475ed78d1049a2c518e655761ea50fe1171fc855fcab7/pyarrow-24.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1183baeb14c5f587b1ec52831e665718ce632caab84b7cd6b85fd44f96114495", size = 49385733, upload-time = "2026-04-21T10:48:34.7Z" }, + { url = "https://files.pythonhosted.org/packages/c3/be/c3d8b06a1ba35f2260f8e1f771abbee7d5e345c0937aab90675706b1690a/pyarrow-24.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:806f24b4085453c197a5078218d1ee08783ebbba271badd153d1ae22a3ee804f", size = 51934335, upload-time = "2026-04-21T10:48:42.099Z" }, + { url = "https://files.pythonhosted.org/packages/9c/62/89e07a1e7329d2cde3e3c6994ba0839a24977a2beda8be6005ea3d860b99/pyarrow-24.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:e4505fc6583f7b05ab854934896bcac8253b04ac1171a77dfb73efef92076d91", size = 27271748, upload-time = "2026-04-21T10:49:42.532Z" }, + { url = "https://files.pythonhosted.org/packages/17/1a/cff3a59f80b5b1658549d46611b67163f65e0664431c076ad728bf9d5af4/pyarrow-24.0.0-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:1a4e45017efbf115032e4475ee876d525e0e36c742214fbe405332480ecd6275", size = 35238554, upload-time = "2026-04-21T10:48:48.526Z" }, + { url = "https://files.pythonhosted.org/packages/a8/99/cce0f42a327bfef2c420fb6078a3eb834826e5d6697bf3009fe11d2ad051/pyarrow-24.0.0-cp313-cp313t-macosx_12_0_x86_64.whl", hash = "sha256:7986f1fa71cee060ad00758bcc79d3a93bab8559bf978fab9e53472a2e25a17b", size = 36782301, upload-time = "2026-04-21T10:48:55.181Z" }, + { url = "https://files.pythonhosted.org/packages/2a/66/8e560d5ff6793ca29aca213c53eec0dd482dd46cb93b2819e5aab52e4252/pyarrow-24.0.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:d3e0b61e8efb24ed38898e5cdc5fffa9124be480008d401a1f8071500494ae42", size = 45721929, upload-time = "2026-04-21T10:49:03.676Z" }, + { url = "https://files.pythonhosted.org/packages/27/0c/a26e25505d030716e078d9f16eb74973cbf0b33b672884e9f9da1c83b871/pyarrow-24.0.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:55a3bc1e3df3b5567b7d27ef551b2283f0c68a5e86f1cd56abc569da4f31335b", size = 48825365, upload-time = "2026-04-21T10:49:11.714Z" }, + { url = "https://files.pythonhosted.org/packages/5f/eb/771f9ecb0c65e73fe9dccdd1717901b9594f08c4515d000c7c62df573811/pyarrow-24.0.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:641f795b361874ac9da5294f8f443dfdbee355cf2bd9e3b8d97aaac2306b9b37", size = 49451819, upload-time = "2026-04-21T10:49:21.474Z" }, + { url = "https://files.pythonhosted.org/packages/48/da/61ae89a88732f5a785646f3ec6125dbb640fa98a540eb2b9889caa561403/pyarrow-24.0.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8adc8e6ce5fccf5dc707046ae4914fd537def529709cc0d285d37a7f9cd442ca", size = 51909252, upload-time = "2026-04-21T10:49:31.164Z" }, + { url = "https://files.pythonhosted.org/packages/cb/1a/8dd5cafab7b66573fa91c03d06d213356ad4edd71813aa75e08ce2b3a844/pyarrow-24.0.0-cp313-cp313t-win_amd64.whl", hash = "sha256:9b18371ad2f44044b81a8d23bc2d8a9b6a6226dca775e8e16cfee640473d6c5d", size = 27388127, upload-time = "2026-04-21T10:49:37.334Z" }, + { url = "https://files.pythonhosted.org/packages/ad/80/d022a34ff05d2cbedd8ccf841fc1f532ecfa9eb5ed1711b56d0e0ea71fc9/pyarrow-24.0.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:1cc9057f0319e26333b357e17f3c2c022f1a83739b48a88b25bfd5fa2dc18838", size = 35007997, upload-time = "2026-04-21T10:49:48.796Z" }, + { url = "https://files.pythonhosted.org/packages/1a/ff/f01485fda6f4e5d441afb8dd5e7681e4db18826c1e271852f5d3957d6a80/pyarrow-24.0.0-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:e6f1278ee4785b6db21229374a1c9e54ec7c549de5d1efc9630b6207de7e170b", size = 36678720, upload-time = "2026-04-21T10:49:55.858Z" }, + { url = "https://files.pythonhosted.org/packages/9e/c2/2d2d5fea814237923f71b36495211f20b43a1576f9a4d6da7e751a64ec6f/pyarrow-24.0.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:adbbedc55506cbdabb830890444fb856bfb0060c46c6f8026c6c2f2cf86ae795", size = 45741852, upload-time = "2026-04-21T10:50:04.624Z" }, + { url = "https://files.pythonhosted.org/packages/8e/3a/28ba9c1c1ebdbb5f1b94dfebb46f207e52e6a554b7fe4132540fde29a3a0/pyarrow-24.0.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:ae8a1145af31d903fa9bb166824d7abe9b4681a000b0159c9fb99c11bc11ad26", size = 48889852, upload-time = "2026-04-21T10:50:12.293Z" }, + { url = "https://files.pythonhosted.org/packages/df/51/4a389acfd31dca009f8fb82d7f510bb4130f2b3a8e18cf00194d0687d8ac/pyarrow-24.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d7027eba1df3b2069e2e8d80f644fa0918b68c46432af3d088ddd390d063ecde", size = 49445207, upload-time = "2026-04-21T10:50:20.677Z" }, + { url = "https://files.pythonhosted.org/packages/19/4b/0bab2b23d2ae901b1b9a03c0efd4b2d070256f8ce3fc43f6e58c167b2081/pyarrow-24.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e56a1ffe9bf7b727432b89104cc0849c21582949dd7bdcb34f17b2001a351a76", size = 51954117, upload-time = "2026-04-21T10:50:29.14Z" }, + { url = "https://files.pythonhosted.org/packages/29/88/f4e9145da0417b3d2c12035a8492b35ff4a3dbc653e614fcfb51d9dedb38/pyarrow-24.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:38be1808cdd068605b787e6ca9119b27eb275a0234e50212c3492331680c3b1e", size = 28001155, upload-time = "2026-04-21T10:51:22.337Z" }, + { url = "https://files.pythonhosted.org/packages/79/4f/46a49a63f43526da895b1a45bbb51d5baf8e4d77159f8528fc3e5490007f/pyarrow-24.0.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:418e48ce50a45a6a6c73c454677203a9c75c966cb1e92ca3370959185f197a05", size = 35250387, upload-time = "2026-04-21T10:50:35.552Z" }, + { url = "https://files.pythonhosted.org/packages/a0/da/d5e0cd5ef00796922404806d5f00325cdadc3441ce2c13fe7115f2df9a64/pyarrow-24.0.0-cp314-cp314t-macosx_12_0_x86_64.whl", hash = "sha256:2f16197705a230a78270cdd4ea8a1d57e86b2fdcbc34a1f6aebc72e65c986f9a", size = 36797102, upload-time = "2026-04-21T10:50:42.417Z" }, + { url = "https://files.pythonhosted.org/packages/34/c7/5904145b0a593a05236c882933d439b5720f0a145381179063722fbfc123/pyarrow-24.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:fb24ac194bfc5e86839d7dcd52092ee31e5fe6733fe11f5e3b06ef0812b20072", size = 45745118, upload-time = "2026-04-21T10:50:49.324Z" }, + { url = "https://files.pythonhosted.org/packages/13/d3/cca42fe166d1c6e4d5b80e530b7949104d10e17508a90ae202dac205ce2a/pyarrow-24.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:9700ebd9a51f5895ce75ff4ac4b3c47a7d4b42bc618be8e713e5d56bacf5f931", size = 48844765, upload-time = "2026-04-21T10:50:55.579Z" }, + { url = "https://files.pythonhosted.org/packages/b0/49/942c3b79878ba928324d1e17c274ed84581db8c0a749b24bcf4cbdf15bd3/pyarrow-24.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:d8ddd2768da81d3ee08cfea9b597f4abb4e8e1dc8ae7e204b608d23a0d3ab699", size = 49471890, upload-time = "2026-04-21T10:51:02.439Z" }, + { url = "https://files.pythonhosted.org/packages/76/97/ff71431000a75d84135a1ace5ca4ba11726a231a8007bbb320a4c54075d5/pyarrow-24.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:61a3d7eaa97a14768b542f3d284dc6400dd2470d9f080708b13cd46b6ae18136", size = 51932250, upload-time = "2026-04-21T10:51:10.576Z" }, + { url = "https://files.pythonhosted.org/packages/51/be/6f79d55816d5c22557cf27533543d5d70dfe692adfbee4b99f2760674f38/pyarrow-24.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:c91d00057f23b8d353039520dc3a6c09d8608164c692e9f59a175a42b2ae0c19", size = 28131282, upload-time = "2026-04-21T10:51:16.815Z" }, ] [[package]] From 59c5d11873a5da74f31bb5b6b4819a3b1ca8ba8c Mon Sep 17 00:00:00 2001 From: gkennos Date: Sun, 5 Jul 2026 23:22:33 +1000 Subject: [PATCH 4/4] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- src/orm_loader/loaders/loading_helpers.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/orm_loader/loaders/loading_helpers.py b/src/orm_loader/loaders/loading_helpers.py index 6574aac..7df8571 100644 --- a/src/orm_loader/loaders/loading_helpers.py +++ b/src/orm_loader/loaders/loading_helpers.py @@ -168,10 +168,10 @@ def resolve_quote_mode( guess wrong when the discriminating rows fall outside the sample. - ``"by_delimiter"`` — derive the mode from the delimiter alone, no content scan: tab-delimited input never embeds the delimiter in a field so any - double-quote is literal data (``"literal"``); comma-delimited input must - wrap fields containing the delimiter in quotes so quoting is meaningful - (``"csv"``). Deterministic and immune to the sampling gamble; use it when - the source guarantees tab fields never contain embedded tabs/newlines. + double-quote is literal data (``"literal"``); for non-tab delimiters this + resolves to ``"csv"`` where RFC-4180 quoting can be meaningful. Deterministic + and immune to the sampling gamble; use it when the source guarantees tab + fields never contain embedded tabs/newlines. """ if quote_mode in ("csv", "literal"): return quote_mode