From 8cee2e844f40677e853990a056b18d5ccc066975 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Tue, 7 Jul 2026 20:24:16 +0530 Subject: [PATCH 1/9] Add standalone mssql-python-odbc package for ODBC driver binaries Introduce mssql_python_odbc, a pure-data sibling package (v18.6.0) that ships the ODBC driver libs/ tree, plus setup_odbc.py which builds platform-specific, Python-agnostic (py3-none-) wheels for the 7 supported platforms. ddbc_bindings.cpp now resolves the driver/libs base directory from mssql_python_odbc when installed, and falls back to the bundled mssql_python libs during the Phase-2 transition (bundled libs are retained). Add tests/test_024_odbc_package_split.py to verify Python/C++ driver-path parity, and .gitignore entries for local build copies. --- .gitignore | 7 + mssql_python/pybind/ddbc_bindings.cpp | 43 +++++- mssql_python_odbc/__init__.py | 123 +++++++++++++++ setup_odbc.py | 213 ++++++++++++++++++++++++++ tests/test_024_odbc_package_split.py | 142 +++++++++++++++++ 5 files changed, 526 insertions(+), 2 deletions(-) create mode 100644 mssql_python_odbc/__init__.py create mode 100644 setup_odbc.py create mode 100644 tests/test_024_odbc_package_split.py diff --git a/.gitignore b/.gitignore index 080fc9e5..cd64adb9 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,13 @@ build/ **/build/ mssql_python.egg-info/ +# ODBC package (mssql-python-odbc): transition-period build artifacts. +# During Phase 2, setup_odbc.py copies the current platform's driver binaries +# from mssql_python/libs into this tree so a wheel can be built locally. The +# release pipeline populates these per-platform; do not commit the copies. +mssql_python_odbc/libs/ +mssql_python_odbc.egg-info/ + # Python bytecode __pycache__/ *.py[cod] diff --git a/mssql_python/pybind/ddbc_bindings.cpp b/mssql_python/pybind/ddbc_bindings.cpp index 3cb00814..b1713263 100644 --- a/mssql_python/pybind/ddbc_bindings.cpp +++ b/mssql_python/pybind/ddbc_bindings.cpp @@ -1125,6 +1125,41 @@ std::string GetModuleDirectory() { return parentDir.string(); } +// Resolve the base directory that contains the ODBC driver `libs/` tree. +// +// Post-split, the driver binaries ship in the standalone `mssql_python_odbc` +// package (a pure-data sibling with no native extension). We import it and use +// its directory as the base that `GetDriverPathCpp` (and the Windows +// `mssql-auth.dll` lookup) append `libs` to. +// +// During the Phase-2 transition we fall back to the bundled `mssql_python` +// directory when the external package is not installed, so a wheel that still +// bundles `libs/` keeps working. Importing `mssql_python_odbc` here is +// Alpine/musl-safe precisely because it is a separate pure package: it cannot +// trigger the partially-initialized-module circular import that motivated +// resolving these paths in C++ in the first place. +std::string GetOdbcLibsBaseDir() { + namespace fs = std::filesystem; + try { + py::object module = py::module::import("mssql_python_odbc"); + py::object module_path = module.attr("__file__"); + std::string module_file = module_path.cast(); + + fs::path parentDir = fs::path(module_file).parent_path(); + LOG("GetOdbcLibsBaseDir: Using external mssql_python_odbc package - directory='%s'", + parentDir.string().c_str()); + return parentDir.string(); + } catch (const py::error_already_set& e) { + // External package not installed. pybind11 has already fetched and + // cleared the CPython error indicator when constructing this + // exception, so it is safe to import `mssql_python` again below. + LOG("GetOdbcLibsBaseDir: mssql_python_odbc not available (%s); " + "falling back to bundled libs in mssql_python", + e.what()); + return GetModuleDirectory(); + } +} + // Platform-agnostic function to load the driver dynamic library DriverHandle LoadDriverLibrary(const std::string& driverPath) { LOG("LoadDriverLibrary: Attempting to load ODBC driver from path='%s'", driverPath.c_str()); @@ -1240,8 +1275,12 @@ std::string GetDriverPathCpp(const std::string& moduleDir) { DriverHandle LoadDriverOrThrowException() { namespace fs = std::filesystem; - std::string moduleDir = GetModuleDirectory(); - LOG("LoadDriverOrThrowException: Module directory resolved to '%s'", moduleDir.c_str()); + // Resolve the base dir from the standalone `mssql_python_odbc` package + // (falls back to the bundled `mssql_python` libs during the transition). + // Both the driver path and the Windows `mssql-auth.dll` path below are + // derived from this directory. + std::string moduleDir = GetOdbcLibsBaseDir(); + LOG("LoadDriverOrThrowException: ODBC libs base directory resolved to '%s'", moduleDir.c_str()); std::string archStr = ARCHITECTURE; LOG("LoadDriverOrThrowException: Architecture detected as '%s'", archStr.c_str()); diff --git a/mssql_python_odbc/__init__.py b/mssql_python_odbc/__init__.py new file mode 100644 index 00000000..f8c64e1a --- /dev/null +++ b/mssql_python_odbc/__init__.py @@ -0,0 +1,123 @@ +""" +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. + +mssql_python_odbc — Microsoft ODBC Driver 18 for SQL Server binaries. + +Internal implementation package for ``mssql-python``. It ships the +platform-specific ODBC driver binaries (``msodbcsql18``) and their supporting +libraries so that ``mssql-python`` does not have to bundle them in its own +wheel. It is not meant for direct consumption — install ``mssql-python`` +instead, which depends on this package. + +The public surface is :func:`get_driver_path`, which returns the absolute path +to the ODBC driver shared library for the current platform/architecture. The +native ``mssql_python.ddbc_bindings`` extension resolves the same location in +C++ (see ``GetOdbcLibsBaseDir`` / ``GetDriverPathCpp``); this Python API mirrors +that logic for tooling, tests, and diagnostics. +""" + +import os +import platform +import sys + +__all__ = ["get_driver_path", "get_libs_dir", "__version__"] + +# Version tracks the bundled Microsoft ODBC Driver 18 for SQL Server release. +__version__ = "18.6.0" + +# Driver shared-library file names per platform (must match the names produced +# by the ODBC driver packaging and expected by the native loader). +_DRIVER_FILENAME = { + "windows": "msodbcsql18.dll", + "linux": "libmsodbcsql-18.6.so.2.1", + "macos": "libmsodbcsql.18.dylib", +} + + +def get_libs_dir() -> str: + """Return the absolute path to this package's ``libs/`` directory. + + This is the root under which the platform-specific ODBC binaries live + (``libs///...``). It is the base the native loader appends + ``libs`` to when resolving the driver. + """ + return os.path.join(os.path.dirname(os.path.abspath(__file__)), "libs") + + +def _detect_arch() -> str: + """Return the architecture directory name for the current interpreter. + + Mirrors the compile-time detection in ``GetDriverPathCpp``: + * Windows uses ``x64`` / ``x86`` / ``arm64``. + * Linux and macOS use ``x86_64`` / ``arm64``. + """ + machine = platform.machine().lower() + + if sys.platform.startswith("win"): + if machine in ("amd64", "x86_64"): + return "x64" + if machine in ("arm64", "aarch64"): + return "arm64" + if machine in ("x86", "i386", "i686"): + return "x86" + raise OSError(f"Unsupported Windows architecture: {platform.machine()!r}") + + # Linux / macOS + if machine in ("x86_64", "amd64"): + return "x86_64" + if machine in ("arm64", "aarch64"): + return "arm64" + raise OSError(f"Unsupported architecture: {platform.machine()!r}") + + +def _detect_linux_distro_family() -> str: + """Return the Linux distro family directory name. + + Mirrors the ``/etc/*-release`` probing in ``GetDriverPathCpp`` so the Python + and C++ paths agree. + """ + if os.path.exists("/etc/alpine-release"): + return "alpine" + if os.path.exists("/etc/redhat-release") or os.path.exists("/etc/centos-release"): + return "rhel" + if os.path.exists("/etc/SuSE-release") or os.path.exists("/etc/SUSE-brand"): + return "suse" + return "debian_ubuntu" # default for Debian/Ubuntu and other glibc distros + + +def get_driver_path() -> str: + """Return the absolute path to the ODBC driver shared library. + + Resolves the platform/architecture (and, on Linux, the distro family) and + returns the full path to ``msodbcsql18`` inside this package. + + Raises: + OSError: if the platform/architecture is unsupported. + FileNotFoundError: if the resolved driver file is not present (e.g. the + package was built without this platform's binaries). + """ + libs_dir = get_libs_dir() + arch = _detect_arch() + + if sys.platform.startswith("win"): + driver_path = os.path.join(libs_dir, "windows", arch, _DRIVER_FILENAME["windows"]) + elif sys.platform.startswith("darwin"): + driver_path = os.path.join( + libs_dir, "macos", arch, "lib", _DRIVER_FILENAME["macos"] + ) + elif sys.platform.startswith("linux"): + distro = _detect_linux_distro_family() + driver_path = os.path.join( + libs_dir, "linux", distro, arch, "lib", _DRIVER_FILENAME["linux"] + ) + else: + raise OSError(f"Unsupported platform: {sys.platform!r}") + + if not os.path.isfile(driver_path): + raise FileNotFoundError( + f"ODBC driver not found at {driver_path!r}. The mssql-python-odbc " + f"package may not include binaries for this platform/architecture." + ) + + return driver_path diff --git a/setup_odbc.py b/setup_odbc.py new file mode 100644 index 00000000..e6bed156 --- /dev/null +++ b/setup_odbc.py @@ -0,0 +1,213 @@ +""" +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. + +Build script for the ``mssql-python-odbc`` package. + +This packages the Microsoft ODBC Driver 18 for SQL Server binaries into a +standalone, platform-specific wheel that ``mssql-python`` depends on. Build it +with:: + + python setup_odbc.py bdist_wheel + +During the transition period the driver binaries still live under +``mssql_python/libs/``. This script copies the current platform's subtree into +``mssql_python_odbc/libs/`` so a wheel can be produced locally. The release +pipeline populates ``libs/`` per-platform and is the source of truth for the +full release matrix. +""" + +import os +import shutil +import sys +from pathlib import Path + +from setuptools import setup +from setuptools.dist import Distribution +from wheel.bdist_wheel import bdist_wheel + +PROJECT_ROOT = Path(__file__).resolve().parent +PACKAGE_NAME = "mssql_python_odbc" +PACKAGE_DIR = PROJECT_ROOT / PACKAGE_NAME +BUNDLED_LIBS_ROOT = PROJECT_ROOT / "mssql_python" / "libs" + + +class BinaryDistribution(Distribution): + """Force a platform-specific wheel (the package ships native binaries).""" + + def has_ext_modules(self): + return True + + +def get_platform_info(): + """Get platform-specific architecture and platform tag information. + + Kept in sync with ``setup.py`` so the ODBC wheel carries the same platform + tags as the main ``mssql-python`` wheel. + """ + if sys.platform.startswith("win"): + arch = os.environ.get("ARCHITECTURE", "x64") + if isinstance(arch, str): + arch = arch.strip("\"'") + if arch in ["x86", "win32"]: + return "x86", "win32" + elif arch == "arm64": + return "arm64", "win_arm64" + else: + return "x64", "win_amd64" + + elif sys.platform.startswith("darwin"): + return "universal2", "macosx_15_0_universal2" + + elif sys.platform.startswith("linux"): + import platform + + target_arch = os.environ.get("targetArch", platform.machine()) + libc_name, _ = platform.libc_ver() + is_musl = libc_name == "" or "musl" in libc_name.lower() + manylinux_tag = os.environ.get("MANYLINUX_TAG", "manylinux_2_28") + + if target_arch == "x86_64": + return "x86_64", "musllinux_1_2_x86_64" if is_musl else f"{manylinux_tag}_x86_64" + elif target_arch in ["aarch64", "arm64"]: + return "aarch64", "musllinux_1_2_aarch64" if is_musl else f"{manylinux_tag}_aarch64" + else: + raise OSError( + f"Unsupported architecture '{target_arch}' for Linux; " + f"expected 'x86_64' or 'aarch64'." + ) + + raise OSError(f"Unsupported platform: {sys.platform!r}") + + +def _libs_arch(build_arch: str) -> str: + """Map the build arch from ``get_platform_info`` to the ``libs/`` dir name. + + ``libs/`` uses ``x64``/``x86``/``arm64`` on Windows and + ``x86_64``/``arm64`` on Linux/macOS. + """ + if sys.platform.startswith("win"): + return build_arch # already x64 / x86 / arm64 + if build_arch in ("x86_64", "amd64"): + return "x86_64" + if build_arch in ("aarch64", "arm64"): + return "arm64" + return build_arch + + +def _copytree(src: Path, dst: Path) -> None: + if src.is_dir(): + shutil.copytree(src, dst, dirs_exist_ok=True) + print(f" Copied {src} -> {dst}") + + +def sync_libs() -> None: + """Copy the current platform's ODBC libs into ``mssql_python_odbc/libs/``. + + Convenience for local/transition builds while ``mssql_python`` still bundles + the binaries. If ``mssql_python/libs`` is absent (e.g. the pipeline places + binaries directly into the package), this is a no-op. + """ + target_root = PACKAGE_DIR / "libs" + if not BUNDLED_LIBS_ROOT.is_dir(): + print(f"sync_libs: bundled libs not found at {BUNDLED_LIBS_ROOT}; skipping copy") + return + + build_arch, _ = get_platform_info() + arch = _libs_arch(build_arch) + + # Always carry the licensing files. + _copytree(BUNDLED_LIBS_ROOT / "LICENSING", target_root / "LICENSING") + + if sys.platform.startswith("win"): + _copytree(BUNDLED_LIBS_ROOT / "windows" / arch, target_root / "windows" / arch) + + elif sys.platform.startswith("darwin"): + # universal2 wheel serves both architectures. + for mac_arch in ("arm64", "x86_64"): + _copytree(BUNDLED_LIBS_ROOT / "macos" / mac_arch, target_root / "macos" / mac_arch) + + elif sys.platform.startswith("linux"): + # A single Linux wheel serves all distro families for its libc/arch; + # the driver is selected at runtime via /etc/*-release detection. + for distro in ("alpine", "debian_ubuntu", "rhel", "suse"): + _copytree( + BUNDLED_LIBS_ROOT / "linux" / distro / arch, + target_root / "linux" / distro / arch, + ) + + +class CustomBdistWheel(bdist_wheel): + """Force a platform-specific but Python-agnostic tag and sync libs. + + The package ships only pre-built ODBC driver binaries (data), not a compiled + Python extension, so one ``py3-none-`` wheel serves every supported + Python version (3.10+). The wheel stays platform-specific + (``root_is_pure = False``) because the binaries differ per OS/arch/libc, but + the interpreter/ABI tags are forced to ``py3``/``none``. Without this the + ``BinaryDistribution`` (``has_ext_modules`` -> True) would produce a + ``cp3XX``-specific tag, forcing a needless per-Python-version build matrix. + """ + + def finalize_options(self): + bdist_wheel.finalize_options(self) + arch, platform_tag = get_platform_info() + self.plat_name = platform_tag + # Platform-specific (ships native binaries) but not tied to a CPython ABI. + self.root_is_pure = False + print(f"Setting wheel platform tag to: {self.plat_name} (arch: {arch})") + + def get_tag(self): + # Preserve the platform tag from the base implementation but relabel the + # interpreter/ABI tags as Python-agnostic ("py3"/"none"). + _python, _abi, plat = bdist_wheel.get_tag(self) + return "py3", "none", plat + + def run(self): + sync_libs() + bdist_wheel.run(self) + + +setup( + name="mssql-python-odbc", + version="18.6.0", + description=( + "Internal implementation package for mssql-python: Microsoft ODBC " + "Driver 18 for SQL Server binaries. Not intended for direct use." + ), + long_description=( + "Internal implementation package not meant for direct consumption. " + "Install `mssql-python`, which depends on this package." + ), + long_description_content_type="text/plain", + author="Microsoft Corporation", + author_email="mssql-python@microsoft.com", + url="https://github.com/microsoft/mssql-python", + license="MIT", + packages=[PACKAGE_NAME], + package_data={ + PACKAGE_NAME: [ + "libs/*", + "libs/**/*", + ], + }, + include_package_data=True, + python_requires=">=3.10", + classifiers=[ + "License :: OSI Approved :: MIT License", + "Operating System :: Microsoft :: Windows", + "Operating System :: MacOS", + "Operating System :: POSIX :: Linux", + ], + zip_safe=False, + distclass=BinaryDistribution, + exclude_package_data={ + PACKAGE_NAME: [ + "libs/*/vcredist/*", + "libs/*/vcredist/**/*", + ], + }, + cmdclass={ + "bdist_wheel": CustomBdistWheel, + }, +) diff --git a/tests/test_024_odbc_package_split.py b/tests/test_024_odbc_package_split.py new file mode 100644 index 00000000..d337a21d --- /dev/null +++ b/tests/test_024_odbc_package_split.py @@ -0,0 +1,142 @@ +""" +Tests for the ODBC driver package split (mssql-python-odbc). + +Part of splitting the bundled ODBC driver binaries out of ``mssql-python`` into +the standalone ``mssql-python-odbc`` package. These tests validate the Python +driver-path API and, crucially, that it agrees with the native C++ resolver +(``ddbc_bindings.GetDriverPathCpp``) so the path Python reports is exactly the +one the driver is loaded from at connect time. + +Notes: +- ``mssql_python_odbc/libs`` is populated per-platform at build time (and copied + locally by ``setup_odbc.py`` during the transition). Tests that need the actual + driver file skip when the libs directory is absent (e.g. a fresh CI checkout + where the binaries have not been synced). +- The C++ fallback (``GetOdbcLibsBaseDir`` -> bundled ``mssql_python`` libs when + the external package is not installed) is verified manually rather than here, + since exercising it requires uninstalling the package mid-run. +""" + +import os +import sys + +import pytest + +# Skip the whole module cleanly if the package is not importable. +mssql_python_odbc = pytest.importorskip("mssql_python_odbc") + + +EXPECTED_DRIVER_FILENAME = { + "win32": "msodbcsql18.dll", + "darwin": "libmsodbcsql.18.dylib", + "linux": "libmsodbcsql-18.6.so.2.1", +} + + +def _platform_key(): + if sys.platform.startswith("win"): + return "win32" + if sys.platform.startswith("darwin"): + return "darwin" + if sys.platform.startswith("linux"): + return "linux" + return sys.platform + + +def _package_dir(): + """Base directory the native loader appends ``libs`` to.""" + return os.path.dirname(os.path.abspath(mssql_python_odbc.__file__)) + + +def _libs_present(): + return os.path.isdir(mssql_python_odbc.get_libs_dir()) + + +_NO_LIBS_REASON = ( + "ODBC libs not present in package (fresh checkout, or not built/synced for " + "this platform)" +) + + +class TestOdbcPackageMetadata: + def test_version_is_driver_version(self): + assert mssql_python_odbc.__version__ == "18.6.0" + + def test_public_api_present(self): + assert callable(mssql_python_odbc.get_driver_path) + assert callable(mssql_python_odbc.get_libs_dir) + + +class TestLibsDir: + def test_libs_dir_under_package(self): + libs_dir = mssql_python_odbc.get_libs_dir() + assert os.path.basename(libs_dir) == "libs" + assert libs_dir.startswith(_package_dir()) + + +class TestArchDetection: + def test_detect_arch_matches_platform(self): + arch = mssql_python_odbc._detect_arch() + if sys.platform.startswith("win"): + assert arch in ("x64", "x86", "arm64") + else: + assert arch in ("x86_64", "arm64") + + @pytest.mark.skipif( + not sys.platform.startswith("linux"), + reason="Linux-only distro-family detection", + ) + def test_detect_linux_distro_family(self): + distro = mssql_python_odbc._detect_linux_distro_family() + assert distro in ("alpine", "rhel", "suse", "debian_ubuntu") + + +class TestDriverPath: + @pytest.mark.skipif(not _libs_present(), reason=_NO_LIBS_REASON) + def test_driver_path_layout_and_exists(self): + driver_path = mssql_python_odbc.get_driver_path() + # Correct driver filename for this platform. + assert os.path.basename(driver_path) == EXPECTED_DRIVER_FILENAME[_platform_key()] + # Resolved under the package's own libs directory. + assert mssql_python_odbc.get_libs_dir() in driver_path + # The resolved driver file actually exists on disk. + assert os.path.isfile(driver_path), f"driver not found at {driver_path}" + + @pytest.mark.skipif(not _libs_present(), reason=_NO_LIBS_REASON) + def test_driver_path_contains_expected_platform_dir(self): + driver_path = mssql_python_odbc.get_driver_path().replace("\\", "/") + expected_segment = { + "win32": "/libs/windows/", + "darwin": "/libs/macos/", + "linux": "/libs/linux/", + }[_platform_key()] + assert expected_segment in driver_path + + +class TestPythonCppParity: + """The Python ``get_driver_path()`` must resolve to the exact same path the + native C++ loader (``GetDriverPathCpp``) computes for the same base dir, so + tooling/tests agree with the driver actually loaded at connect time.""" + + @pytest.mark.skipif(not _libs_present(), reason=_NO_LIBS_REASON) + def test_get_driver_path_matches_cpp(self): + try: + from mssql_python import ddbc_bindings + except Exception as exc: # native extension not built / driver load failed + pytest.skip(f"ddbc_bindings unavailable: {exc}") + + get_cpp = getattr(ddbc_bindings, "GetDriverPathCpp", None) + if get_cpp is None: + pytest.skip("GetDriverPathCpp not exposed by this build") + + py_path = mssql_python_odbc.get_driver_path() + cpp_path = get_cpp(_package_dir()) + + def norm(p): + return os.path.normcase(os.path.normpath(p)) + + assert norm(py_path) == norm(cpp_path), ( + "Python/C++ driver path mismatch:\n" + f" python={py_path}\n" + f" cpp ={cpp_path}" + ) From 69ca1b7eb6cfbc059e3491f4ca0275220c67489b Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Tue, 7 Jul 2026 20:57:56 +0530 Subject: [PATCH 2/9] FIX: Robust ODBC driver fallback and address review feedback for package split --- mssql_python/pybind/ddbc_bindings.cpp | 42 ++++++++++++++++++++++----- mssql_python_odbc/__init__.py | 5 ++-- setup_odbc.py | 4 +-- tests/test_024_odbc_package_split.py | 21 +++++++++----- 4 files changed, 53 insertions(+), 19 deletions(-) diff --git a/mssql_python/pybind/ddbc_bindings.cpp b/mssql_python/pybind/ddbc_bindings.cpp index b1713263..1412b1d5 100644 --- a/mssql_python/pybind/ddbc_bindings.cpp +++ b/mssql_python/pybind/ddbc_bindings.cpp @@ -1138,6 +1138,11 @@ std::string GetModuleDirectory() { // Alpine/musl-safe precisely because it is a separate pure package: it cannot // trigger the partially-initialized-module circular import that motivated // resolving these paths in C++ in the first place. +// +// (`GetDriverPathCpp` is defined further below; forward-declared here so we can +// verify the external package actually ships this platform's driver binary.) +std::string GetDriverPathCpp(const std::string& moduleDir); + std::string GetOdbcLibsBaseDir() { namespace fs = std::filesystem; try { @@ -1146,17 +1151,38 @@ std::string GetOdbcLibsBaseDir() { std::string module_file = module_path.cast(); fs::path parentDir = fs::path(module_file).parent_path(); - LOG("GetOdbcLibsBaseDir: Using external mssql_python_odbc package - directory='%s'", + + // Only treat the external package as authoritative if it actually ships + // this platform's driver binary. In a source/dev checkout (and in CI) + // the package is importable from the repo root but its `libs/` tree is + // gitignored and absent; in that case fall back to the bundled + // `mssql_python` libs so driver resolution still points at a real binary. + std::error_code ec; + if (fs::exists(GetDriverPathCpp(parentDir.string()), ec)) { + LOG("GetOdbcLibsBaseDir: Using external mssql_python_odbc package - directory='%s'", + parentDir.string().c_str()); + return parentDir.string(); + } + LOG("GetOdbcLibsBaseDir: mssql_python_odbc present at '%s' but has no driver binary; " + "falling back to bundled libs in mssql_python", parentDir.string().c_str()); - return parentDir.string(); + return GetModuleDirectory(); } catch (const py::error_already_set& e) { - // External package not installed. pybind11 has already fetched and - // cleared the CPython error indicator when constructing this - // exception, so it is safe to import `mssql_python` again below. - LOG("GetOdbcLibsBaseDir: mssql_python_odbc not available (%s); " - "falling back to bundled libs in mssql_python", + if (e.matches(PyExc_ModuleNotFoundError)) { + // Expected in Phase 2 when the standalone package is not installed. + // pybind11 has already fetched and cleared the CPython error + // indicator, so re-importing `mssql_python` below is safe. + LOG("GetOdbcLibsBaseDir: mssql_python_odbc not installed (%s); " + "falling back to bundled libs in mssql_python", + e.what()); + return GetModuleDirectory(); + } + // A different import-time error means the package is installed but + // broken; surface it instead of silently masking the real problem. + LOG("GetOdbcLibsBaseDir: importing mssql_python_odbc failed unexpectedly (%s); " + "re-raising", e.what()); - return GetModuleDirectory(); + throw; } } diff --git a/mssql_python_odbc/__init__.py b/mssql_python_odbc/__init__.py index f8c64e1a..90219724 100644 --- a/mssql_python_odbc/__init__.py +++ b/mssql_python_odbc/__init__.py @@ -39,8 +39,9 @@ def get_libs_dir() -> str: """Return the absolute path to this package's ``libs/`` directory. This is the root under which the platform-specific ODBC binaries live - (``libs///...``). It is the base the native loader appends - ``libs`` to when resolving the driver. + (``libs///...``). The parent of this path (the package + directory) is the base the native loader appends ``libs`` to when resolving + the driver. """ return os.path.join(os.path.dirname(os.path.abspath(__file__)), "libs") diff --git a/setup_odbc.py b/setup_odbc.py index e6bed156..31c83fdd 100644 --- a/setup_odbc.py +++ b/setup_odbc.py @@ -203,8 +203,8 @@ def run(self): distclass=BinaryDistribution, exclude_package_data={ PACKAGE_NAME: [ - "libs/*/vcredist/*", - "libs/*/vcredist/**/*", + "libs/windows/*/vcredist/*", + "libs/windows/*/vcredist/**/*", ], }, cmdclass={ diff --git a/tests/test_024_odbc_package_split.py b/tests/test_024_odbc_package_split.py index d337a21d..f937e225 100644 --- a/tests/test_024_odbc_package_split.py +++ b/tests/test_024_odbc_package_split.py @@ -49,12 +49,21 @@ def _package_dir(): def _libs_present(): - return os.path.isdir(mssql_python_odbc.get_libs_dir()) + """True only when this platform's driver binary is actually present. + + Checks for the driver file rather than just the ``libs/`` directory, so a + stray or empty ``libs/`` in a dev checkout makes the driver tests skip + cleanly instead of failing. Mirrors the C++ resolver, which falls back to + the bundled libs unless the external package ships a real driver binary. + """ + try: + return os.path.isfile(mssql_python_odbc.get_driver_path()) + except OSError: + return False _NO_LIBS_REASON = ( - "ODBC libs not present in package (fresh checkout, or not built/synced for " - "this platform)" + "ODBC libs not present in package (fresh checkout, or not built/synced for " "this platform)" ) @@ -71,7 +80,7 @@ class TestLibsDir: def test_libs_dir_under_package(self): libs_dir = mssql_python_odbc.get_libs_dir() assert os.path.basename(libs_dir) == "libs" - assert libs_dir.startswith(_package_dir()) + assert os.path.normcase(libs_dir).startswith(os.path.normcase(_package_dir())) class TestArchDetection: @@ -136,7 +145,5 @@ def norm(p): return os.path.normcase(os.path.normpath(p)) assert norm(py_path) == norm(cpp_path), ( - "Python/C++ driver path mismatch:\n" - f" python={py_path}\n" - f" cpp ={cpp_path}" + "Python/C++ driver path mismatch:\n" f" python={py_path}\n" f" cpp ={cpp_path}" ) From 362b1df4e66711be8a95f0896818aa6f1a1dfc46 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Tue, 7 Jul 2026 21:16:31 +0530 Subject: [PATCH 3/9] FIX: Require complete odbc libs (driver + mssql-auth.dll) before using external package --- mssql_python/pybind/ddbc_bindings.cpp | 30 ++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/mssql_python/pybind/ddbc_bindings.cpp b/mssql_python/pybind/ddbc_bindings.cpp index 1412b1d5..39dda2e4 100644 --- a/mssql_python/pybind/ddbc_bindings.cpp +++ b/mssql_python/pybind/ddbc_bindings.cpp @@ -1153,18 +1153,34 @@ std::string GetOdbcLibsBaseDir() { fs::path parentDir = fs::path(module_file).parent_path(); // Only treat the external package as authoritative if it actually ships - // this platform's driver binary. In a source/dev checkout (and in CI) - // the package is importable from the repo root but its `libs/` tree is - // gitignored and absent; in that case fall back to the bundled - // `mssql_python` libs so driver resolution still points at a real binary. + // a COMPLETE set of this platform's driver binaries. In a source/dev + // checkout (and in CI) the package is importable from the repo root but + // its `libs/` tree is gitignored and either absent or only partially + // populated; in that case fall back to the bundled `mssql_python` libs + // so driver resolution points at a real, complete installation. + // + // "Complete" means the ODBC driver itself and, on Windows, the + // co-located `mssql-auth.dll` that LoadDriverOrThrowException loads + // unconditionally. Verifying both here keeps this resolver's notion of a + // usable base dir consistent with what the loader below actually needs, + // so we never select a directory that would later make the loader throw + // (e.g. a dir that has msodbcsql18.dll but is missing mssql-auth.dll). std::error_code ec; - if (fs::exists(GetDriverPathCpp(parentDir.string()), ec)) { + fs::path externalDriver(GetDriverPathCpp(parentDir.string())); + bool externalComplete = fs::exists(externalDriver, ec); +#ifdef _WIN32 + if (externalComplete) { + fs::path externalAuthDll = externalDriver.parent_path() / "mssql-auth.dll"; + externalComplete = fs::exists(externalAuthDll, ec); + } +#endif + if (externalComplete) { LOG("GetOdbcLibsBaseDir: Using external mssql_python_odbc package - directory='%s'", parentDir.string().c_str()); return parentDir.string(); } - LOG("GetOdbcLibsBaseDir: mssql_python_odbc present at '%s' but has no driver binary; " - "falling back to bundled libs in mssql_python", + LOG("GetOdbcLibsBaseDir: mssql_python_odbc present at '%s' but its libs are missing or " + "incomplete; falling back to bundled libs in mssql_python", parentDir.string().c_str()); return GetModuleDirectory(); } catch (const py::error_already_set& e) { From 25ff315de8c003be34990ee0a2252f69a415e097 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Tue, 7 Jul 2026 21:34:20 +0530 Subject: [PATCH 4/9] Include vcredist (msvcp140.dll) in odbc package for full parity with main libs --- setup_odbc.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/setup_odbc.py b/setup_odbc.py index 31c83fdd..91b10a9a 100644 --- a/setup_odbc.py +++ b/setup_odbc.py @@ -201,12 +201,6 @@ def run(self): ], zip_safe=False, distclass=BinaryDistribution, - exclude_package_data={ - PACKAGE_NAME: [ - "libs/windows/*/vcredist/*", - "libs/windows/*/vcredist/**/*", - ], - }, cmdclass={ "bdist_wheel": CustomBdistWheel, }, From c3c3001847d0419be9562810ec6479929fdfd570 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Tue, 7 Jul 2026 22:06:05 +0530 Subject: [PATCH 5/9] Address review: acquire GIL in GetOdbcLibsBaseDir, document driver-version coupling, guard minimum setuptools --- mssql_python/pybind/ddbc_bindings.cpp | 17 +++++++++++++++++ setup_odbc.py | 27 +++++++++++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/mssql_python/pybind/ddbc_bindings.cpp b/mssql_python/pybind/ddbc_bindings.cpp index 39dda2e4..c5099b8b 100644 --- a/mssql_python/pybind/ddbc_bindings.cpp +++ b/mssql_python/pybind/ddbc_bindings.cpp @@ -1145,6 +1145,13 @@ std::string GetDriverPathCpp(const std::string& moduleDir); std::string GetOdbcLibsBaseDir() { namespace fs = std::filesystem; + // This function calls into the Python C-API (py::module::import, attribute + // access, casts), so it must run with the GIL held. It is a no-op when the + // GIL is already held — which it is at the sole current call site, during + // module initialization — but acquiring it here self-documents the C-API + // dependency and keeps a future GIL-released caller from turning this into a + // hard crash. + py::gil_scoped_acquire gil; try { py::object module = py::module::import("mssql_python_odbc"); py::object module_path = module.attr("__file__"); @@ -1292,6 +1299,16 @@ std::string GetDriverPathCpp(const std::string& moduleDir) { platform = "debian_ubuntu"; // Default to debian_ubuntu for other distros } + // NOTE (version coupling): the Linux driver filename is pinned to the exact + // msodbcsql minor version (18.6 -> libmsodbcsql-18.6.so.2.1), matching the + // binaries bundled under mssql_python/libs and shipped by the standalone + // mssql_python_odbc package. If those driver binaries are upgraded (e.g. to + // 18.7) this literal MUST be updated in lockstep: GetOdbcLibsBaseDir() calls + // fs::exists() on this exact path to decide whether the external package is + // "complete", so a stale name would make it silently fall back to the + // bundled libs. (The macOS/Windows paths below key off the major version + // only and are more forgiving.) Tracked for the Phase 3 driver-version- + // agnostic resolution work. fs::path driverPath = basePath / "libs" / "linux" / platform / arch / "lib" / "libmsodbcsql-18.6.so.2.1"; return driverPath.string(); diff --git a/setup_odbc.py b/setup_odbc.py index 91b10a9a..3e756df5 100644 --- a/setup_odbc.py +++ b/setup_odbc.py @@ -18,10 +18,12 @@ """ import os +import re import shutil import sys from pathlib import Path +import setuptools from setuptools import setup from setuptools.dist import Distribution from wheel.bdist_wheel import bdist_wheel @@ -31,6 +33,29 @@ PACKAGE_DIR = PROJECT_ROOT / PACKAGE_NAME BUNDLED_LIBS_ROOT = PROJECT_ROOT / "mssql_python" / "libs" +# The driver binaries are packaged via the recursive ``libs/**/*`` glob in +# ``package_data`` below. Support for ``**`` recursive globs in ``package_data`` +# landed in setuptools 62.3.0; with an older setuptools the glob silently +# matches nothing and the wheel ships WITHOUT any driver binaries. The ``libs/`` +# tree is gitignored, so ``include_package_data`` cannot recover the files from +# version control either -- a stale setuptools would produce a broken-but-quiet +# wheel. Fail fast instead. +MIN_SETUPTOOLS = (62, 3, 0) + + +def _require_min_setuptools() -> None: + raw = setuptools.__version__ + parts = tuple(int(m) for m in re.findall(r"\d+", raw)[:3]) + parts += (0,) * (3 - len(parts)) + if parts < MIN_SETUPTOOLS: + raise SystemExit( + "setup_odbc.py requires setuptools >= " + f"{'.'.join(map(str, MIN_SETUPTOOLS))} to package the ODBC driver " + "binaries via the recursive 'libs/**/*' glob; found setuptools " + f"{raw}. Upgrade with:\n" + ' python -m pip install --upgrade "setuptools>=62.3.0"' + ) + class BinaryDistribution(Distribution): """Force a platform-specific wheel (the package ships native binaries).""" @@ -168,6 +193,8 @@ def run(self): bdist_wheel.run(self) +_require_min_setuptools() + setup( name="mssql-python-odbc", version="18.6.0", From d69ce14416c7e5554918f38fd8d01dbbc961abfc Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Tue, 7 Jul 2026 22:17:07 +0530 Subject: [PATCH 6/9] Document deliberate vcredist/msvcp140.dll inclusion in odbc wheel Corrects the earlier 'parity' framing. Main does not strip the VC++ runtime: build.bat relocates msvcp140.dll next to the .pyd and setup.py excludes the duplicate vcredist/ folder (de-duplication). The odbc package has no .pyd, so it keeps msvcp140.dll in vcredist/ with its license. No Phase-2 skew: the driver's runtime dependency is satisfied in-process by the /MD-built extension in both paths. --- setup_odbc.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/setup_odbc.py b/setup_odbc.py index 3e756df5..ba75f794 100644 --- a/setup_odbc.py +++ b/setup_odbc.py @@ -212,6 +212,27 @@ def run(self): url="https://github.com/microsoft/mssql-python", license="MIT", packages=[PACKAGE_NAME], + # vcredist / VC++ runtime (deliberate divergence from the main wheel's SHAPE, + # not a behavior change): unlike setup.py we do NOT exclude + # ``libs//vcredist/``. This is intentional and consistent with what the + # main wheel actually ships: + # * The main mssql-python wheel still ships the VC++ runtime + # (``msvcp140.dll``) -- build.bat copies it out of + # ``libs/windows//vcredist/`` to the package root next to the + # compiled ``ddbc_bindings`` extension, and setup.py then excludes the + # now-duplicate ``vcredist/`` folder. That exclusion is DE-DUPLICATION, + # not a licensing/size-driven removal of the runtime. + # * This package is pure driver-binary data with no compiled extension, so + # there is no package-root ``.pyd`` to relocate the runtime beside. We + # therefore keep ``msvcp140.dll`` in its original ``vcredist/`` folder + # (with its license text) so the driver binaries and the runtime they + # were built against travel together and the wheel stays self-contained. + # This does not create a Phase-2 runtime skew: the driver's ``msvcp140.dll`` + # dependency is satisfied in-process by the mssql-python extension module + # (built ``/MD``, which loads ``msvcp140.dll`` from beside the ``.pyd``), so + # the external-package and bundled-libs paths behave identically. This copy + # is completeness + attribution, which is also why GetOdbcLibsBaseDir()'s + # completeness check does not need to verify vcredist. package_data={ PACKAGE_NAME: [ "libs/*", From 40961b72014018dd4ba93c20e7e797dc52a3431f Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Wed, 8 Jul 2026 10:35:18 +0530 Subject: [PATCH 7/9] DOCS: Add CHANGELOG entry for the mssql-python-odbc package split --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 61b6e571..e6720691 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), connection. The callback is invoked by mssql-tds mid-handshake (FedAuth workflow 0x02) so the tenant id can be resolved from the server-supplied STS URL. Requires `mssql-py-core` 0.1.5+. Partial fix for #534. +- **Standalone `mssql-python-odbc` package (PRs #663, #664):** the Microsoft + ODBC Driver 18 for SQL Server binaries are now also published as a separate, + platform-specific `mssql-python-odbc` package. When it is installed, the + native driver loader resolves the driver from it; when it is absent or + incomplete, mssql-python transparently falls back to its own bundled `libs/`. + This is a non-breaking step toward decoupling driver-binary updates from + mssql-python releases; a future major version will make the dependency + explicit and drop the bundled binaries. ### Changed - Improved error handling in the connection module. From 74b6d62053b989294f817f587d3a95a069ba5ca3 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Wed, 8 Jul 2026 14:27:51 +0530 Subject: [PATCH 8/9] Fix inaccurate setup_odbc.py docstring on libs packaging The comment above MIN_SETUPTOOLS claimed the libs/ tree is gitignored and that include_package_data cannot recover the binaries from version control. Both are inaccurate: the driver binaries are committed, and package_data globs enumerate files from the on-disk mssql_python_odbc/libs/ subtree, not from git. Replace with an accurate description; the real rationale for the guard (setuptools 62.3.0 '**' recursive-glob support) is retained. --- setup_odbc.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/setup_odbc.py b/setup_odbc.py index ba75f794..4f5d2424 100644 --- a/setup_odbc.py +++ b/setup_odbc.py @@ -34,12 +34,12 @@ BUNDLED_LIBS_ROOT = PROJECT_ROOT / "mssql_python" / "libs" # The driver binaries are packaged via the recursive ``libs/**/*`` glob in -# ``package_data`` below. Support for ``**`` recursive globs in ``package_data`` -# landed in setuptools 62.3.0; with an older setuptools the glob silently -# matches nothing and the wheel ships WITHOUT any driver binaries. The ``libs/`` -# tree is gitignored, so ``include_package_data`` cannot recover the files from -# version control either -- a stale setuptools would produce a broken-but-quiet -# wheel. Fail fast instead. +# ``package_data`` below, which enumerates files from the copied +# ``mssql_python_odbc/libs/`` subtree on disk (not from version control). +# Support for ``**`` recursive globs in ``package_data`` landed in setuptools +# 62.3.0; with an older setuptools the glob silently matches nothing and the +# wheel ships WITHOUT any driver binaries. Fail fast instead of producing a +# broken-but-quiet wheel. MIN_SETUPTOOLS = (62, 3, 0) From 088462002107492cf18c01ed3c7588d55abad684 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Fri, 10 Jul 2026 15:30:26 +0530 Subject: [PATCH 9/9] odbc: bump mssql-python-odbc package version 18.6.0 -> 18.6.2 --- mssql_python_odbc/__init__.py | 2 +- setup_odbc.py | 2 +- tests/test_024_odbc_package_split.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mssql_python_odbc/__init__.py b/mssql_python_odbc/__init__.py index 90219724..f51c2333 100644 --- a/mssql_python_odbc/__init__.py +++ b/mssql_python_odbc/__init__.py @@ -24,7 +24,7 @@ __all__ = ["get_driver_path", "get_libs_dir", "__version__"] # Version tracks the bundled Microsoft ODBC Driver 18 for SQL Server release. -__version__ = "18.6.0" +__version__ = "18.6.2" # Driver shared-library file names per platform (must match the names produced # by the ODBC driver packaging and expected by the native loader). diff --git a/setup_odbc.py b/setup_odbc.py index 4f5d2424..0d83846d 100644 --- a/setup_odbc.py +++ b/setup_odbc.py @@ -197,7 +197,7 @@ def run(self): setup( name="mssql-python-odbc", - version="18.6.0", + version="18.6.2", description=( "Internal implementation package for mssql-python: Microsoft ODBC " "Driver 18 for SQL Server binaries. Not intended for direct use." diff --git a/tests/test_024_odbc_package_split.py b/tests/test_024_odbc_package_split.py index f937e225..8212c2c4 100644 --- a/tests/test_024_odbc_package_split.py +++ b/tests/test_024_odbc_package_split.py @@ -69,7 +69,7 @@ def _libs_present(): class TestOdbcPackageMetadata: def test_version_is_driver_version(self): - assert mssql_python_odbc.__version__ == "18.6.0" + assert mssql_python_odbc.__version__ == "18.6.2" def test_public_api_present(self): assert callable(mssql_python_odbc.get_driver_path)