CHORE: Add standalone mssql-python-odbc package for ODBC driver binaries

.gitignore

@@ -23,6 +23,13 @@ build/
 **/build/
 mssql_python.egg-info/
 
+# ODBC package (mssql-python-odbc): transition-period build artifacts.
+# During Phase 2, setup_odbc.py copies the current platform's driver binaries
+# from mssql_python/libs into this tree so a wheel can be built locally. The
+# release pipeline populates these per-platform; do not commit the copies.
+mssql_python_odbc/libs/
+mssql_python_odbc.egg-info/
+
 # Python bytecode
 __pycache__/
 *.py[cod]

CHANGELOG.md

@@ -14,6 +14,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
   connection. The callback is invoked by mssql-tds mid-handshake (FedAuth
   workflow 0x02) so the tenant id can be resolved from the server-supplied
   STS URL. Requires `mssql-py-core` 0.1.5+. Partial fix for #534.
+- **Standalone `mssql-python-odbc` package (PRs #663, #664):** the Microsoft
+  ODBC Driver 18 for SQL Server binaries are now also published as a separate,
+  platform-specific `mssql-python-odbc` package. When it is installed, the
+  native driver loader resolves the driver from it; when it is absent or
+  incomplete, mssql-python transparently falls back to its own bundled `libs/`.
+  This is a non-breaking step toward decoupling driver-binary updates from
+  mssql-python releases; a future major version will make the dependency
+  explicit and drop the bundled binaries.
 
 ### Changed
 - Improved error handling in the connection module.

mssql_python/pybind/ddbc_bindings.cpp

@@ -1125,6 +1125,90 @@ std::string GetModuleDirectory() {
     return parentDir.string();
 }
 
+// Resolve the base directory that contains the ODBC driver `libs/` tree.
+//
+// Post-split, the driver binaries ship in the standalone `mssql_python_odbc`
+// package (a pure-data sibling with no native extension). We import it and use
+// its directory as the base that `GetDriverPathCpp` (and the Windows
+// `mssql-auth.dll` lookup) append `libs` to.
+//
+// During the Phase-2 transition we fall back to the bundled `mssql_python`
+// directory when the external package is not installed, so a wheel that still
+// bundles `libs/` keeps working. Importing `mssql_python_odbc` here is
+// Alpine/musl-safe precisely because it is a separate pure package: it cannot
+// trigger the partially-initialized-module circular import that motivated
+// resolving these paths in C++ in the first place.
+//
+// (`GetDriverPathCpp` is defined further below; forward-declared here so we can
+// verify the external package actually ships this platform's driver binary.)
+std::string GetDriverPathCpp(const std::string& moduleDir);
+
+std::string GetOdbcLibsBaseDir() {
+    namespace fs = std::filesystem;
+    // This function calls into the Python C-API (py::module::import, attribute
+    // access, casts), so it must run with the GIL held. It is a no-op when the
+    // GIL is already held — which it is at the sole current call site, during
+    // module initialization — but acquiring it here self-documents the C-API
+    // dependency and keeps a future GIL-released caller from turning this into a
+    // hard crash.
+    py::gil_scoped_acquire gil;
+    try {
+        py::object module = py::module::import("mssql_python_odbc");
+        py::object module_path = module.attr("__file__");
+        std::string module_file = module_path.cast<std::string>();
+
+        fs::path parentDir = fs::path(module_file).parent_path();
+
+        // Only treat the external package as authoritative if it actually ships
+        // a COMPLETE set of this platform's driver binaries. In a source/dev
+        // checkout (and in CI) the package is importable from the repo root but
+        // its `libs/` tree is gitignored and either absent or only partially
+        // populated; in that case fall back to the bundled `mssql_python` libs
+        // so driver resolution points at a real, complete installation.
+        //
+        // "Complete" means the ODBC driver itself and, on Windows, the
+        // co-located `mssql-auth.dll` that LoadDriverOrThrowException loads
+        // unconditionally. Verifying both here keeps this resolver's notion of a
+        // usable base dir consistent with what the loader below actually needs,
+        // so we never select a directory that would later make the loader throw
+        // (e.g. a dir that has msodbcsql18.dll but is missing mssql-auth.dll).
+        std::error_code ec;
+        fs::path externalDriver(GetDriverPathCpp(parentDir.string()));
+        bool externalComplete = fs::exists(externalDriver, ec);
+#ifdef _WIN32
+        if (externalComplete) {
+            fs::path externalAuthDll = externalDriver.parent_path() / "mssql-auth.dll";
+            externalComplete = fs::exists(externalAuthDll, ec);
+        }
+#endif
+        if (externalComplete) {
+            LOG("GetOdbcLibsBaseDir: Using external mssql_python_odbc package - directory='%s'",
+                parentDir.string().c_str());
+            return parentDir.string();
+        }
+        LOG("GetOdbcLibsBaseDir: mssql_python_odbc present at '%s' but its libs are missing or "
+            "incomplete; falling back to bundled libs in mssql_python",
+            parentDir.string().c_str());
+        return GetModuleDirectory();
+    } catch (const py::error_already_set& e) {
+        if (e.matches(PyExc_ModuleNotFoundError)) {
+            // Expected in Phase 2 when the standalone package is not installed.
+            // pybind11 has already fetched and cleared the CPython error
+            // indicator, so re-importing `mssql_python` below is safe.
+            LOG("GetOdbcLibsBaseDir: mssql_python_odbc not installed (%s); "
+                "falling back to bundled libs in mssql_python",
+                e.what());
+            return GetModuleDirectory();
+        }
+        // A different import-time error means the package is installed but
+        // broken; surface it instead of silently masking the real problem.
+        LOG("GetOdbcLibsBaseDir: importing mssql_python_odbc failed unexpectedly (%s); "
+            "re-raising",
+            e.what());
+        throw;
+    }
+}
+
 // Platform-agnostic function to load the driver dynamic library
 DriverHandle LoadDriverLibrary(const std::string& driverPath) {
     LOG("LoadDriverLibrary: Attempting to load ODBC driver from path='%s'", driverPath.c_str());
@@ -1215,6 +1299,16 @@ std::string GetDriverPathCpp(const std::string& moduleDir) {
         platform = "debian_ubuntu";  // Default to debian_ubuntu for other distros
     }
 
+    // NOTE (version coupling): the Linux driver filename is pinned to the exact
+    // msodbcsql minor version (18.6 -> libmsodbcsql-18.6.so.2.1), matching the
+    // binaries bundled under mssql_python/libs and shipped by the standalone
+    // mssql_python_odbc package. If those driver binaries are upgraded (e.g. to
+    // 18.7) this literal MUST be updated in lockstep: GetOdbcLibsBaseDir() calls
+    // fs::exists() on this exact path to decide whether the external package is
+    // "complete", so a stale name would make it silently fall back to the
+    // bundled libs. (The macOS/Windows paths below key off the major version
+    // only and are more forgiving.) Tracked for the Phase 3 driver-version-
+    // agnostic resolution work.
     fs::path driverPath =
         basePath / "libs" / "linux" / platform / arch / "lib" / "libmsodbcsql-18.6.so.2.1";
     return driverPath.string();
@@ -1240,8 +1334,12 @@ std::string GetDriverPathCpp(const std::string& moduleDir) {
 DriverHandle LoadDriverOrThrowException() {
     namespace fs = std::filesystem;
 
-    std::string moduleDir = GetModuleDirectory();
-    LOG("LoadDriverOrThrowException: Module directory resolved to '%s'", moduleDir.c_str());
+    // Resolve the base dir from the standalone `mssql_python_odbc` package
+    // (falls back to the bundled `mssql_python` libs during the transition).
+    // Both the driver path and the Windows `mssql-auth.dll` path below are
+    // derived from this directory.
+    std::string moduleDir = GetOdbcLibsBaseDir();
+    LOG("LoadDriverOrThrowException: ODBC libs base directory resolved to '%s'", moduleDir.c_str());
 
     std::string archStr = ARCHITECTURE;
     LOG("LoadDriverOrThrowException: Architecture detected as '%s'", archStr.c_str());

mssql_python_odbc/__init__.py

@@ -0,0 +1,124 @@
+"""
+Copyright (c) Microsoft Corporation.
+Licensed under the MIT license.
+
+mssql_python_odbc — Microsoft ODBC Driver 18 for SQL Server binaries.
+
+Internal implementation package for ``mssql-python``. It ships the
+platform-specific ODBC driver binaries (``msodbcsql18``) and their supporting
+libraries so that ``mssql-python`` does not have to bundle them in its own
+wheel. It is not meant for direct consumption — install ``mssql-python``
+instead, which depends on this package.
+
+The public surface is :func:`get_driver_path`, which returns the absolute path
+to the ODBC driver shared library for the current platform/architecture. The
+native ``mssql_python.ddbc_bindings`` extension resolves the same location in
+C++ (see ``GetOdbcLibsBaseDir`` / ``GetDriverPathCpp``); this Python API mirrors
+that logic for tooling, tests, and diagnostics.
+"""
+
+import os
+import platform
+import sys
+
+__all__ = ["get_driver_path", "get_libs_dir", "__version__"]
+
+# Version tracks the bundled Microsoft ODBC Driver 18 for SQL Server release.
+__version__ = "18.6.2"
+
+# Driver shared-library file names per platform (must match the names produced
+# by the ODBC driver packaging and expected by the native loader).
+_DRIVER_FILENAME = {
+    "windows": "msodbcsql18.dll",
+    "linux": "libmsodbcsql-18.6.so.2.1",
+    "macos": "libmsodbcsql.18.dylib",
+}
+
+
+def get_libs_dir() -> str:
+    """Return the absolute path to this package's ``libs/`` directory.
+
+    This is the root under which the platform-specific ODBC binaries live
+    (``libs/<platform>/<arch>/...``). The parent of this path (the package
+    directory) is the base the native loader appends ``libs`` to when resolving
+    the driver.
+    """
+    return os.path.join(os.path.dirname(os.path.abspath(__file__)), "libs")
+
+
+def _detect_arch() -> str:
+    """Return the architecture directory name for the current interpreter.
+
+    Mirrors the compile-time detection in ``GetDriverPathCpp``:
+      * Windows uses ``x64`` / ``x86`` / ``arm64``.
+      * Linux and macOS use ``x86_64`` / ``arm64``.
+    """
+    machine = platform.machine().lower()
+
+    if sys.platform.startswith("win"):
+        if machine in ("amd64", "x86_64"):
+            return "x64"
+        if machine in ("arm64", "aarch64"):
+            return "arm64"
+        if machine in ("x86", "i386", "i686"):
+            return "x86"
+        raise OSError(f"Unsupported Windows architecture: {platform.machine()!r}")
+
+    # Linux / macOS
+    if machine in ("x86_64", "amd64"):
+        return "x86_64"
+    if machine in ("arm64", "aarch64"):
+        return "arm64"
+    raise OSError(f"Unsupported architecture: {platform.machine()!r}")
+
+
+def _detect_linux_distro_family() -> str:
+    """Return the Linux distro family directory name.
+
+    Mirrors the ``/etc/*-release`` probing in ``GetDriverPathCpp`` so the Python
+    and C++ paths agree.
+    """
+    if os.path.exists("/etc/alpine-release"):
+        return "alpine"
+    if os.path.exists("/etc/redhat-release") or os.path.exists("/etc/centos-release"):
+        return "rhel"
+    if os.path.exists("/etc/SuSE-release") or os.path.exists("/etc/SUSE-brand"):
+        return "suse"
+    return "debian_ubuntu"  # default for Debian/Ubuntu and other glibc distros
+
+
+def get_driver_path() -> str:
+    """Return the absolute path to the ODBC driver shared library.
+
+    Resolves the platform/architecture (and, on Linux, the distro family) and
+    returns the full path to ``msodbcsql18`` inside this package.
+
+    Raises:
+        OSError: if the platform/architecture is unsupported.
+        FileNotFoundError: if the resolved driver file is not present (e.g. the
+            package was built without this platform's binaries).
+    """
+    libs_dir = get_libs_dir()
+    arch = _detect_arch()
+
+    if sys.platform.startswith("win"):
+        driver_path = os.path.join(libs_dir, "windows", arch, _DRIVER_FILENAME["windows"])
+    elif sys.platform.startswith("darwin"):
+        driver_path = os.path.join(
+            libs_dir, "macos", arch, "lib", _DRIVER_FILENAME["macos"]
+        )
+    elif sys.platform.startswith("linux"):
+        distro = _detect_linux_distro_family()
+        driver_path = os.path.join(
+            libs_dir, "linux", distro, arch, "lib", _DRIVER_FILENAME["linux"]
+        )
+    else:
+        raise OSError(f"Unsupported platform: {sys.platform!r}")
+
+    if not os.path.isfile(driver_path):
+        raise FileNotFoundError(
+            f"ODBC driver not found at {driver_path!r}. The mssql-python-odbc "
+            f"package may not include binaries for this platform/architecture."
+        )
+
+    return driver_path