diff --git a/OneBranchPipelines/build-release-package-pipeline.yml b/OneBranchPipelines/build-release-package-pipeline.yml index 3bd789e85..c485cc8ee 100644 --- a/OneBranchPipelines/build-release-package-pipeline.yml +++ b/OneBranchPipelines/build-release-package-pipeline.yml @@ -69,6 +69,23 @@ parameters: displayName: 'Run SDL Security Tasks' type: boolean default: true + + # Which package to build. Both are produced from this same pipeline but are + # fully independent stage groups, so building only one halves the run time and + # lets you run two builds (one per package) concurrently. + # mssql-python -> Windows/macOS/Linux wheels + Consolidate + # mssql-python-odbc -> ODBC data wheels + ConsolidateOdbc + # both -> everything (previous default behaviour) + # Scheduled (daily) builds ignore this and always build 'both' (see + # effectiveBuildPackage below) so the nightly keeps producing every artifact. + - name: buildPackage + displayName: 'Package to Build' + type: string + values: + - 'mssql-python' + - 'mssql-python-odbc' + - 'both' + default: 'mssql-python' # ========================= # PLATFORM CONFIGURATIONS @@ -138,6 +155,33 @@ parameters: - { tag: 'musllinux', arch: 'x86_64', platform: 'linux/amd64' } - { tag: 'musllinux', arch: 'aarch64', platform: 'linux/arm64' } + # ========================= + # ODBC PACKAGE CONFIGURATIONS (mssql-python-odbc) + # ========================= + # The standalone mssql-python-odbc package ships ONLY pre-built ODBC driver + # binaries (data) — there is NO compiled extension — so ONE py3-none- + # wheel per platform serves every supported Python version. Hence there is no + # per-Python matrix here: 7 wheels total (2 Windows + 1 macOS + 4 Linux). + # These stages build alongside the mssql-python wheels and are consolidated + # SEPARATELY (see the ConsolidateOdbc stage) so the release pipeline can publish + # mssql-python and mssql-python-odbc independently. + - name: odbcWindowsConfigs + type: object + default: + - { arch: 'x64' } + - { arch: 'arm64' } + - name: odbcMacosConfigs + type: object + default: + - { name: 'universal2' } + - name: odbcLinuxConfigs + type: object + default: + - { tag: 'manylinux_2_28', arch: 'x86_64', platform: 'linux/amd64' } + - { tag: 'manylinux_2_28', arch: 'aarch64', platform: 'linux/arm64' } + - { tag: 'musllinux', arch: 'x86_64', platform: 'linux/amd64' } + - { tag: 'musllinux', arch: 'aarch64', platform: 'linux/arm64' } + # ========================= # PIPELINE VARIABLES # ========================= @@ -149,6 +193,14 @@ variables: value: 'Official' ${{ else }}: value: '${{ parameters.oneBranchType }}' + + # Determine which package(s) to build: scheduled (nightly) builds always build + # 'both' so every artifact keeps refreshing; manual/PR builds honour the parameter. + - name: effectiveBuildPackage + ${{ if eq(variables['Build.Reason'], 'Schedule') }}: + value: 'both' + ${{ else }}: + value: '${{ parameters.buildPackage }}' # Variable template imports # Each file provides specific variable groups: @@ -338,8 +390,10 @@ extends: # ========================= # PIPELINE STAGES # ========================= - # Total stages: 9 Windows + 5 macOS + 4 Linux + 1 Consolidate = 19 stages - # Stages run in parallel (no dependencies between platform builds) + # mssql-python: 9 Windows + 5 macOS + 4 Linux + 1 Consolidate = 19 stages + # mssql-python-odbc: 2 Windows + 1 macOS + 4 Linux + 1 ConsolidateOdbc = 8 stages + # Total: 27 stages. All build stages run in parallel; each Consolidate* stage + # waits only on its own package's build stages. stages: # ========================= # WINDOWS BUILD STAGES @@ -355,16 +409,18 @@ extends: # 5. Builds wheel # 6. Publishes artifacts (wheels + PYD + PDB) # 7. ESRP malware scanning - - ${{ each config in parameters.windowsConfigs }}: - - template: /OneBranchPipelines/stages/build-windows-single-stage.yml@self - parameters: - stageName: Win_py${{ config.pyVer }}_${{ config.arch }} - jobName: BuildWheel - # Convert pyVer '310' → pythonVersion '3.10' - pythonVersion: ${{ format('{0}.{1}', substring(config.pyVer, 0, 1), substring(config.pyVer, 1, 2)) }} - shortPyVer: ${{ config.pyVer }} - architecture: ${{ config.arch }} - oneBranchType: '${{ variables.effectiveOneBranchType }}' + # Gate: build mssql-python stages unless an odbc-only build was requested + - ${{ if ne(variables.effectiveBuildPackage, 'mssql-python-odbc') }}: + - ${{ each config in parameters.windowsConfigs }}: + - template: /OneBranchPipelines/stages/build-windows-single-stage.yml@self + parameters: + stageName: Win_py${{ config.pyVer }}_${{ config.arch }} + jobName: BuildWheel + # Convert pyVer '310' → pythonVersion '3.10' + pythonVersion: ${{ format('{0}.{1}', substring(config.pyVer, 0, 1), substring(config.pyVer, 1, 2)) }} + shortPyVer: ${{ config.pyVer }} + architecture: ${{ config.arch }} + oneBranchType: '${{ variables.effectiveOneBranchType }}' # ========================= # MACOS BUILD STAGES @@ -381,15 +437,16 @@ extends: # 6. Builds wheel # 7. Publishes artifacts (wheels + .so) # 8. ESRP malware scanning - - ${{ each config in parameters.macosConfigs }}: - - template: /OneBranchPipelines/stages/build-macos-single-stage.yml@self - parameters: - stageName: MacOS_py${{ config.pyVer }} - jobName: BuildWheel - # Convert pyVer '310' → pythonVersion '3.10' - pythonVersion: ${{ format('{0}.{1}', substring(config.pyVer, 0, 1), substring(config.pyVer, 1, 2)) }} - shortPyVer: ${{ config.pyVer }} - oneBranchType: '${{ variables.effectiveOneBranchType }}' + - ${{ if ne(variables.effectiveBuildPackage, 'mssql-python-odbc') }}: + - ${{ each config in parameters.macosConfigs }}: + - template: /OneBranchPipelines/stages/build-macos-single-stage.yml@self + parameters: + stageName: MacOS_py${{ config.pyVer }} + jobName: BuildWheel + # Convert pyVer '310' → pythonVersion '3.10' + pythonVersion: ${{ format('{0}.{1}', substring(config.pyVer, 0, 1), substring(config.pyVer, 1, 2)) }} + shortPyVer: ${{ config.pyVer }} + oneBranchType: '${{ variables.effectiveOneBranchType }}' # ========================= # LINUX BUILD STAGES @@ -411,15 +468,16 @@ extends: # d. Runs pytest against SQL Server # 4. Publishes artifacts (all 5 wheels) # 5. Component Governance + AntiMalware scanning - - ${{ each config in parameters.linuxConfigs }}: - - template: /OneBranchPipelines/stages/build-linux-single-stage.yml@self - parameters: - stageName: Linux_${{ config.tag }}_${{ config.arch }} - jobName: BuildWheels - linuxTag: ${{ config.tag }} - arch: ${{ config.arch }} - dockerPlatform: ${{ config.platform }} - oneBranchType: '${{ variables.effectiveOneBranchType }}' + - ${{ if ne(variables.effectiveBuildPackage, 'mssql-python-odbc') }}: + - ${{ each config in parameters.linuxConfigs }}: + - template: /OneBranchPipelines/stages/build-linux-single-stage.yml@self + parameters: + stageName: Linux_${{ config.tag }}_${{ config.arch }} + jobName: BuildWheels + linuxTag: ${{ config.tag }} + arch: ${{ config.arch }} + dockerPlatform: ${{ config.platform }} + oneBranchType: '${{ variables.effectiveOneBranchType }}' # ========================= # CONSOLIDATE STAGE @@ -433,36 +491,103 @@ extends: # - dist/bindings/macOS/*.so (macOS universal2 binaries) # - dist/bindings/Linux/*.so (Linux native extensions) # This stage also runs final BinSkim scan on all binaries - - stage: Consolidate - displayName: 'Consolidate All Artifacts' - dependsOn: - # Windows dependencies (9 stages) - - Win_py310_x64 - - Win_py311_x64 - - Win_py312_x64 - - Win_py313_x64 - - Win_py314_x64 - - Win_py311_arm64 - - Win_py312_arm64 - - Win_py313_arm64 - - Win_py314_arm64 - # macOS dependencies (5 stages) - - MacOS_py310 - - MacOS_py311 - - MacOS_py312 - - MacOS_py313 - - MacOS_py314 - # Linux dependencies (4 stages) - - Linux_manylinux_2_28_x86_64 - - Linux_manylinux_2_28_aarch64 - - Linux_musllinux_x86_64 - - Linux_musllinux_aarch64 - jobs: - - template: /OneBranchPipelines/jobs/consolidate-artifacts-job.yml@self - parameters: - # CRITICAL: Use effectiveOneBranchType to ensure scheduled builds run as 'Official' - # Using parameters.oneBranchType would break scheduled builds (they'd run as 'NonOfficial') - oneBranchType: '${{ variables.effectiveOneBranchType }}' + - ${{ if ne(variables.effectiveBuildPackage, 'mssql-python-odbc') }}: + - stage: Consolidate + displayName: 'Consolidate All Artifacts' + dependsOn: + # Windows dependencies (9 stages) + - Win_py310_x64 + - Win_py311_x64 + - Win_py312_x64 + - Win_py313_x64 + - Win_py314_x64 + - Win_py311_arm64 + - Win_py312_arm64 + - Win_py313_arm64 + - Win_py314_arm64 + # macOS dependencies (5 stages) + - MacOS_py310 + - MacOS_py311 + - MacOS_py312 + - MacOS_py313 + - MacOS_py314 + # Linux dependencies (4 stages) + - Linux_manylinux_2_28_x86_64 + - Linux_manylinux_2_28_aarch64 + - Linux_musllinux_x86_64 + - Linux_musllinux_aarch64 + jobs: + - template: /OneBranchPipelines/jobs/consolidate-artifacts-job.yml@self + parameters: + # CRITICAL: Use effectiveOneBranchType to ensure scheduled builds run as 'Official' + # Using parameters.oneBranchType would break scheduled builds (they'd run as 'NonOfficial') + oneBranchType: '${{ variables.effectiveOneBranchType }}' # Note: Symbol publishing handled directly in Windows build stages # PDB files uploaded to Microsoft Symbol Server for debugging + + # ========================================================================= + # ODBC PACKAGE BUILD STAGES (mssql-python-odbc) + # ========================================================================= + # 7 data-only wheels (no native compile, no pytest) built from setup_odbc.py. + # They run in parallel with the mssql-python stages and are consolidated by a + # SEPARATE ConsolidateOdbc stage so the release pipeline can ship each package + # independently. SDL "just works": these wheels contain the SAME ODBC driver + # binaries already scanned in mssql_python/libs by the main build — no new + # first-party native code is introduced. + # + # ODBC Windows stages (2): ODBC_Win_x64, ODBC_Win_arm64 + # Gate: build mssql-python-odbc stages unless an mssql-python-only build was requested + - ${{ if ne(variables.effectiveBuildPackage, 'mssql-python') }}: + - ${{ each config in parameters.odbcWindowsConfigs }}: + - template: /OneBranchPipelines/stages/build-odbc-windows-stage.yml@self + parameters: + stageName: ODBC_Win_${{ config.arch }} + jobName: BuildWheel + architecture: ${{ config.arch }} + oneBranchType: '${{ variables.effectiveOneBranchType }}' + + # ODBC macOS stage (1): ODBC_MacOS_universal2 + - ${{ if ne(variables.effectiveBuildPackage, 'mssql-python') }}: + - ${{ each config in parameters.odbcMacosConfigs }}: + - template: /OneBranchPipelines/stages/build-odbc-macos-stage.yml@self + parameters: + stageName: ODBC_MacOS_${{ config.name }} + jobName: BuildWheel + oneBranchType: '${{ variables.effectiveOneBranchType }}' + + # ODBC Linux stages (4): manylinux_2_28 x86_64/aarch64, musllinux x86_64/aarch64 + - ${{ if ne(variables.effectiveBuildPackage, 'mssql-python') }}: + - ${{ each config in parameters.odbcLinuxConfigs }}: + - template: /OneBranchPipelines/stages/build-odbc-linux-stage.yml@self + parameters: + stageName: ODBC_Linux_${{ config.tag }}_${{ config.arch }} + jobName: BuildWheel + linuxTag: ${{ config.tag }} + arch: ${{ config.arch }} + dockerPlatform: ${{ config.platform }} + oneBranchType: '${{ variables.effectiveOneBranchType }}' + + # ========================================================================= + # CONSOLIDATE ODBC STAGE + # ========================================================================= + # Collects the 7 mssql-python-odbc wheels into a single dist/ folder and + # publishes them as `drop_ConsolidateOdbc_ConsolidateArtifacts` (distinct from + # the mssql-python `drop_Consolidate_ConsolidateArtifacts`). The release + # pipeline selects one artifact based on its `releasePackage` parameter. + - ${{ if ne(variables.effectiveBuildPackage, 'mssql-python') }}: + - stage: ConsolidateOdbc + displayName: 'Consolidate All ODBC Artifacts' + dependsOn: + - ODBC_Win_x64 + - ODBC_Win_arm64 + - ODBC_MacOS_universal2 + - ODBC_Linux_manylinux_2_28_x86_64 + - ODBC_Linux_manylinux_2_28_aarch64 + - ODBC_Linux_musllinux_x86_64 + - ODBC_Linux_musllinux_aarch64 + jobs: + - template: /OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml@self + parameters: + oneBranchType: '${{ variables.effectiveOneBranchType }}' + expectedWheelCount: 7 diff --git a/OneBranchPipelines/dummy-release-pipeline.yml b/OneBranchPipelines/dummy-release-pipeline.yml index f9e548fb0..6cbb7c44b 100644 --- a/OneBranchPipelines/dummy-release-pipeline.yml +++ b/OneBranchPipelines/dummy-release-pipeline.yml @@ -12,6 +12,16 @@ pr: none # Parameters for DUMMY release pipeline parameters: + # Which package to test-release. Both are produced by the same build pipeline + # (definition 2199); this switches the consolidated artifact and messaging. + - name: releasePackage + displayName: '[TEST] Package to Release' + type: string + values: + - 'mssql-python' + - 'mssql-python-odbc' + default: 'mssql-python' + - name: publishSymbols displayName: '[TEST] Publish Symbols to Symbol Servers' type: boolean @@ -32,6 +42,16 @@ variables: - group: 'ESRP Federated Creds (AME)' # Contains ESRP signing credentials - group: 'Symbols Publishing' # Contains SymbolServer, SymbolTokenUri variables + # Select which consolidated artifact to download based on the target package. + # Both are produced by the same build pipeline (definition 2199): + # mssql-python -> drop_Consolidate_ConsolidateArtifacts + # mssql-python-odbc -> drop_ConsolidateOdbc_ConsolidateArtifacts + - name: consolidatedArtifactName + ${{ if eq(parameters.releasePackage, 'mssql-python-odbc') }}: + value: 'drop_ConsolidateOdbc_ConsolidateArtifacts' + ${{ else }}: + value: 'drop_Consolidate_ConsolidateArtifacts' + # OneBranch resources resources: repositories: @@ -114,7 +134,7 @@ extends: definition: 2199 # Build-Release-Package-Pipeline definition ID buildVersionToDownload: 'specific' buildId: $(resources.pipeline.buildPipeline.runID) # Use the build run selected in UI - artifactName: 'drop_Consolidate_ConsolidateArtifacts' # Consolidated artifact with dist/ and symbols/ + artifactName: '${{ variables.consolidatedArtifactName }}' # mssql-python or mssql-python-odbc consolidated wheels targetPath: '$(Build.SourcesDirectory)/artifacts' # Step 2: List downloaded artifacts for verification @@ -175,13 +195,58 @@ extends: Write-Host "Symbols: $(if ($symbols) { $symbols.Count } else { 0 }) files" Write-Host "=====================================" - # Step 3: Validate mssql-py-core is a stable version (no dev/alpha/beta/rc) - - task: PowerShell@2 - displayName: '[TEST] Validate mssql-py-core is a stable version' - inputs: - targetType: 'filePath' - filePath: '$(Build.SourcesDirectory)\OneBranchPipelines\scripts\validate-release-versions.ps1' - arguments: '-VersionFile "$(Build.SourcesDirectory)\artifacts\dist\mssql-py-core.version"' + # Step 3: Validate mssql-py-core is a stable version (mssql-python only; + # mssql-python-odbc ships no mssql-py-core.version file) + - ${{ if eq(parameters.releasePackage, 'mssql-python') }}: + - task: PowerShell@2 + displayName: '[TEST] Validate mssql-py-core is a stable version' + inputs: + targetType: 'filePath' + filePath: '$(Build.SourcesDirectory)\OneBranchPipelines\scripts\validate-release-versions.ps1' + arguments: '-VersionFile "$(Build.SourcesDirectory)\artifacts\dist\mssql-py-core.version"' + + # Guard: the released mssql-python wheel MUST depend on the pinned + # mssql-python-odbc package. Catches releasing from a PRE-pin build + # (e.g. the wrong build run), which would ship a driver-less package + # that breaks 'pip install mssql-python'. By design this FAILS on a + # pre-pin build. Update $expectedVersion when the odbc pin bumps. + - task: PowerShell@2 + displayName: '[TEST] Validate mssql-python-odbc pin in wheel metadata' + inputs: + targetType: 'inline' + script: | + $ErrorActionPreference = 'Stop' + $expectedVersion = '18.6.2' + $wheels = Get-ChildItem -Path "$(Build.SourcesDirectory)/artifacts/dist" -Filter "mssql_python-*.whl" + if ($wheels.Count -eq 0) { + Write-Error "No mssql_python-*.whl found in artifacts/dist to validate the odbc pin." + exit 1 + } + Add-Type -AssemblyName System.IO.Compression.FileSystem + $wheel = $wheels[0].FullName + Write-Host "Inspecting wheel metadata: $wheel" + $zip = [System.IO.Compression.ZipFile]::OpenRead($wheel) + $metaEntry = $zip.Entries | Where-Object { $_.FullName -match '\.dist-info/METADATA$' } | Select-Object -First 1 + if (-not $metaEntry) { + $zip.Dispose() + Write-Error "METADATA not found inside $wheel" + exit 1 + } + $reader = New-Object System.IO.StreamReader($metaEntry.Open()) + $metadata = $reader.ReadToEnd() + $reader.Dispose() + $zip.Dispose() + $odbcLines = ($metadata -split "`n") | Where-Object { $_ -match 'Requires-Dist:\s*mssql[-_]python[-_]odbc' } + if ($odbcLines) { + Write-Host "odbc dependency in metadata:" + $odbcLines | ForEach-Object { Write-Host " $($_.Trim())" } + } + $pinRegex = 'Requires-Dist:\s*mssql[-_]python[-_]odbc\s*==\s*' + [regex]::Escape($expectedVersion) + if ($metadata -notmatch $pinRegex) { + Write-Error "mssql-python wheel does not pin mssql-python-odbc==$expectedVersion. Are you releasing from the POST-pin build? mssql-python must depend on the pinned odbc package before release." + exit 1 + } + Write-Host "OK: wheel depends on mssql-python-odbc==$expectedVersion" # Step 4: Verify wheel integrity - task: PowerShell@2 @@ -217,8 +282,8 @@ extends: Write-Host "`nAll wheels verified successfully!" - # Step 5: Publish Symbols (if enabled and symbols exist) - - ${{ if eq(parameters.publishSymbols, true) }}: + # Step 5: Publish Symbols (if enabled and symbols exist; mssql-python only) + - ${{ if and(eq(parameters.publishSymbols, true), eq(parameters.releasePackage, 'mssql-python')) }}: - template: /OneBranchPipelines/steps/symbol-publishing-step.yml@self parameters: SymbolsFolder: '$(Build.SourcesDirectory)/symbols' @@ -233,13 +298,13 @@ extends: Write-Host "====================================" Write-Host "⚠️ TEST PIPELINE - DRY RUN MODE ⚠️" Write-Host "====================================" - Write-Host "Package: mssql-python (TEST)" + Write-Host "Package: ${{ parameters.releasePackage }} (TEST)" Write-Host "" Write-Host "Actions performed:" Write-Host "✓ Downloaded wheels from build pipeline" Write-Host "✓ Verified wheel integrity" Write-Host "✓ Downloaded symbols from build pipeline" - if ("${{ parameters.publishSymbols }}" -eq "True") { + if ("${{ parameters.publishSymbols }}" -eq "True" -and "${{ parameters.releasePackage }}" -eq "mssql-python") { Write-Host "✓ Published symbols to SqlClientDrivers org" } Write-Host "✗ ESRP dummy release NOT performed (parameter disabled)" @@ -280,7 +345,7 @@ extends: definition: 2199 buildVersionToDownload: 'specific' buildId: $(resources.pipeline.buildPipeline.runID) - artifactName: 'drop_Consolidate_ConsolidateArtifacts' + artifactName: '${{ variables.consolidatedArtifactName }}' targetPath: '$(Build.SourcesDirectory)/artifacts' # ⚠️ IMPORTANT: Uses Maven ContentType for testing - NOT PyPI! @@ -313,11 +378,11 @@ extends: Write-Host "====================================" Write-Host "⚠️ TEST PIPELINE - DUMMY RELEASE COMPLETED ⚠️" Write-Host "====================================" - Write-Host "Package: mssql-python (TEST)" + Write-Host "Package: ${{ parameters.releasePackage }} (TEST)" Write-Host "ContentType: Maven (NOT PyPI - Safe for Testing)" Write-Host "Owners: $(owner)" Write-Host "Approvers: $(approver)" - Write-Host "Symbols Published: ${{ parameters.publishSymbols }}" + Write-Host "Symbols Published: $(if ('${{ parameters.publishSymbols }}' -eq 'True' -and '${{ parameters.releasePackage }}' -eq 'mssql-python') { 'True' } else { 'False' })" Write-Host "=====================================" Write-Host "" Write-Host "⚠️ IMPORTANT: This was a DUMMY release using Maven ContentType" @@ -326,7 +391,7 @@ extends: Write-Host "What was tested:" Write-Host "✓ Artifact download from build pipeline" Write-Host "✓ Wheel integrity verification" - if ("${{ parameters.publishSymbols }}" -eq "True") { + if ("${{ parameters.publishSymbols }}" -eq "True" -and "${{ parameters.releasePackage }}" -eq "mssql-python") { Write-Host "✓ Symbol publishing to SqlClientDrivers org" } Write-Host "✓ ESRP release workflow (Maven ContentType)" diff --git a/OneBranchPipelines/jobs/consolidate-artifacts-job.yml b/OneBranchPipelines/jobs/consolidate-artifacts-job.yml index 478803aa3..40cda12b0 100644 --- a/OneBranchPipelines/jobs/consolidate-artifacts-job.yml +++ b/OneBranchPipelines/jobs/consolidate-artifacts-job.yml @@ -1,7 +1,7 @@ # Consolidate Artifacts Job Template # Downloads artifacts from all platform build stages and consolidates into single dist/ folder -# Works with individual stage artifacts (15 stages total: 7 Windows + 4 macOS + 4 Linux) -# Each Linux stage builds 4 Python versions, resulting in 27 total wheels +# Works with individual stage artifacts (18 stages total: 9 Windows + 5 macOS + 4 Linux) +# Each Linux stage builds 5 Python versions (3.10-3.14), resulting in 34 total wheels parameters: - name: oneBranchType type: string @@ -29,13 +29,20 @@ jobs: - checkout: self fetchDepth: 1 - # Download ALL artifacts from current build - # Matrix jobs publish as: Windows_, macOS_, Linux_ - # This downloads all of them automatically (27 total artifacts) + # Download only the mssql-python platform-stage artifacts from the current build. + # The build definition now ALSO produces mssql-python-odbc artifacts (drop_ODBC_*) + # in the SAME run, so we scope this download with itemPattern to the mssql-python + # stages (drop_Win_*, drop_MacOS_*, drop_Linux_*). Without this, the odbc wheels + # would be swept into this package's dist/ and shipped to the mssql-python PyPI + # project. The odbc wheels are consolidated separately (ConsolidateOdbc stage). - task: DownloadPipelineArtifact@2 displayName: 'Download All Platform Artifacts' inputs: buildType: 'current' + itemPattern: | + drop_Win_*/** + drop_MacOS_*/** + drop_Linux_*/** targetPath: '$(Pipeline.Workspace)/all-artifacts' # Consolidate all wheels into single dist/ directory @@ -68,12 +75,17 @@ jobs: echo "" WHEEL_COUNT=$(ls -1 $(ob_outputDirectory)/dist/*.whl 2>/dev/null | wc -l) echo "Total wheel count: $WHEEL_COUNT" - echo "Expected: 27 wheels (7 Windows + 4 macOS + 16 Linux)" + echo "Expected: 34 wheels (9 Windows + 5 macOS + 20 Linux)" - if [ "$WHEEL_COUNT" -ne 27 ]; then - echo "WARNING: Expected 27 wheels but found $WHEEL_COUNT" + # Hard-fail on a wheel-count mismatch (symmetric with the odbc consolidate job). + # A mismatch means a build stage silently produced a partial set (e.g. a missing + # Python version) or the build matrix changed without updating this count. + # Update the expected 34 whenever the Windows/macOS/Linux build matrix changes. + if [ "$WHEEL_COUNT" -ne 34 ]; then + echo "ERROR: Expected 34 wheels but found $WHEEL_COUNT" >&2 + exit 1 else - echo "SUCCESS: All 27 wheels consolidated!" + echo "SUCCESS: All 34 wheels consolidated!" fi displayName: 'Consolidate wheels from all platforms' diff --git a/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml b/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml new file mode 100644 index 000000000..f0d8a0715 --- /dev/null +++ b/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml @@ -0,0 +1,75 @@ +# Consolidate ODBC Artifacts Job Template +# Downloads the per-platform `mssql-python-odbc` wheels from all build stages and +# consolidates them into a single dist/ folder for the release pipeline to publish. +# Expected: 7 wheels (2 Windows + 1 macOS universal2 + 4 Linux). +parameters: + - name: oneBranchType + type: string + default: 'Official' + - name: expectedWheelCount + type: number + default: 7 + +jobs: + - job: ConsolidateArtifacts + displayName: 'Consolidate All ODBC Platform Artifacts' + condition: succeeded() + + pool: + type: linux + isCustom: true + name: Azure Pipelines + vmImage: 'ubuntu-latest' + + variables: + # Consolidation only moves files; no binaries to scan. + - name: ob_sdl_binskim_enabled + value: false + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)' + + steps: + - checkout: self + fetchDepth: 1 + + # Download only the mssql-python-odbc platform-stage artifacts (drop_ODBC_*) from + # the current build. The same run also builds the mssql-python wheels (drop_Win_*, + # drop_MacOS_*, drop_Linux_*); itemPattern scopes this download to the odbc stages + # so ONLY the 7 odbc wheels are consolidated here (not the mssql-python wheels). + - task: DownloadPipelineArtifact@2 + displayName: 'Download All ODBC Platform Artifacts' + inputs: + buildType: 'current' + itemPattern: 'drop_ODBC_*/**' + targetPath: '$(Pipeline.Workspace)/all-artifacts' + + - bash: | + set -e + mkdir -p $(ob_outputDirectory)/dist + + echo "Finding all .whl files..." + find $(Pipeline.Workspace)/all-artifacts -name "*.whl" -exec ls -lh {} \; + + echo "Copying all wheels to consolidated dist/..." + find $(Pipeline.Workspace)/all-artifacts -name "*.whl" -exec cp -v {} $(ob_outputDirectory)/dist/ \; + + echo "Consolidated wheels:" + ls -lh $(ob_outputDirectory)/dist/ + WHEEL_COUNT=$(ls -1 $(ob_outputDirectory)/dist/*.whl 2>/dev/null | wc -l) + echo "Total wheel count: $WHEEL_COUNT (expected ${{ parameters.expectedWheelCount }})" + if [ "$WHEEL_COUNT" -ne "${{ parameters.expectedWheelCount }}" ]; then + echo "ERROR: expected ${{ parameters.expectedWheelCount }} wheels but found $WHEEL_COUNT" >&2 + exit 1 + fi + echo "SUCCESS: all ${{ parameters.expectedWheelCount }} ODBC wheels consolidated." + displayName: 'Consolidate ODBC wheels' + + - task: PublishPipelineArtifact@1 + displayName: 'Publish Consolidated ODBC Artifacts' + inputs: + targetPath: '$(ob_outputDirectory)' + # Distinct name so it does not collide with the mssql-python consolidate + # artifact (drop_Consolidate_ConsolidateArtifacts) in the same build run. + # Matches the OneBranch auto-name for a stage named `ConsolidateOdbc`. + artifact: 'drop_ConsolidateOdbc_ConsolidateArtifacts' + publishLocation: 'pipeline' diff --git a/OneBranchPipelines/official-release-pipeline.yml b/OneBranchPipelines/official-release-pipeline.yml index a5ec227f6..8937fd0f6 100644 --- a/OneBranchPipelines/official-release-pipeline.yml +++ b/OneBranchPipelines/official-release-pipeline.yml @@ -1,5 +1,7 @@ -# OneBranch Official Release Pipeline for mssql-python -# Downloads wheel and symbol artifacts from build pipeline, publishes symbols, and releases wheels to PyPI via ESRP +# OneBranch Official Release Pipeline (mssql-python and mssql-python-odbc) +# Selects the package via the `releasePackage` parameter, downloads that package's +# consolidated wheels (and symbols for mssql-python) from the build pipeline +# (definition 2199), publishes symbols, and releases wheels to PyPI via ESRP. # This pipeline is ALWAYS Official - no NonOfficial option name: $(Year:YY)$(DayOfYear)$(Rev:.r)-Release @@ -10,6 +12,16 @@ pr: none # Parameters for release pipeline parameters: + # Which package to release. Both are produced by the same build pipeline + # (definition 2199); this switches the consolidated artifact and messaging. + - name: releasePackage + displayName: 'Package to Release' + type: string + values: + - 'mssql-python' + - 'mssql-python-odbc' + default: 'mssql-python' + - name: publishSymbols displayName: 'Publish Symbols to Symbol Servers' type: boolean @@ -30,6 +42,16 @@ variables: - group: 'ESRP Federated Creds (AME)' # Contains ESRP signing credentials - group: 'Symbols Publishing' # Contains SymbolServer, SymbolTokenUri variables + # Select which consolidated artifact to download/publish based on the target + # package. Both are produced by the same build pipeline (definition 2199): + # mssql-python -> drop_Consolidate_ConsolidateArtifacts + # mssql-python-odbc -> drop_ConsolidateOdbc_ConsolidateArtifacts + - name: consolidatedArtifactName + ${{ if eq(parameters.releasePackage, 'mssql-python-odbc') }}: + value: 'drop_ConsolidateOdbc_ConsolidateArtifacts' + ${{ else }}: + value: 'drop_Consolidate_ConsolidateArtifacts' + # OneBranch resources resources: repositories: @@ -117,7 +139,7 @@ extends: definition: 2199 # Build-Release-Package-Pipeline definition ID buildVersionToDownload: 'specific' buildId: $(resources.pipeline.buildPipeline.runID) # Use the build run selected in UI - artifactName: 'drop_Consolidate_ConsolidateArtifacts' # Consolidated artifact with dist/ and symbols/ + artifactName: '${{ variables.consolidatedArtifactName }}' # mssql-python or mssql-python-odbc consolidated wheels targetPath: '$(Build.SourcesDirectory)/artifacts' # Step 2: List downloaded artifacts for verification @@ -173,13 +195,58 @@ extends: Write-Host "Symbols: $(if ($symbols) { $symbols.Count } else { 0 }) files" Write-Host "=====================================" - # Step 3: Validate mssql-py-core is a stable version (no dev/alpha/beta/rc) - - task: PowerShell@2 - displayName: 'Validate mssql-py-core is a stable version' - inputs: - targetType: 'filePath' - filePath: '$(Build.SourcesDirectory)\OneBranchPipelines\scripts\validate-release-versions.ps1' - arguments: '-VersionFile "$(Build.SourcesDirectory)\artifacts\dist\mssql-py-core.version"' + # Step 3: Validate mssql-py-core is a stable version (mssql-python only; + # mssql-python-odbc ships no mssql-py-core.version file) + - ${{ if eq(parameters.releasePackage, 'mssql-python') }}: + - task: PowerShell@2 + displayName: 'Validate mssql-py-core is a stable version' + inputs: + targetType: 'filePath' + filePath: '$(Build.SourcesDirectory)\OneBranchPipelines\scripts\validate-release-versions.ps1' + arguments: '-VersionFile "$(Build.SourcesDirectory)\artifacts\dist\mssql-py-core.version"' + + # Guard: the released mssql-python wheel MUST depend on the pinned + # mssql-python-odbc package. Catches releasing from a PRE-pin build + # (e.g. the wrong build run), which would ship a driver-less package + # that breaks 'pip install mssql-python'. By design this FAILS on a + # pre-pin build. Update $expectedVersion when the odbc pin bumps. + - task: PowerShell@2 + displayName: 'Validate mssql-python-odbc pin in wheel metadata' + inputs: + targetType: 'inline' + script: | + $ErrorActionPreference = 'Stop' + $expectedVersion = '18.6.2' + $wheels = Get-ChildItem -Path "$(Build.SourcesDirectory)/artifacts/dist" -Filter "mssql_python-*.whl" + if ($wheels.Count -eq 0) { + Write-Error "No mssql_python-*.whl found in artifacts/dist to validate the odbc pin." + exit 1 + } + Add-Type -AssemblyName System.IO.Compression.FileSystem + $wheel = $wheels[0].FullName + Write-Host "Inspecting wheel metadata: $wheel" + $zip = [System.IO.Compression.ZipFile]::OpenRead($wheel) + $metaEntry = $zip.Entries | Where-Object { $_.FullName -match '\.dist-info/METADATA$' } | Select-Object -First 1 + if (-not $metaEntry) { + $zip.Dispose() + Write-Error "METADATA not found inside $wheel" + exit 1 + } + $reader = New-Object System.IO.StreamReader($metaEntry.Open()) + $metadata = $reader.ReadToEnd() + $reader.Dispose() + $zip.Dispose() + $odbcLines = ($metadata -split "`n") | Where-Object { $_ -match 'Requires-Dist:\s*mssql[-_]python[-_]odbc' } + if ($odbcLines) { + Write-Host "odbc dependency in metadata:" + $odbcLines | ForEach-Object { Write-Host " $($_.Trim())" } + } + $pinRegex = 'Requires-Dist:\s*mssql[-_]python[-_]odbc\s*==\s*' + [regex]::Escape($expectedVersion) + if ($metadata -notmatch $pinRegex) { + Write-Error "mssql-python wheel does not pin mssql-python-odbc==$expectedVersion. Are you releasing from the POST-pin build? mssql-python must depend on the pinned odbc package before release." + exit 1 + } + Write-Host "OK: wheel depends on mssql-python-odbc==$expectedVersion" # Step 4: Verify wheel integrity - task: PowerShell@2 @@ -215,8 +282,8 @@ extends: Write-Host "`nAll wheels verified successfully!" - # Step 5: Publish Symbols (if enabled and symbols exist) - - ${{ if eq(parameters.publishSymbols, true) }}: + # Step 5: Publish Symbols (mssql-python only; mssql-python-odbc has no PDBs) + - ${{ if and(eq(parameters.publishSymbols, true), eq(parameters.releasePackage, 'mssql-python')) }}: - template: /OneBranchPipelines/steps/symbol-publishing-step.yml@self parameters: SymbolsFolder: '$(Build.SourcesDirectory)/symbols' @@ -231,13 +298,13 @@ extends: Write-Host "====================================" Write-Host "DRY RUN MODE - No Release Performed" Write-Host "====================================" - Write-Host "Package: mssql-python" + Write-Host "Package: ${{ parameters.releasePackage }}" Write-Host "" Write-Host "Actions performed:" Write-Host "- Downloaded wheels from build pipeline" Write-Host "- Verified wheel integrity" Write-Host "- Downloaded symbols from build pipeline" - if ("${{ parameters.publishSymbols }}" -eq "True") { + if ("${{ parameters.publishSymbols }}" -eq "True" -and "${{ parameters.releasePackage }}" -eq "mssql-python") { Write-Host "- Published symbols to SqlClientDrivers org" } Write-Host "" @@ -273,7 +340,7 @@ extends: definition: 2199 buildVersionToDownload: 'specific' buildId: $(resources.pipeline.buildPipeline.runID) - artifactName: 'drop_Consolidate_ConsolidateArtifacts' + artifactName: '${{ variables.consolidatedArtifactName }}' targetPath: '$(Build.SourcesDirectory)/artifacts' - task: EsrpRelease@9 @@ -303,16 +370,17 @@ extends: Write-Host "====================================" Write-Host "ESRP Release Completed" Write-Host "====================================" - Write-Host "Package: mssql-python" + Write-Host "Package: ${{ parameters.releasePackage }}" Write-Host "Target: PyPI" Write-Host "Owners: $(owner)" Write-Host "Approvers: $(approver)" - Write-Host "Symbols Published: ${{ parameters.publishSymbols }}" + $symbolsPublished = ("${{ parameters.publishSymbols }}" -eq "True") -and ("${{ parameters.releasePackage }}" -eq "mssql-python") + Write-Host "Symbols Published: $symbolsPublished" Write-Host "=====================================" Write-Host "" Write-Host "Next steps:" Write-Host "1. Verify release in ESRP portal" Write-Host "2. Wait for approval workflow completion" - Write-Host "3. Verify package on PyPI: https://pypi.org/project/mssql-python/" + Write-Host "3. Verify package on PyPI: https://pypi.org/project/${{ parameters.releasePackage }}/" Write-Host "4. Verify symbols in SqlClientDrivers org (if published)" Write-Host "=====================================" diff --git a/OneBranchPipelines/stages/build-odbc-linux-stage.yml b/OneBranchPipelines/stages/build-odbc-linux-stage.yml new file mode 100644 index 000000000..969c0b8b3 --- /dev/null +++ b/OneBranchPipelines/stages/build-odbc-linux-stage.yml @@ -0,0 +1,191 @@ +# ODBC Linux Single Configuration Stage Template +# Builds the platform-specific `mssql-python-odbc` wheel for ONE libc/arch combo +# (manylinux_2_28 or musllinux_1_2 x x86_64 or aarch64). +# +# Data-only package: no native compilation, no per-Python matrix, no pytest. +# A PyPA build container is still used so `platform.libc_ver()` reports the right +# libc and the wheel is tagged manylinux_2_28_* or musllinux_1_2_* correctly. +# aarch64 wheels are produced under QEMU emulation. Driver binaries come from the +# committed mssql_python/libs/ tree that is bind-mounted into the container. +parameters: + # Stage identifier (e.g., 'ODBC_Linux_manylinux_2_28_x86_64'). + - name: stageName + type: string + - name: jobName + type: string + default: 'BuildWheel' + # Distribution type: 'manylinux_2_28' (glibc 2.28+) or 'musllinux' (musl libc). + - name: linuxTag + type: string + # CPU architecture: 'x86_64' (AMD64) or 'aarch64' (ARM64). + - name: arch + type: string + # Docker platform for QEMU emulation: 'linux/amd64' or 'linux/arm64'. + - name: dockerPlatform + type: string + - name: oneBranchType + type: string + default: 'Official' + +stages: + - stage: ${{ parameters.stageName }} + displayName: 'ODBC Linux ${{ parameters.linuxTag }} ${{ parameters.arch }}' + jobs: + - job: ${{ parameters.jobName }} + displayName: 'Build ODBC Wheel - ${{ parameters.linuxTag }} ${{ parameters.arch }}' + # Reuse the same custom 1ES Linux pool as the mssql-python build. + pool: + type: linux + isCustom: true + name: Python-1ES-pool + demands: + - imageOverride -equals PYTHON-1ES-UB2404 + timeoutInMinutes: 60 + + variables: + # BinSkim needs ICU libs not present in manylinux/musllinux containers. + - name: ob_sdl_binskim_enabled + value: false + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)' + - name: LinuxContainerImage + value: 'onebranch.azurecr.io/linux/ubuntu-2204:latest' + - name: LINUX_TAG + value: ${{ parameters.linuxTag }} + - name: ARCH + value: ${{ parameters.arch }} + - name: DOCKER_PLATFORM + value: ${{ parameters.dockerPlatform }} + + steps: + - checkout: self + fetchDepth: 1 + + - bash: | + set -e + if ! docker info > /dev/null 2>&1; then + echo "Starting Docker daemon..." + sudo dockerd > docker.log 2>&1 & + sleep 10 + fi + docker --version + displayName: 'Setup and start Docker daemon' + + - script: | + sudo apt-get install -y qemu-user-static + displayName: 'Enable QEMU (for aarch64)' + + - script: | + rm -rf $(ob_outputDirectory)/wheels + mkdir -p $(ob_outputDirectory)/wheels + displayName: 'Prepare artifact directories' + + - task: AzureCLI@2 + displayName: 'Login to ACR (tdslibrs)' + inputs: + azureSubscription: 'Magnitude Test-mssql-rs-mssql-python' + scriptType: 'bash' + scriptLocation: 'inlineScript' + inlineScript: | + az acr login --name tdslibrs + + - script: | + set -euxo pipefail + # Same PyPA images as the mssql-python Linux build (mirrored into ACR). + if [[ "$(LINUX_TAG)" == "manylinux_2_28" ]]; then + IMAGE="tdslibrs.azurecr.io/import/python-build/manylinux_2_28_$(ARCH):latest" + elif [[ "$(LINUX_TAG)" == "musllinux" ]]; then + IMAGE="tdslibrs.azurecr.io/import/python-build/musllinux_1_2_$(ARCH):latest" + else + echo "ERROR: Unsupported LINUX_TAG '$(LINUX_TAG)'. Expected: manylinux_2_28 or musllinux" >&2 + exit 1 + fi + docker run -d --name build-$(LINUX_TAG)-$(ARCH) \ + --platform $(DOCKER_PLATFORM) \ + -v $(Build.SourcesDirectory):/workspace \ + -w /workspace \ + $IMAGE tail -f /dev/null + displayName: 'Start $(LINUX_TAG) $(ARCH) container' + + # Build the single py3-none wheel inside the container. Any CPython works; no + # compilation runs. setup_odbc.py's get_platform_info() auto-detects musl via + # platform.libc_ver(): on musllinux images it ALWAYS emits musllinux_1_2_ + # and ignores MANYLINUX_TAG. MANYLINUX_TAG is only consulted on glibc images, so + # hardcoding "manylinux_2_28" below is correct for both image families. + - script: | + set -euxo pipefail + if [[ "$(LINUX_TAG)" == "manylinux_2_28" ]]; then SHELL_EXE=bash; else SHELL_EXE=sh; fi + docker exec \ + -e targetArch="$(ARCH)" \ + -e MANYLINUX_TAG="manylinux_2_28" \ + build-$(LINUX_TAG)-$(ARCH) $SHELL_EXE -lc ' + set -eux + # Pick any available CPython from the PyPA image. + PY=$(ls -d /opt/python/cp312-cp312/bin/python 2>/dev/null || ls -d /opt/python/cp3*/bin/python | head -1) + echo "Using: $($PY --version)" + $PY -m pip install -q -U pip setuptools wheel twine + cd /workspace + $PY setup_odbc.py bdist_wheel + $PY -m twine check /workspace/dist/*.whl + echo "Produced wheels:"; ls -lh /workspace/dist/*.whl + ' + displayName: 'Build ODBC wheel in container' + + - script: | + set -euxo pipefail + cp -v $(Build.SourcesDirectory)/dist/*.whl $(ob_outputDirectory)/wheels/ + displayName: 'Stage wheel artifact' + + # Guard against the .gitignore '*.so' trap or a silent sync_libs() skip + # shipping a data wheel with NO driver binary (twine check would still pass). + # The driver filename contains 'msodbcsql' on every platform. + - script: | + set -eu + wheel_dir="$(ob_outputDirectory)/wheels" + matches=$(ls "$wheel_dir"/*.whl 2>/dev/null || true) + if [ -z "$matches" ]; then + echo "ERROR: no wheels found to verify in $wheel_dir" >&2 + exit 1 + fi + for whl in $matches; do + echo "Verifying ODBC driver binary present in: $whl" + if python3 -m zipfile -l "$whl" | grep -qi 'msodbcsql'; then + echo "OK: $whl contains the ODBC driver binary" + else + echo "ERROR: ODBC driver binary (msodbcsql*) missing from $whl" >&2 + exit 1 + fi + done + displayName: 'Assert wheel contains ODBC driver binary' + + - script: | + docker rm -f build-$(LINUX_TAG)-$(ARCH) || true + displayName: 'Stop and remove build container' + condition: always() + + - task: PublishPipelineArtifact@1 + displayName: 'Publish ODBC Linux Artifact' + inputs: + targetPath: '$(ob_outputDirectory)' + artifact: 'drop_${{ parameters.stageName }}_${{ parameters.jobName }}' + publishLocation: 'pipeline' + + - template: /OneBranchPipelines/steps/malware-scanning-step.yml@self + parameters: + scanPath: '$(ob_outputDirectory)' + artifactType: 'pkg' + + - ${{ if eq(parameters.oneBranchType, 'Official') }}: + - task: EsrpMalwareScanning@5 + displayName: 'ESRP MalwareScanning - ODBC Wheel (Official)' + inputs: + ConnectedServiceName: '$(SigningEsrpConnectedServiceName)' + AppRegistrationClientId: '$(SigningAppRegistrationClientId)' + AppRegistrationTenantId: '$(SigningAppRegistrationTenantId)' + EsrpClientId: '$(SigningEsrpClientId)' + UseMSIAuthentication: true + FolderPath: '$(ob_outputDirectory)/wheels' + Pattern: '*.whl' + SessionTimeout: 60 + CleanupTempStorage: 1 + VerboseLogin: 1 diff --git a/OneBranchPipelines/stages/build-odbc-macos-stage.yml b/OneBranchPipelines/stages/build-odbc-macos-stage.yml new file mode 100644 index 000000000..c1e7541b1 --- /dev/null +++ b/OneBranchPipelines/stages/build-odbc-macos-stage.yml @@ -0,0 +1,128 @@ +# ODBC macOS Single Configuration Stage Template +# Builds the universal2 `mssql-python-odbc` wheel (arm64 + x86_64 driver binaries in +# one wheel). +# +# Data-only package: no native compilation, no per-Python matrix, no pytest. +# `setup_odbc.py` returns the macosx_15_0_universal2 platform tag and copies both +# macOS arch subtrees from mssql_python/libs, so a single build serves both slices. +parameters: + # Stage identifier (e.g., 'ODBC_MacOS_universal2'). + - name: stageName + type: string + - name: jobName + type: string + default: 'BuildWheel' + - name: oneBranchType + type: string + default: 'Official' + - name: pythonVersion + type: string + default: '3.12' + +stages: + - stage: ${{ parameters.stageName }} + displayName: 'ODBC macOS Universal2' + jobs: + - job: ${{ parameters.jobName }} + displayName: 'Build ODBC Wheel - macOS Universal2' + # macOS pools declare as 'linux' type (Azure Pipelines quirk). + pool: + type: linux + isCustom: true + name: Azure Pipelines + vmImage: 'macOS-14' + timeoutInMinutes: 60 + + variables: + # BinSkim targets PE binaries; macOS uses Mach-O, so disable it here. + - name: ob_sdl_binskim_enabled + value: false + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)' + # OneBranch-required variable (unused on macOS stages). + - name: LinuxContainerImage + value: 'onebranch.azurecr.io/linux/ubuntu-2204:latest' + + steps: + - checkout: self + fetchDepth: 1 + + - task: UsePythonVersion@0 + inputs: + versionSpec: '${{ parameters.pythonVersion }}' + addToPath: true + displayName: 'Use Python ${{ parameters.pythonVersion }}' + + - script: | + set -e + python -m pip install --upgrade pip + python -m pip install setuptools wheel build twine + displayName: 'Install build tooling' + + # get_platform_info() forces the macosx_15_0_universal2 tag; sync_libs() copies + # both arm64 + x86_64 driver subtrees, so no post-build retag is required. + - script: | + set -e + python setup_odbc.py bdist_wheel + echo "Produced wheels:"; ls -lh dist/*.whl + displayName: 'Build ODBC wheel (universal2)' + + - script: | + set -e + python -m twine check dist/*.whl + displayName: 'twine check wheel' + + - task: CopyFiles@2 + inputs: + SourceFolder: '$(Build.SourcesDirectory)/dist' + Contents: '*.whl' + TargetFolder: '$(ob_outputDirectory)/wheels' + displayName: 'Stage wheel artifact' + + # Guard against a data wheel that packaged no driver binary (twine check + # would still pass). The driver filename contains 'msodbcsql' on every platform. + - script: | + set -eu + wheel_dir="$(ob_outputDirectory)/wheels" + matches=$(ls "$wheel_dir"/*.whl 2>/dev/null || true) + if [ -z "$matches" ]; then + echo "ERROR: no wheels found to verify in $wheel_dir" >&2 + exit 1 + fi + for whl in $matches; do + echo "Verifying ODBC driver binary present in: $whl" + if python -m zipfile -l "$whl" | grep -qi 'msodbcsql'; then + echo "OK: $whl contains the ODBC driver binary" + else + echo "ERROR: ODBC driver binary (msodbcsql*) missing from $whl" >&2 + exit 1 + fi + done + displayName: 'Assert wheel contains ODBC driver binary' + + - task: PublishPipelineArtifact@1 + displayName: 'Publish ODBC macOS Artifact' + inputs: + targetPath: '$(ob_outputDirectory)' + artifact: 'drop_${{ parameters.stageName }}_${{ parameters.jobName }}' + publishLocation: 'pipeline' + + - template: /OneBranchPipelines/steps/malware-scanning-step.yml@self + parameters: + scanPath: '$(ob_outputDirectory)' + artifactType: 'pkg' + + - ${{ if eq(parameters.oneBranchType, 'Official') }}: + - task: EsrpMalwareScanning@5 + displayName: 'ESRP MalwareScanning - ODBC Wheel (Official)' + inputs: + ConnectedServiceName: '$(SigningEsrpConnectedServiceName)' + AppRegistrationClientId: '$(SigningAppRegistrationClientId)' + AppRegistrationTenantId: '$(SigningAppRegistrationTenantId)' + EsrpClientId: '$(SigningEsrpClientId)' + UseMSIAuthentication: true + FolderPath: '$(ob_outputDirectory)/wheels' + Pattern: '*.whl' + SessionTimeout: 60 + CleanupTempStorage: 1 + VerboseLogin: 1 diff --git a/OneBranchPipelines/stages/build-odbc-windows-stage.yml b/OneBranchPipelines/stages/build-odbc-windows-stage.yml new file mode 100644 index 000000000..dea809ed0 --- /dev/null +++ b/OneBranchPipelines/stages/build-odbc-windows-stage.yml @@ -0,0 +1,145 @@ +# ODBC Windows Single Configuration Stage Template +# Builds the platform-specific `mssql-python-odbc` wheel for ONE Windows architecture. +# +# Unlike the mssql-python build, this package ships ONLY pre-built ODBC driver +# binaries (data) — there is NO compiled Python extension. A single +# `py3-none-` wheel therefore serves every supported Python version +# (3.10+), so this stage does NOT use a per-Python-version matrix and does NOT +# run pytest against SQL Server. The driver binaries are already committed under +# `mssql_python/libs/`, so `checkout: self` is all that is needed; `setup_odbc.py` +# packages the current platform's subtree. +parameters: + # Stage identifier (e.g., 'ODBC_Win_x64'). + - name: stageName + type: string + # Job identifier within the stage. + - name: jobName + type: string + default: 'BuildWheel' + # Target architecture: 'x64' (AMD64) or 'arm64' (ARM64). + - name: architecture + type: string + # OneBranch build type: 'Official' (production) or 'NonOfficial' (dev/test). + - name: oneBranchType + type: string + default: 'Official' + # Any supported interpreter can produce the Python-agnostic wheel. + - name: pythonVersion + type: string + default: '3.12' + +stages: + - stage: ${{ parameters.stageName }} + displayName: 'ODBC Windows ${{ parameters.architecture }}' + jobs: + - job: ${{ parameters.jobName }} + displayName: 'Build ODBC Wheel - Windows ${{ parameters.architecture }}' + # Reuse the same custom 1ES Windows pool as the mssql-python build. + pool: + type: windows + isCustom: true + name: Python-1ES-pool + demands: + - imageOverride -equals PYTHON-1ES-MMS2022 + timeoutInMinutes: 60 + + variables: + # OneBranch output directory for artifacts. + ob_outputDirectory: '$(Build.ArtifactStagingDirectory)' + # OneBranch-required variable (unused on Windows stages). + LinuxContainerImage: 'onebranch.azurecr.io/linux/ubuntu-2204:latest' + + steps: + # Driver binaries live in mssql_python/libs (committed); shallow checkout is enough. + - checkout: self + fetchDepth: 1 + + # The ARM64 wheel is pure data, so it can be produced on an x64 host — there is + # no native compilation and hence no ARM64 python.lib cross-compile machinery. + - task: UsePythonVersion@0 + inputs: + versionSpec: '${{ parameters.pythonVersion }}' + architecture: 'x64' + addToPath: true + displayName: 'Use Python ${{ parameters.pythonVersion }}' + + - powershell: | + $ErrorActionPreference = "Stop" + python -m pip install --upgrade pip + python -m pip install setuptools wheel build twine + displayName: 'Install build tooling' + + # Build the platform-specific, Python-agnostic wheel. ARCHITECTURE controls the + # platform tag (win_amd64 / win_arm64). No native compilation happens here — + # setup_odbc.py only packages the driver binaries from mssql_python/libs. + - powershell: | + $ErrorActionPreference = "Stop" + $env:ARCHITECTURE = "${{ parameters.architecture }}" + python setup_odbc.py bdist_wheel + Write-Host "Produced wheels:" + Get-ChildItem dist\*.whl | ForEach-Object { Write-Host " - $($_.Name)" } + displayName: 'Build ODBC wheel (ARCHITECTURE=${{ parameters.architecture }})' + + - powershell: | + $ErrorActionPreference = "Stop" + python -m twine check dist\*.whl + displayName: 'twine check wheel' + + - task: CopyFiles@2 + inputs: + SourceFolder: '$(Build.SourcesDirectory)\dist' + Contents: '*.whl' + TargetFolder: '$(ob_outputDirectory)\wheels' + displayName: 'Stage wheel artifact' + + # Guard against a data wheel that packaged no driver binary (twine check + # would still pass). The driver DLL name contains 'msodbcsql' on every platform. + - powershell: | + $ErrorActionPreference = "Stop" + $wheels = Get-ChildItem -Path "$(ob_outputDirectory)\wheels" -Filter "*.whl" + if ($wheels.Count -eq 0) { + Write-Error "No wheels found to verify in $(ob_outputDirectory)\wheels" + exit 1 + } + Add-Type -AssemblyName System.IO.Compression.FileSystem + foreach ($whl in $wheels) { + $zip = [System.IO.Compression.ZipFile]::OpenRead($whl.FullName) + $hasDriver = $zip.Entries | Where-Object { $_.FullName -match 'msodbcsql' } + $zip.Dispose() + if (-not $hasDriver) { + Write-Error "ODBC driver binary (msodbcsql*) missing from $($whl.Name)" + exit 1 + } + Write-Host "OK: $($whl.Name) contains the ODBC driver binary" + } + displayName: 'Assert wheel contains ODBC driver binary' + + # OneBranch requires artifact naming: drop__. + - task: PublishPipelineArtifact@1 + displayName: 'Publish ODBC Windows Artifact' + inputs: + targetPath: '$(ob_outputDirectory)' + artifact: 'drop_${{ parameters.stageName }}_${{ parameters.jobName }}' + publishLocation: 'pipeline' + + # Component Governance + OneBranch AntiMalware notification. + - template: /OneBranchPipelines/steps/malware-scanning-step.yml@self + parameters: + scanPath: '$(ob_outputDirectory)' + artifactType: 'pkg' + + # Scan the redistributed driver binaries + wheel for malware (Official only). + - ${{ if eq(parameters.oneBranchType, 'Official') }}: + - task: EsrpMalwareScanning@5 + displayName: 'ESRP MalwareScanning - ODBC Wheel (Official)' + inputs: + ConnectedServiceName: '$(SigningEsrpConnectedServiceName)' + AppRegistrationClientId: '$(SigningAppRegistrationClientId)' + AppRegistrationTenantId: '$(SigningAppRegistrationTenantId)' + EsrpClientId: '$(SigningEsrpClientId)' + UseMSIAuthentication: true + FolderPath: '$(ob_outputDirectory)/wheels' + Pattern: '*.whl' + SessionTimeout: 60 + CleanupTempStorage: 1 + VerboseLogin: 1