From b8456215dafd9c5ef1ca0de4af7c37c5c04f88e7 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Tue, 7 Jul 2026 20:27:59 +0530 Subject: [PATCH 01/13] Add OneBranch release pipelines for mssql-python-odbc Adds the build and release pipelines for the standalone mssql-python-odbc package: per-platform build stages (Windows x64/arm64, macOS universal2, Linux manylinux_2_28 + musllinux_1_2 for x86_64/aarch64), a consolidate job that gathers all 7 py3-none wheels, the build orchestration pipeline, and the ESRP-based official release pipeline. The build stages run no compilation (data-only package); Linux builds run in tdslibrs ACR PyPA containers and the musllinux tag is auto-detected via libc_ver. Infra follow-ups (ESRP onboarding of the mssql-python-odbc name, registering the ADO build definition to set odbcBuildDefinitionId, and sharing the ESRP variable group) are marked with TODO(infra) comments. --- .../build-release-odbc-pipeline.yml | 210 ++++++++++++++++ .../jobs/consolidate-odbc-artifacts-job.yml | 68 ++++++ .../official-release-odbc-pipeline.yml | 227 ++++++++++++++++++ .../stages/build-odbc-linux-stage.yml | 163 +++++++++++++ .../stages/build-odbc-macos-stage.yml | 107 +++++++++ .../stages/build-odbc-windows-stage.yml | 125 ++++++++++ 6 files changed, 900 insertions(+) create mode 100644 OneBranchPipelines/build-release-odbc-pipeline.yml create mode 100644 OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml create mode 100644 OneBranchPipelines/official-release-odbc-pipeline.yml create mode 100644 OneBranchPipelines/stages/build-odbc-linux-stage.yml create mode 100644 OneBranchPipelines/stages/build-odbc-macos-stage.yml create mode 100644 OneBranchPipelines/stages/build-odbc-windows-stage.yml diff --git a/OneBranchPipelines/build-release-odbc-pipeline.yml b/OneBranchPipelines/build-release-odbc-pipeline.yml new file mode 100644 index 000000000..260fd1c3c --- /dev/null +++ b/OneBranchPipelines/build-release-odbc-pipeline.yml @@ -0,0 +1,210 @@ +# ========================================================================================= +# OneBranch Build Pipeline for mssql-python-odbc +# ========================================================================================= +# Builds the standalone ODBC driver-binary package `mssql-python-odbc`. +# +# This package ships ONLY the Microsoft ODBC Driver 18 binaries as data. There is +# NO compiled Python extension, so every wheel is tagged `py3-none-` and a +# SINGLE build per platform covers all supported Python versions (3.10+). That is why +# this pipeline has NO per-Python-version matrix (unlike mssql-python) and produces +# only 7 wheels total: +# - Windows: x64, arm64 +# - macOS: universal2 +# - Linux: manylinux_2_28 (x86_64, aarch64), musllinux_1_2 (x86_64, aarch64) +# +# Publishing to PyPI is handled by the companion official-release-odbc-pipeline.yml +# using the SAME shared ESRP identity (variable group 'ESRP Federated Creds (AME)'). +# +# INFRA PREREQUISITES (see companion release pipeline for the definition ID): +# - Register this YAML as a new ADO build definition (e.g. 'Build-Release-ODBC-Pipeline'). +# - Share the 'ESRP Federated Creds (AME)' variable group with the new pipeline. +# - Ensure the reused pools/images are available: Python-1ES-pool +# (PYTHON-1ES-MMS2022 for Windows, PYTHON-1ES-UB2404 for Linux) and the +# 'Magnitude Test-mssql-rs-mssql-python' Azure subscription for ACR login. +# ========================================================================================= + +# Build number format: YYDDD.r +name: $(Year:YY)$(DayOfYear)$(Rev:.r) + +# Only trigger when ODBC-package inputs change (path filters are literal prefixes, +# not globs, in Azure DevOps). Unrelated mssql-python commits do not rebuild this. +trigger: + branches: + include: + - main + paths: + include: + - setup_odbc.py + - mssql_python_odbc + - mssql_python/libs + - OneBranchPipelines/build-release-odbc-pipeline.yml + - OneBranchPipelines/stages/build-odbc-windows-stage.yml + - OneBranchPipelines/stages/build-odbc-macos-stage.yml + - OneBranchPipelines/stages/build-odbc-linux-stage.yml + - OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml + +pr: + branches: + include: + - main + paths: + include: + - setup_odbc.py + - mssql_python_odbc + - mssql_python/libs + - OneBranchPipelines/build-release-odbc-pipeline.yml + - OneBranchPipelines/stages/build-odbc-windows-stage.yml + - OneBranchPipelines/stages/build-odbc-macos-stage.yml + - OneBranchPipelines/stages/build-odbc-linux-stage.yml + - OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml + +parameters: + - name: oneBranchType + displayName: 'OneBranch Template Type' + type: string + values: + - 'Official' + - 'NonOfficial' + default: 'NonOfficial' + + - name: runSdlTasks + displayName: 'Run SDL Security Tasks' + type: boolean + default: true + + # Windows: one wheel per architecture (NO per-Python matrix). + - name: odbcWindowsConfigs + type: object + default: + - arch: 'x64' + - arch: 'arm64' + + # macOS: single universal2 wheel. + - name: odbcMacosConfigs + type: object + default: + - name: 'universal2' + + # Linux: one wheel per libc/arch combo. + - name: odbcLinuxConfigs + type: object + default: + - { tag: 'manylinux_2_28', arch: 'x86_64', platform: 'linux/amd64' } + - { tag: 'manylinux_2_28', arch: 'aarch64', platform: 'linux/arm64' } + - { tag: 'musllinux', arch: 'x86_64', platform: 'linux/amd64' } + - { tag: 'musllinux', arch: 'aarch64', platform: 'linux/arm64' } + +variables: + # Scheduled builds run as Official; manual/PR builds use the parameter. + - name: effectiveOneBranchType + ${{ if eq(variables['Build.Reason'], 'Schedule') }}: + value: 'Official' + ${{ else }}: + value: '${{ parameters.oneBranchType }}' + + - template: /OneBranchPipelines/variables/common-variables.yml@self + - template: /OneBranchPipelines/variables/onebranch-variables.yml@self + # Maps 'ESRP Federated Creds (AME)' group vars to the Signing* names used by the + # per-stage EsrpMalwareScanning tasks. + - template: /OneBranchPipelines/variables/signing-variables.yml@self + - group: 'ESRP Federated Creds (AME)' + +resources: + repositories: + - repository: templates + type: git + name: 'OneBranch.Pipelines/GovernedTemplates' + ref: 'refs/heads/main' + +extends: + template: 'v2/OneBranch.${{ variables.effectiveOneBranchType }}.CrossPlat.yml@templates' + + parameters: + featureFlags: + WindowsHostVersion: + Version: '2022' + + globalSdl: + baseline: + baselineFile: $(Build.SourcesDirectory)/.gdn/.gdnbaselines + suppressionSet: default + suppression: + suppressionFile: $(Build.SourcesDirectory)/.gdn/.gdnsuppress + suppressionSet: default + + # This pipeline compiles NO first-party native code — it only packages the + # pre-built ODBC driver binaries, which are signed and governed by the Microsoft + # ODBC Driver team's own SDL. Native/binary analyzers are therefore disabled here; + # every wheel is still malware-scanned per stage (EsrpMalwareScanning@5). + binskim: + enabled: false + justificationForDisabling: 'No first-party native code is built here; the packaged ODBC driver binaries are pre-built/pre-signed and covered by the ODBC Driver team SDL. Wheels are malware-scanned per stage.' + apiscan: + enabled: false + justificationForDisabling: 'No first-party native binaries are produced by this pipeline.' + armory: + enabled: false + codeql: + enabled: false + justificationForDisabling: 'This pipeline only packages pre-built driver binaries; no first-party source is compiled.' + + credscan: + enabled: ${{ parameters.runSdlTasks }} + policheck: + enabled: ${{ parameters.runSdlTasks }} + break: true + exclusionFile: '$(REPO_ROOT)/.config/PolicheckExclusions.xml' + publishLogs: + enabled: ${{ parameters.runSdlTasks }} + sbom: + enabled: ${{ parameters.runSdlTasks }} + packageName: 'mssql-python-odbc' + tsa: + enabled: ${{ and(eq(variables.effectiveOneBranchType, 'Official'), parameters.runSdlTasks) }} + configFile: '$(REPO_ROOT)/.config/tsaoptions.json' + + stages: + # WINDOWS BUILD STAGES (2: x64, arm64) + - ${{ each config in parameters.odbcWindowsConfigs }}: + - template: /OneBranchPipelines/stages/build-odbc-windows-stage.yml@self + parameters: + stageName: ODBC_Win_${{ config.arch }} + jobName: BuildWheel + architecture: ${{ config.arch }} + oneBranchType: '${{ variables.effectiveOneBranchType }}' + + # MACOS BUILD STAGE (1: universal2) + - ${{ each config in parameters.odbcMacosConfigs }}: + - template: /OneBranchPipelines/stages/build-odbc-macos-stage.yml@self + parameters: + stageName: ODBC_MacOS_${{ config.name }} + jobName: BuildWheel + oneBranchType: '${{ variables.effectiveOneBranchType }}' + + # LINUX BUILD STAGES (4: manylinux/musllinux x x86_64/aarch64) + - ${{ each config in parameters.odbcLinuxConfigs }}: + - template: /OneBranchPipelines/stages/build-odbc-linux-stage.yml@self + parameters: + stageName: ODBC_Linux_${{ config.tag }}_${{ config.arch }} + jobName: BuildWheel + linuxTag: ${{ config.tag }} + arch: ${{ config.arch }} + dockerPlatform: ${{ config.platform }} + oneBranchType: '${{ variables.effectiveOneBranchType }}' + + # CONSOLIDATE STAGE (waits for all 7 platform stages) + - stage: Consolidate + displayName: 'Consolidate All ODBC Artifacts' + dependsOn: + - ODBC_Win_x64 + - ODBC_Win_arm64 + - ODBC_MacOS_universal2 + - ODBC_Linux_manylinux_2_28_x86_64 + - ODBC_Linux_manylinux_2_28_aarch64 + - ODBC_Linux_musllinux_x86_64 + - ODBC_Linux_musllinux_aarch64 + jobs: + - template: /OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml@self + parameters: + oneBranchType: '${{ variables.effectiveOneBranchType }}' + expectedWheelCount: 7 diff --git a/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml b/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml new file mode 100644 index 000000000..8ac829631 --- /dev/null +++ b/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml @@ -0,0 +1,68 @@ +# Consolidate ODBC Artifacts Job Template +# Downloads the per-platform `mssql-python-odbc` wheels from all build stages and +# consolidates them into a single dist/ folder for the release pipeline to publish. +# Expected: 7 wheels (2 Windows + 1 macOS universal2 + 4 Linux). +parameters: + - name: oneBranchType + type: string + default: 'Official' + - name: expectedWheelCount + type: number + default: 7 + +jobs: + - job: ConsolidateArtifacts + displayName: 'Consolidate All ODBC Platform Artifacts' + condition: succeeded() + + pool: + type: linux + isCustom: true + name: Azure Pipelines + vmImage: 'ubuntu-latest' + + variables: + # Consolidation only moves files; no binaries to scan. + - name: ob_sdl_binskim_enabled + value: false + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)' + + steps: + - checkout: self + fetchDepth: 1 + + # Download ALL artifacts from the current build (one per platform stage). + - task: DownloadPipelineArtifact@2 + displayName: 'Download All ODBC Platform Artifacts' + inputs: + buildType: 'current' + targetPath: '$(Pipeline.Workspace)/all-artifacts' + + - bash: | + set -e + mkdir -p $(ob_outputDirectory)/dist + + echo "Finding all .whl files..." + find $(Pipeline.Workspace)/all-artifacts -name "*.whl" -exec ls -lh {} \; + + echo "Copying all wheels to consolidated dist/..." + find $(Pipeline.Workspace)/all-artifacts -name "*.whl" -exec cp -v {} $(ob_outputDirectory)/dist/ \; + + echo "Consolidated wheels:" + ls -lh $(ob_outputDirectory)/dist/ + WHEEL_COUNT=$(ls -1 $(ob_outputDirectory)/dist/*.whl 2>/dev/null | wc -l) + echo "Total wheel count: $WHEEL_COUNT (expected ${{ parameters.expectedWheelCount }})" + if [ "$WHEEL_COUNT" -ne "${{ parameters.expectedWheelCount }}" ]; then + echo "WARNING: expected ${{ parameters.expectedWheelCount }} wheels but found $WHEEL_COUNT" + else + echo "SUCCESS: all ${{ parameters.expectedWheelCount }} ODBC wheels consolidated." + fi + displayName: 'Consolidate ODBC wheels' + + - task: PublishPipelineArtifact@1 + displayName: 'Publish Consolidated ODBC Artifacts' + inputs: + targetPath: '$(ob_outputDirectory)' + artifact: 'drop_Consolidate_ConsolidateArtifacts' + publishLocation: 'pipeline' diff --git a/OneBranchPipelines/official-release-odbc-pipeline.yml b/OneBranchPipelines/official-release-odbc-pipeline.yml new file mode 100644 index 000000000..3a9898c25 --- /dev/null +++ b/OneBranchPipelines/official-release-odbc-pipeline.yml @@ -0,0 +1,227 @@ +# ========================================================================================= +# OneBranch Official Release Pipeline for mssql-python-odbc +# ========================================================================================= +# Downloads the consolidated wheels from the ODBC build pipeline and publishes them to +# PyPI via ESRP, using the SAME shared ESRP identity as mssql-python (variable group +# 'ESRP Federated Creds (AME)'). The EsrpRelease task is identical to the mssql-python +# release (same inputs and FolderLocation); the only differences are the build pipeline +# source and that this package has no debug symbols to publish. +# +# This pipeline is ALWAYS Official. It is runnable the moment ESRP onboarding of the +# `mssql-python-odbc` project name lands and the two TODO(infra) values below are set. +# ========================================================================================= + +name: $(Year:YY)$(DayOfYear)$(Rev:.r)-Release-ODBC + +# Manual trigger only — releases should be deliberate. +trigger: none +pr: none + +parameters: + - name: releaseToPyPI + displayName: 'Release to PyPI (Production)' + type: boolean + default: false # Safety: default false to prevent accidental releases. + +variables: + - template: /OneBranchPipelines/variables/common-variables.yml@self + - template: /OneBranchPipelines/variables/onebranch-variables.yml@self + # Shared ESRP identity — same group used by the mssql-python release pipeline. + - group: 'ESRP Federated Creds (AME)' + # TODO(infra): set to the ADO build definition ID of 'Build-Release-ODBC-Pipeline' + # once it is registered (the mssql-python build definition is 2199). Left as 0 so the + # pipeline compiles; the download tasks will fail at runtime until this is set. + - name: odbcBuildDefinitionId + value: '0' + +resources: + repositories: + - repository: templates + type: git + name: 'OneBranch.Pipelines/GovernedTemplates' + ref: 'refs/heads/main' + + # Reference to the ODBC build pipeline (select a run in the UI when releasing). + pipelines: + - pipeline: buildPipeline + source: 'Build-Release-ODBC-Pipeline' # TODO(infra): match the registered build pipeline name. + trigger: none + +extends: + template: 'v2/OneBranch.Official.CrossPlat.yml@templates' + + parameters: + featureFlags: + WindowsHostVersion: + Version: '2022' + + globalSdl: + baseline: + baselineFile: $(Build.SourcesDirectory)\.gdn\.gdnbaselines + suppressionSet: default + suppression: + suppressionFile: $(Build.SourcesDirectory)\.gdn\.gdnsuppress + suppressionSet: default + + # Artifacts were already scanned during the build; this pipeline only republishes. + binskim: + enabled: false + justificationForDisabling: 'Release pipeline only republishes pre-scanned wheels; no binaries are built here.' + credscan: + enabled: true + policheck: + enabled: true + break: true + exclusionFile: '$(REPO_ROOT)/.config/PolicheckExclusions.xml' + publishLogs: + enabled: true + tsa: + enabled: true + configFile: '$(REPO_ROOT)/.config/tsaoptions.json' + + stages: + - stage: ReleasePackages + displayName: 'Release mssql-python-odbc to PyPI' + + jobs: + # Job 1: Download + validate the consolidated wheels (custom pool). + - job: PrepAndValidate + displayName: 'Download and Validate Artifacts' + + pool: + type: windows + isCustom: true + name: Python-1ES-pool + demands: + - imageOverride -equals PYTHON-1ES-MMS2022 + + variables: + ob_outputDirectory: '$(Build.ArtifactStagingDirectory)' + + steps: + - task: DownloadPipelineArtifact@2 + displayName: 'Download Consolidated ODBC Artifacts' + inputs: + buildType: 'specific' + project: '$(System.TeamProject)' + definition: $(odbcBuildDefinitionId) + buildVersionToDownload: 'specific' + buildId: $(resources.pipeline.buildPipeline.runID) + artifactName: 'drop_Consolidate_ConsolidateArtifacts' + targetPath: '$(Build.SourcesDirectory)/artifacts' + + - task: PowerShell@2 + displayName: 'List and Verify Downloaded Wheels' + inputs: + targetType: 'inline' + script: | + $wheelsPath = "$(Build.SourcesDirectory)/artifacts/dist" + if (-not (Test-Path $wheelsPath)) { + Write-Error "Wheel directory not found at: $wheelsPath" + exit 1 + } + $wheels = Get-ChildItem -Path $wheelsPath -Filter "*.whl" + Write-Host "Total wheel files found: $($wheels.Count)" + + $allValid = $true + Add-Type -AssemblyName System.IO.Compression.FileSystem + foreach ($wheel in $wheels) { + try { + $zip = [System.IO.Compression.ZipFile]::OpenRead($wheel.FullName) + $entryCount = $zip.Entries.Count + $zip.Dispose() + Write-Host " OK $($wheel.Name) ($entryCount entries)" + } catch { + Write-Error " BAD $($wheel.Name): $_" + $allValid = $false + } + } + if (-not $allValid) { + Write-Error "One or more wheel files are corrupted" + exit 1 + } + if ($wheels.Count -ne 7) { + Write-Warning "Expected 7 ODBC wheels but found $($wheels.Count)" + } + Write-Host "All wheels verified successfully!" + + # Dry-run summary (when release is disabled). + - ${{ if eq(parameters.releaseToPyPI, false) }}: + - task: PowerShell@2 + displayName: 'Dry Run - Release Skipped' + inputs: + targetType: 'inline' + script: | + Write-Host "====================================" + Write-Host "DRY RUN MODE - No Release Performed" + Write-Host "Package: mssql-python-odbc" + Write-Host "====================================" + Write-Host "To perform an actual release:" + Write-Host "1. Set 'releaseToPyPI' parameter to true" + Write-Host "2. Re-run pipeline" + + # Job 2: ESRP Release (releaseJob on 1ES hosted pool — required by OneBranch policy). + # EsrpRelease is not allowed in custom pools; must use templateContext.type: releaseJob. + - ${{ if eq(parameters.releaseToPyPI, true) }}: + - job: PyPIRelease + displayName: 'ESRP Release to PyPI' + dependsOn: PrepAndValidate + + templateContext: + type: releaseJob + isProduction: true + + pool: + type: windows + + variables: + ob_outputDirectory: '$(Build.ArtifactStagingDirectory)' + WindowsContainerImage: 'onebranch.azurecr.io/windows/ltsc2022/vse2022:latest' + + steps: + - task: DownloadPipelineArtifact@2 + displayName: 'Download Artifacts from ODBC Build Pipeline' + inputs: + buildType: 'specific' + project: '$(System.TeamProject)' + definition: $(odbcBuildDefinitionId) + buildVersionToDownload: 'specific' + buildId: $(resources.pipeline.buildPipeline.runID) + artifactName: 'drop_Consolidate_ConsolidateArtifacts' + targetPath: '$(Build.SourcesDirectory)/artifacts' + + # ESRP release task — identical to the mssql-python release pipeline + # (same shared ESRP identity and PyPI publish inputs). + - task: EsrpRelease@9 + displayName: 'ESRP Release to PyPI' + inputs: + connectedservicename: '$(ESRPConnectedServiceName)' + usemanagedidentity: true + keyvaultname: '$(AuthAKVName)' + signcertname: '$(AuthSignCertName)' + clientid: '$(EsrpClientId)' + Intent: 'PackageDistribution' + ContentType: 'PyPI' + ContentSource: 'Folder' + FolderLocation: '$(Build.SourcesDirectory)/artifacts/dist' + WaitForReleaseCompletion: true + Owners: '$(owner)' + Approvers: '$(approver)' + ServiceEndpointUrl: 'https://api.esrp.microsoft.com' + MainPublisher: 'ESRPRELPACMAN' + DomainTenantId: '$(DomainTenantId)' + + - task: PowerShell@2 + displayName: 'Release Summary' + inputs: + targetType: 'inline' + script: | + Write-Host "====================================" + Write-Host "ESRP Release Completed" + Write-Host "Package: mssql-python-odbc" + Write-Host "Target: PyPI" + Write-Host "====================================" + Write-Host "Next steps:" + Write-Host "1. Verify release in the ESRP portal" + Write-Host "2. Wait for approval workflow completion" + Write-Host "3. Verify on PyPI: https://pypi.org/project/mssql-python-odbc/" diff --git a/OneBranchPipelines/stages/build-odbc-linux-stage.yml b/OneBranchPipelines/stages/build-odbc-linux-stage.yml new file mode 100644 index 000000000..798b3c341 --- /dev/null +++ b/OneBranchPipelines/stages/build-odbc-linux-stage.yml @@ -0,0 +1,163 @@ +# ODBC Linux Single Configuration Stage Template +# Builds the platform-specific `mssql-python-odbc` wheel for ONE libc/arch combo +# (manylinux_2_28 or musllinux_1_2 x x86_64 or aarch64). +# +# Data-only package: no native compilation, no per-Python matrix, no pytest. +# A PyPA build container is still used so `platform.libc_ver()` reports the right +# libc and the wheel is tagged manylinux_2_28_* or musllinux_1_2_* correctly. +# aarch64 wheels are produced under QEMU emulation. Driver binaries come from the +# committed mssql_python/libs/ tree that is bind-mounted into the container. +parameters: + # Stage identifier (e.g., 'ODBC_Linux_manylinux_2_28_x86_64'). + - name: stageName + type: string + - name: jobName + type: string + default: 'BuildWheel' + # Distribution type: 'manylinux_2_28' (glibc 2.28+) or 'musllinux' (musl libc). + - name: linuxTag + type: string + # CPU architecture: 'x86_64' (AMD64) or 'aarch64' (ARM64). + - name: arch + type: string + # Docker platform for QEMU emulation: 'linux/amd64' or 'linux/arm64'. + - name: dockerPlatform + type: string + - name: oneBranchType + type: string + default: 'Official' + +stages: + - stage: ${{ parameters.stageName }} + displayName: 'ODBC Linux ${{ parameters.linuxTag }} ${{ parameters.arch }}' + jobs: + - job: ${{ parameters.jobName }} + displayName: 'Build ODBC Wheel - ${{ parameters.linuxTag }} ${{ parameters.arch }}' + # Reuse the same custom 1ES Linux pool as the mssql-python build. + pool: + type: linux + isCustom: true + name: Python-1ES-pool + demands: + - imageOverride -equals PYTHON-1ES-UB2404 + timeoutInMinutes: 60 + + variables: + # BinSkim needs ICU libs not present in manylinux/musllinux containers. + - name: ob_sdl_binskim_enabled + value: false + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)' + - name: LinuxContainerImage + value: 'onebranch.azurecr.io/linux/ubuntu-2204:latest' + - name: LINUX_TAG + value: ${{ parameters.linuxTag }} + - name: ARCH + value: ${{ parameters.arch }} + - name: DOCKER_PLATFORM + value: ${{ parameters.dockerPlatform }} + + steps: + - checkout: self + fetchDepth: 1 + + - bash: | + set -e + if ! docker info > /dev/null 2>&1; then + echo "Starting Docker daemon..." + sudo dockerd > docker.log 2>&1 & + sleep 10 + fi + docker --version + displayName: 'Setup and start Docker daemon' + + - script: | + sudo apt-get install -y qemu-user-static + displayName: 'Enable QEMU (for aarch64)' + + - script: | + rm -rf $(ob_outputDirectory)/wheels + mkdir -p $(ob_outputDirectory)/wheels + displayName: 'Prepare artifact directories' + + - task: AzureCLI@2 + displayName: 'Login to ACR (tdslibrs)' + inputs: + azureSubscription: 'Magnitude Test-mssql-rs-mssql-python' + scriptType: 'bash' + scriptLocation: 'inlineScript' + inlineScript: | + az acr login --name tdslibrs + + - script: | + set -euxo pipefail + # Same PyPA images as the mssql-python Linux build (mirrored into ACR). + if [[ "$(LINUX_TAG)" == "manylinux_2_28" ]]; then + IMAGE="tdslibrs.azurecr.io/import/python-build/manylinux_2_28_$(ARCH):latest" + elif [[ "$(LINUX_TAG)" == "musllinux" ]]; then + IMAGE="tdslibrs.azurecr.io/import/python-build/musllinux_1_2_$(ARCH):latest" + else + echo "ERROR: Unsupported LINUX_TAG '$(LINUX_TAG)'. Expected: manylinux_2_28 or musllinux" >&2 + exit 1 + fi + docker run -d --name build-$(LINUX_TAG)-$(ARCH) \ + --platform $(DOCKER_PLATFORM) \ + -v $(Build.SourcesDirectory):/workspace \ + -w /workspace \ + $IMAGE tail -f /dev/null + displayName: 'Start $(LINUX_TAG) $(ARCH) container' + + # Build the single py3-none wheel inside the container. Any CPython works; no + # compilation runs. setup_odbc.py's get_platform_info() auto-detects musl via + # platform.libc_ver(): on musllinux images it ALWAYS emits musllinux_1_2_ + # and ignores MANYLINUX_TAG. MANYLINUX_TAG is only consulted on glibc images, so + # hardcoding "manylinux_2_28" below is correct for both image families. + - script: | + set -euxo pipefail + if [[ "$(LINUX_TAG)" == "manylinux_2_28" ]]; then SHELL_EXE=bash; else SHELL_EXE=sh; fi + docker exec \ + -e targetArch="$(ARCH)" \ + -e MANYLINUX_TAG="manylinux_2_28" \ + build-$(LINUX_TAG)-$(ARCH) $SHELL_EXE -lc ' + set -eux + # Pick any available CPython from the PyPA image. + PY=$(ls -d /opt/python/cp312-cp312/bin/python 2>/dev/null || ls -d /opt/python/cp3*/bin/python | head -1) + echo "Using: $($PY --version)" + $PY -m pip install -q -U pip setuptools wheel + cd /workspace + $PY setup_odbc.py bdist_wheel + echo "Produced wheels:"; ls -lh /workspace/dist/*.whl + ' + displayName: 'Build ODBC wheel in container' + + - script: | + set -euxo pipefail + cp -v $(Build.SourcesDirectory)/dist/*.whl $(ob_outputDirectory)/wheels/ + displayName: 'Stage wheel artifact' + + - task: PublishPipelineArtifact@1 + displayName: 'Publish ODBC Linux Artifact' + inputs: + targetPath: '$(ob_outputDirectory)' + artifact: 'drop_${{ parameters.stageName }}_${{ parameters.jobName }}' + publishLocation: 'pipeline' + + - template: /OneBranchPipelines/steps/malware-scanning-step.yml@self + parameters: + scanPath: '$(ob_outputDirectory)' + artifactType: 'pkg' + + - ${{ if eq(parameters.oneBranchType, 'Official') }}: + - task: EsrpMalwareScanning@5 + displayName: 'ESRP MalwareScanning - ODBC Wheel (Official)' + inputs: + ConnectedServiceName: '$(SigningEsrpConnectedServiceName)' + AppRegistrationClientId: '$(SigningAppRegistrationClientId)' + AppRegistrationTenantId: '$(SigningAppRegistrationTenantId)' + EsrpClientId: '$(SigningEsrpClientId)' + UseMSIAuthentication: true + FolderPath: '$(ob_outputDirectory)/wheels' + Pattern: '*.whl' + SessionTimeout: 60 + CleanupTempStorage: 1 + VerboseLogin: 1 diff --git a/OneBranchPipelines/stages/build-odbc-macos-stage.yml b/OneBranchPipelines/stages/build-odbc-macos-stage.yml new file mode 100644 index 000000000..9b47388e3 --- /dev/null +++ b/OneBranchPipelines/stages/build-odbc-macos-stage.yml @@ -0,0 +1,107 @@ +# ODBC macOS Single Configuration Stage Template +# Builds the universal2 `mssql-python-odbc` wheel (arm64 + x86_64 driver binaries in +# one wheel). +# +# Data-only package: no native compilation, no per-Python matrix, no pytest. +# `setup_odbc.py` returns the macosx_15_0_universal2 platform tag and copies both +# macOS arch subtrees from mssql_python/libs, so a single build serves both slices. +parameters: + # Stage identifier (e.g., 'ODBC_MacOS_universal2'). + - name: stageName + type: string + - name: jobName + type: string + default: 'BuildWheel' + - name: oneBranchType + type: string + default: 'Official' + - name: pythonVersion + type: string + default: '3.12' + +stages: + - stage: ${{ parameters.stageName }} + displayName: 'ODBC macOS Universal2' + jobs: + - job: ${{ parameters.jobName }} + displayName: 'Build ODBC Wheel - macOS Universal2' + # macOS pools declare as 'linux' type (Azure Pipelines quirk). + pool: + type: linux + isCustom: true + name: Azure Pipelines + vmImage: 'macOS-14' + timeoutInMinutes: 60 + + variables: + # BinSkim targets PE binaries; macOS uses Mach-O, so disable it here. + - name: ob_sdl_binskim_enabled + value: false + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)' + # OneBranch-required variable (unused on macOS stages). + - name: LinuxContainerImage + value: 'onebranch.azurecr.io/linux/ubuntu-2204:latest' + + steps: + - checkout: self + fetchDepth: 1 + + - task: UsePythonVersion@0 + inputs: + versionSpec: '${{ parameters.pythonVersion }}' + addToPath: true + displayName: 'Use Python ${{ parameters.pythonVersion }}' + + - script: | + set -e + python -m pip install --upgrade pip + python -m pip install setuptools wheel build twine + displayName: 'Install build tooling' + + # get_platform_info() forces the macosx_15_0_universal2 tag; sync_libs() copies + # both arm64 + x86_64 driver subtrees, so no post-build retag is required. + - script: | + set -e + python setup_odbc.py bdist_wheel + echo "Produced wheels:"; ls -lh dist/*.whl + displayName: 'Build ODBC wheel (universal2)' + + - script: | + set -e + python -m twine check dist/*.whl + displayName: 'twine check wheel' + + - task: CopyFiles@2 + inputs: + SourceFolder: '$(Build.SourcesDirectory)/dist' + Contents: '*.whl' + TargetFolder: '$(ob_outputDirectory)/wheels' + displayName: 'Stage wheel artifact' + + - task: PublishPipelineArtifact@1 + displayName: 'Publish ODBC macOS Artifact' + inputs: + targetPath: '$(ob_outputDirectory)' + artifact: 'drop_${{ parameters.stageName }}_${{ parameters.jobName }}' + publishLocation: 'pipeline' + + - template: /OneBranchPipelines/steps/malware-scanning-step.yml@self + parameters: + scanPath: '$(ob_outputDirectory)' + artifactType: 'pkg' + + - ${{ if eq(parameters.oneBranchType, 'Official') }}: + - task: EsrpMalwareScanning@5 + displayName: 'ESRP MalwareScanning - ODBC Wheel (Official)' + inputs: + ConnectedServiceName: '$(SigningEsrpConnectedServiceName)' + AppRegistrationClientId: '$(SigningAppRegistrationClientId)' + AppRegistrationTenantId: '$(SigningAppRegistrationTenantId)' + EsrpClientId: '$(SigningEsrpClientId)' + UseMSIAuthentication: true + FolderPath: '$(ob_outputDirectory)/wheels' + Pattern: '*.whl' + SessionTimeout: 60 + CleanupTempStorage: 1 + VerboseLogin: 1 diff --git a/OneBranchPipelines/stages/build-odbc-windows-stage.yml b/OneBranchPipelines/stages/build-odbc-windows-stage.yml new file mode 100644 index 000000000..7a812de45 --- /dev/null +++ b/OneBranchPipelines/stages/build-odbc-windows-stage.yml @@ -0,0 +1,125 @@ +# ODBC Windows Single Configuration Stage Template +# Builds the platform-specific `mssql-python-odbc` wheel for ONE Windows architecture. +# +# Unlike the mssql-python build, this package ships ONLY pre-built ODBC driver +# binaries (data) — there is NO compiled Python extension. A single +# `py3-none-` wheel therefore serves every supported Python version +# (3.10+), so this stage does NOT use a per-Python-version matrix and does NOT +# run pytest against SQL Server. The driver binaries are already committed under +# `mssql_python/libs/`, so `checkout: self` is all that is needed; `setup_odbc.py` +# packages the current platform's subtree. +parameters: + # Stage identifier (e.g., 'ODBC_Win_x64'). + - name: stageName + type: string + # Job identifier within the stage. + - name: jobName + type: string + default: 'BuildWheel' + # Target architecture: 'x64' (AMD64) or 'arm64' (ARM64). + - name: architecture + type: string + # OneBranch build type: 'Official' (production) or 'NonOfficial' (dev/test). + - name: oneBranchType + type: string + default: 'Official' + # Any supported interpreter can produce the Python-agnostic wheel. + - name: pythonVersion + type: string + default: '3.12' + +stages: + - stage: ${{ parameters.stageName }} + displayName: 'ODBC Windows ${{ parameters.architecture }}' + jobs: + - job: ${{ parameters.jobName }} + displayName: 'Build ODBC Wheel - Windows ${{ parameters.architecture }}' + # Reuse the same custom 1ES Windows pool as the mssql-python build. + pool: + type: windows + isCustom: true + name: Python-1ES-pool + demands: + - imageOverride -equals PYTHON-1ES-MMS2022 + timeoutInMinutes: 60 + + variables: + # OneBranch output directory for artifacts. + ob_outputDirectory: '$(Build.ArtifactStagingDirectory)' + # OneBranch-required variable (unused on Windows stages). + LinuxContainerImage: 'onebranch.azurecr.io/linux/ubuntu-2204:latest' + # Drives the wheel platform tag in setup_odbc.py (get_platform_info()). + targetArch: ${{ parameters.architecture }} + + steps: + # Driver binaries live in mssql_python/libs (committed); shallow checkout is enough. + - checkout: self + fetchDepth: 1 + + # The ARM64 wheel is pure data, so it can be produced on an x64 host — there is + # no native compilation and hence no ARM64 python.lib cross-compile machinery. + - task: UsePythonVersion@0 + inputs: + versionSpec: '${{ parameters.pythonVersion }}' + architecture: 'x64' + addToPath: true + displayName: 'Use Python ${{ parameters.pythonVersion }}' + + - powershell: | + $ErrorActionPreference = "Stop" + python -m pip install --upgrade pip + python -m pip install setuptools wheel build twine + displayName: 'Install build tooling' + + # Build the platform-specific, Python-agnostic wheel. ARCHITECTURE controls the + # platform tag (win_amd64 / win_arm64). No native compilation happens here — + # setup_odbc.py only packages the driver binaries from mssql_python/libs. + - powershell: | + $ErrorActionPreference = "Stop" + $env:ARCHITECTURE = "${{ parameters.architecture }}" + python setup_odbc.py bdist_wheel + Write-Host "Produced wheels:" + Get-ChildItem dist\*.whl | ForEach-Object { Write-Host " - $($_.Name)" } + displayName: 'Build ODBC wheel (ARCHITECTURE=${{ parameters.architecture }})' + + - powershell: | + $ErrorActionPreference = "Stop" + python -m twine check dist\*.whl + displayName: 'twine check wheel' + + - task: CopyFiles@2 + inputs: + SourceFolder: '$(Build.SourcesDirectory)\dist' + Contents: '*.whl' + TargetFolder: '$(ob_outputDirectory)\wheels' + displayName: 'Stage wheel artifact' + + # OneBranch requires artifact naming: drop__. + - task: PublishPipelineArtifact@1 + displayName: 'Publish ODBC Windows Artifact' + inputs: + targetPath: '$(ob_outputDirectory)' + artifact: 'drop_${{ parameters.stageName }}_${{ parameters.jobName }}' + publishLocation: 'pipeline' + + # Component Governance + OneBranch AntiMalware notification. + - template: /OneBranchPipelines/steps/malware-scanning-step.yml@self + parameters: + scanPath: '$(ob_outputDirectory)' + artifactType: 'pkg' + + # Scan the redistributed driver binaries + wheel for malware (Official only). + - ${{ if eq(parameters.oneBranchType, 'Official') }}: + - task: EsrpMalwareScanning@5 + displayName: 'ESRP MalwareScanning - ODBC Wheel (Official)' + inputs: + ConnectedServiceName: '$(SigningEsrpConnectedServiceName)' + AppRegistrationClientId: '$(SigningAppRegistrationClientId)' + AppRegistrationTenantId: '$(SigningAppRegistrationTenantId)' + EsrpClientId: '$(SigningEsrpClientId)' + UseMSIAuthentication: true + FolderPath: '$(ob_outputDirectory)/wheels' + Pattern: '*.whl' + SessionTimeout: 60 + CleanupTempStorage: 1 + VerboseLogin: 1 From c19a261ed16c625e4f35f9111941600fa9bcd3d4 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Tue, 7 Jul 2026 21:43:11 +0530 Subject: [PATCH 02/13] Address PR review: fail-fast on wheel count, armory justification, Linux twine check + container cleanup, reword infra TODOs --- OneBranchPipelines/build-release-odbc-pipeline.yml | 1 + .../jobs/consolidate-odbc-artifacts-job.yml | 6 +++--- OneBranchPipelines/official-release-odbc-pipeline.yml | 9 +++++---- OneBranchPipelines/stages/build-odbc-linux-stage.yml | 8 +++++++- 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/OneBranchPipelines/build-release-odbc-pipeline.yml b/OneBranchPipelines/build-release-odbc-pipeline.yml index 260fd1c3c..b64dc936b 100644 --- a/OneBranchPipelines/build-release-odbc-pipeline.yml +++ b/OneBranchPipelines/build-release-odbc-pipeline.yml @@ -144,6 +144,7 @@ extends: justificationForDisabling: 'No first-party native binaries are produced by this pipeline.' armory: enabled: false + justificationForDisabling: 'This pipeline only packages pre-built driver binaries; no first-party native code is built. Wheels are malware-scanned per stage.' codeql: enabled: false justificationForDisabling: 'This pipeline only packages pre-built driver binaries; no first-party source is compiled.' diff --git a/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml b/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml index 8ac829631..0225d34a2 100644 --- a/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml +++ b/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml @@ -54,10 +54,10 @@ jobs: WHEEL_COUNT=$(ls -1 $(ob_outputDirectory)/dist/*.whl 2>/dev/null | wc -l) echo "Total wheel count: $WHEEL_COUNT (expected ${{ parameters.expectedWheelCount }})" if [ "$WHEEL_COUNT" -ne "${{ parameters.expectedWheelCount }}" ]; then - echo "WARNING: expected ${{ parameters.expectedWheelCount }} wheels but found $WHEEL_COUNT" - else - echo "SUCCESS: all ${{ parameters.expectedWheelCount }} ODBC wheels consolidated." + echo "ERROR: expected ${{ parameters.expectedWheelCount }} wheels but found $WHEEL_COUNT" >&2 + exit 1 fi + echo "SUCCESS: all ${{ parameters.expectedWheelCount }} ODBC wheels consolidated." displayName: 'Consolidate ODBC wheels' - task: PublishPipelineArtifact@1 diff --git a/OneBranchPipelines/official-release-odbc-pipeline.yml b/OneBranchPipelines/official-release-odbc-pipeline.yml index 3a9898c25..90eaedddb 100644 --- a/OneBranchPipelines/official-release-odbc-pipeline.yml +++ b/OneBranchPipelines/official-release-odbc-pipeline.yml @@ -8,7 +8,7 @@ # source and that this package has no debug symbols to publish. # # This pipeline is ALWAYS Official. It is runnable the moment ESRP onboarding of the -# `mssql-python-odbc` project name lands and the two TODO(infra) values below are set. +# `mssql-python-odbc` project name lands and the two infra values below are set. # ========================================================================================= name: $(Year:YY)$(DayOfYear)$(Rev:.r)-Release-ODBC @@ -28,7 +28,7 @@ variables: - template: /OneBranchPipelines/variables/onebranch-variables.yml@self # Shared ESRP identity — same group used by the mssql-python release pipeline. - group: 'ESRP Federated Creds (AME)' - # TODO(infra): set to the ADO build definition ID of 'Build-Release-ODBC-Pipeline' + # Infra setup: set to the ADO build definition ID of 'Build-Release-ODBC-Pipeline' # once it is registered (the mssql-python build definition is 2199). Left as 0 so the # pipeline compiles; the download tasks will fail at runtime until this is set. - name: odbcBuildDefinitionId @@ -44,7 +44,7 @@ resources: # Reference to the ODBC build pipeline (select a run in the UI when releasing). pipelines: - pipeline: buildPipeline - source: 'Build-Release-ODBC-Pipeline' # TODO(infra): match the registered build pipeline name. + source: 'Build-Release-ODBC-Pipeline' # Infra setup: match the registered build pipeline name. trigger: none extends: @@ -141,7 +141,8 @@ extends: exit 1 } if ($wheels.Count -ne 7) { - Write-Warning "Expected 7 ODBC wheels but found $($wheels.Count)" + Write-Error "Expected 7 ODBC wheels but found $($wheels.Count)" + exit 1 } Write-Host "All wheels verified successfully!" diff --git a/OneBranchPipelines/stages/build-odbc-linux-stage.yml b/OneBranchPipelines/stages/build-odbc-linux-stage.yml index 798b3c341..d0a9883cd 100644 --- a/OneBranchPipelines/stages/build-odbc-linux-stage.yml +++ b/OneBranchPipelines/stages/build-odbc-linux-stage.yml @@ -123,9 +123,10 @@ stages: # Pick any available CPython from the PyPA image. PY=$(ls -d /opt/python/cp312-cp312/bin/python 2>/dev/null || ls -d /opt/python/cp3*/bin/python | head -1) echo "Using: $($PY --version)" - $PY -m pip install -q -U pip setuptools wheel + $PY -m pip install -q -U pip setuptools wheel twine cd /workspace $PY setup_odbc.py bdist_wheel + $PY -m twine check /workspace/dist/*.whl echo "Produced wheels:"; ls -lh /workspace/dist/*.whl ' displayName: 'Build ODBC wheel in container' @@ -135,6 +136,11 @@ stages: cp -v $(Build.SourcesDirectory)/dist/*.whl $(ob_outputDirectory)/wheels/ displayName: 'Stage wheel artifact' + - script: | + docker rm -f build-$(LINUX_TAG)-$(ARCH) || true + displayName: 'Stop and remove build container' + condition: always() + - task: PublishPipelineArtifact@1 displayName: 'Publish ODBC Linux Artifact' inputs: From 62910b4b92233601caa258ead365d7d5417bee92 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Tue, 7 Jul 2026 21:58:20 +0530 Subject: [PATCH 03/13] Address review: preflight guard for odbcBuildDefinitionId placeholder; drop unused targetArch var on Windows stage --- OneBranchPipelines/official-release-odbc-pipeline.yml | 11 +++++++++++ .../stages/build-odbc-windows-stage.yml | 2 -- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/OneBranchPipelines/official-release-odbc-pipeline.yml b/OneBranchPipelines/official-release-odbc-pipeline.yml index 90eaedddb..ef06b0895 100644 --- a/OneBranchPipelines/official-release-odbc-pipeline.yml +++ b/OneBranchPipelines/official-release-odbc-pipeline.yml @@ -99,6 +99,17 @@ extends: ob_outputDirectory: '$(Build.ArtifactStagingDirectory)' steps: + - task: PowerShell@2 + displayName: 'Preflight: verify odbcBuildDefinitionId is configured' + inputs: + targetType: 'inline' + script: | + if ('$(odbcBuildDefinitionId)' -eq '0' -or [string]::IsNullOrWhiteSpace('$(odbcBuildDefinitionId)')) { + Write-Error "odbcBuildDefinitionId is still the '0' placeholder. Set it to the registered 'Build-Release-ODBC-Pipeline' build definition ID before releasing, otherwise the artifact download below fails cryptically." + exit 1 + } + Write-Host "odbcBuildDefinitionId = $(odbcBuildDefinitionId)" + - task: DownloadPipelineArtifact@2 displayName: 'Download Consolidated ODBC Artifacts' inputs: diff --git a/OneBranchPipelines/stages/build-odbc-windows-stage.yml b/OneBranchPipelines/stages/build-odbc-windows-stage.yml index 7a812de45..57de838ed 100644 --- a/OneBranchPipelines/stages/build-odbc-windows-stage.yml +++ b/OneBranchPipelines/stages/build-odbc-windows-stage.yml @@ -48,8 +48,6 @@ stages: ob_outputDirectory: '$(Build.ArtifactStagingDirectory)' # OneBranch-required variable (unused on Windows stages). LinuxContainerImage: 'onebranch.azurecr.io/linux/ubuntu-2204:latest' - # Drives the wheel platform tag in setup_odbc.py (get_platform_info()). - targetArch: ${{ parameters.architecture }} steps: # Driver binaries live in mssql_python/libs (committed); shallow checkout is enough. From 38b9ff9fbae948078f9e62e3355adee23a98cf17 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Wed, 8 Jul 2026 11:33:46 +0530 Subject: [PATCH 04/13] Fold mssql-python-odbc build+release into existing pipelines Per review feedback, build and release the standalone mssql-python-odbc package from the existing Build-Release-Package-Pipeline (def 2199) and official release pipeline instead of separate ADO definitions. Build pipeline: add odbcWindows/Macos/LinuxConfigs params (2+1+4=7 data-only wheels), reference the 3 odbc stage templates alongside the mssql-python stages, and add a separate ConsolidateOdbc stage publishing drop_ConsolidateOdbc_ConsolidateArtifacts (distinct from mssql-python's drop_Consolidate_ConsolidateArtifacts). Release pipeline: add releasePackage parameter (mssql-python | mssql-python-odbc); select the consolidated artifact accordingly and skip symbol publishing + mssql-py-core version validation for odbc. Delete the redundant standalone orchestrators (build-release-odbc-pipeline.yml, official-release-odbc-pipeline.yml). This removes the need to register a new ADO build definition and the odbcBuildDefinitionId placeholder. Both packages now build from def 2199; the odbc stages consume setup_odbc.py from PR #663, so #663 must merge before these stages can build. --- .../build-release-odbc-pipeline.yml | 211 ---------------- .../build-release-package-pipeline.yml | 94 ++++++- .../jobs/consolidate-odbc-artifacts-job.yml | 5 +- .../official-release-odbc-pipeline.yml | 239 ------------------ .../official-release-pipeline.yml | 56 ++-- 5 files changed, 136 insertions(+), 469 deletions(-) delete mode 100644 OneBranchPipelines/build-release-odbc-pipeline.yml delete mode 100644 OneBranchPipelines/official-release-odbc-pipeline.yml diff --git a/OneBranchPipelines/build-release-odbc-pipeline.yml b/OneBranchPipelines/build-release-odbc-pipeline.yml deleted file mode 100644 index b64dc936b..000000000 --- a/OneBranchPipelines/build-release-odbc-pipeline.yml +++ /dev/null @@ -1,211 +0,0 @@ -# ========================================================================================= -# OneBranch Build Pipeline for mssql-python-odbc -# ========================================================================================= -# Builds the standalone ODBC driver-binary package `mssql-python-odbc`. -# -# This package ships ONLY the Microsoft ODBC Driver 18 binaries as data. There is -# NO compiled Python extension, so every wheel is tagged `py3-none-` and a -# SINGLE build per platform covers all supported Python versions (3.10+). That is why -# this pipeline has NO per-Python-version matrix (unlike mssql-python) and produces -# only 7 wheels total: -# - Windows: x64, arm64 -# - macOS: universal2 -# - Linux: manylinux_2_28 (x86_64, aarch64), musllinux_1_2 (x86_64, aarch64) -# -# Publishing to PyPI is handled by the companion official-release-odbc-pipeline.yml -# using the SAME shared ESRP identity (variable group 'ESRP Federated Creds (AME)'). -# -# INFRA PREREQUISITES (see companion release pipeline for the definition ID): -# - Register this YAML as a new ADO build definition (e.g. 'Build-Release-ODBC-Pipeline'). -# - Share the 'ESRP Federated Creds (AME)' variable group with the new pipeline. -# - Ensure the reused pools/images are available: Python-1ES-pool -# (PYTHON-1ES-MMS2022 for Windows, PYTHON-1ES-UB2404 for Linux) and the -# 'Magnitude Test-mssql-rs-mssql-python' Azure subscription for ACR login. -# ========================================================================================= - -# Build number format: YYDDD.r -name: $(Year:YY)$(DayOfYear)$(Rev:.r) - -# Only trigger when ODBC-package inputs change (path filters are literal prefixes, -# not globs, in Azure DevOps). Unrelated mssql-python commits do not rebuild this. -trigger: - branches: - include: - - main - paths: - include: - - setup_odbc.py - - mssql_python_odbc - - mssql_python/libs - - OneBranchPipelines/build-release-odbc-pipeline.yml - - OneBranchPipelines/stages/build-odbc-windows-stage.yml - - OneBranchPipelines/stages/build-odbc-macos-stage.yml - - OneBranchPipelines/stages/build-odbc-linux-stage.yml - - OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml - -pr: - branches: - include: - - main - paths: - include: - - setup_odbc.py - - mssql_python_odbc - - mssql_python/libs - - OneBranchPipelines/build-release-odbc-pipeline.yml - - OneBranchPipelines/stages/build-odbc-windows-stage.yml - - OneBranchPipelines/stages/build-odbc-macos-stage.yml - - OneBranchPipelines/stages/build-odbc-linux-stage.yml - - OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml - -parameters: - - name: oneBranchType - displayName: 'OneBranch Template Type' - type: string - values: - - 'Official' - - 'NonOfficial' - default: 'NonOfficial' - - - name: runSdlTasks - displayName: 'Run SDL Security Tasks' - type: boolean - default: true - - # Windows: one wheel per architecture (NO per-Python matrix). - - name: odbcWindowsConfigs - type: object - default: - - arch: 'x64' - - arch: 'arm64' - - # macOS: single universal2 wheel. - - name: odbcMacosConfigs - type: object - default: - - name: 'universal2' - - # Linux: one wheel per libc/arch combo. - - name: odbcLinuxConfigs - type: object - default: - - { tag: 'manylinux_2_28', arch: 'x86_64', platform: 'linux/amd64' } - - { tag: 'manylinux_2_28', arch: 'aarch64', platform: 'linux/arm64' } - - { tag: 'musllinux', arch: 'x86_64', platform: 'linux/amd64' } - - { tag: 'musllinux', arch: 'aarch64', platform: 'linux/arm64' } - -variables: - # Scheduled builds run as Official; manual/PR builds use the parameter. - - name: effectiveOneBranchType - ${{ if eq(variables['Build.Reason'], 'Schedule') }}: - value: 'Official' - ${{ else }}: - value: '${{ parameters.oneBranchType }}' - - - template: /OneBranchPipelines/variables/common-variables.yml@self - - template: /OneBranchPipelines/variables/onebranch-variables.yml@self - # Maps 'ESRP Federated Creds (AME)' group vars to the Signing* names used by the - # per-stage EsrpMalwareScanning tasks. - - template: /OneBranchPipelines/variables/signing-variables.yml@self - - group: 'ESRP Federated Creds (AME)' - -resources: - repositories: - - repository: templates - type: git - name: 'OneBranch.Pipelines/GovernedTemplates' - ref: 'refs/heads/main' - -extends: - template: 'v2/OneBranch.${{ variables.effectiveOneBranchType }}.CrossPlat.yml@templates' - - parameters: - featureFlags: - WindowsHostVersion: - Version: '2022' - - globalSdl: - baseline: - baselineFile: $(Build.SourcesDirectory)/.gdn/.gdnbaselines - suppressionSet: default - suppression: - suppressionFile: $(Build.SourcesDirectory)/.gdn/.gdnsuppress - suppressionSet: default - - # This pipeline compiles NO first-party native code — it only packages the - # pre-built ODBC driver binaries, which are signed and governed by the Microsoft - # ODBC Driver team's own SDL. Native/binary analyzers are therefore disabled here; - # every wheel is still malware-scanned per stage (EsrpMalwareScanning@5). - binskim: - enabled: false - justificationForDisabling: 'No first-party native code is built here; the packaged ODBC driver binaries are pre-built/pre-signed and covered by the ODBC Driver team SDL. Wheels are malware-scanned per stage.' - apiscan: - enabled: false - justificationForDisabling: 'No first-party native binaries are produced by this pipeline.' - armory: - enabled: false - justificationForDisabling: 'This pipeline only packages pre-built driver binaries; no first-party native code is built. Wheels are malware-scanned per stage.' - codeql: - enabled: false - justificationForDisabling: 'This pipeline only packages pre-built driver binaries; no first-party source is compiled.' - - credscan: - enabled: ${{ parameters.runSdlTasks }} - policheck: - enabled: ${{ parameters.runSdlTasks }} - break: true - exclusionFile: '$(REPO_ROOT)/.config/PolicheckExclusions.xml' - publishLogs: - enabled: ${{ parameters.runSdlTasks }} - sbom: - enabled: ${{ parameters.runSdlTasks }} - packageName: 'mssql-python-odbc' - tsa: - enabled: ${{ and(eq(variables.effectiveOneBranchType, 'Official'), parameters.runSdlTasks) }} - configFile: '$(REPO_ROOT)/.config/tsaoptions.json' - - stages: - # WINDOWS BUILD STAGES (2: x64, arm64) - - ${{ each config in parameters.odbcWindowsConfigs }}: - - template: /OneBranchPipelines/stages/build-odbc-windows-stage.yml@self - parameters: - stageName: ODBC_Win_${{ config.arch }} - jobName: BuildWheel - architecture: ${{ config.arch }} - oneBranchType: '${{ variables.effectiveOneBranchType }}' - - # MACOS BUILD STAGE (1: universal2) - - ${{ each config in parameters.odbcMacosConfigs }}: - - template: /OneBranchPipelines/stages/build-odbc-macos-stage.yml@self - parameters: - stageName: ODBC_MacOS_${{ config.name }} - jobName: BuildWheel - oneBranchType: '${{ variables.effectiveOneBranchType }}' - - # LINUX BUILD STAGES (4: manylinux/musllinux x x86_64/aarch64) - - ${{ each config in parameters.odbcLinuxConfigs }}: - - template: /OneBranchPipelines/stages/build-odbc-linux-stage.yml@self - parameters: - stageName: ODBC_Linux_${{ config.tag }}_${{ config.arch }} - jobName: BuildWheel - linuxTag: ${{ config.tag }} - arch: ${{ config.arch }} - dockerPlatform: ${{ config.platform }} - oneBranchType: '${{ variables.effectiveOneBranchType }}' - - # CONSOLIDATE STAGE (waits for all 7 platform stages) - - stage: Consolidate - displayName: 'Consolidate All ODBC Artifacts' - dependsOn: - - ODBC_Win_x64 - - ODBC_Win_arm64 - - ODBC_MacOS_universal2 - - ODBC_Linux_manylinux_2_28_x86_64 - - ODBC_Linux_manylinux_2_28_aarch64 - - ODBC_Linux_musllinux_x86_64 - - ODBC_Linux_musllinux_aarch64 - jobs: - - template: /OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml@self - parameters: - oneBranchType: '${{ variables.effectiveOneBranchType }}' - expectedWheelCount: 7 diff --git a/OneBranchPipelines/build-release-package-pipeline.yml b/OneBranchPipelines/build-release-package-pipeline.yml index 3bd789e85..984e52bcc 100644 --- a/OneBranchPipelines/build-release-package-pipeline.yml +++ b/OneBranchPipelines/build-release-package-pipeline.yml @@ -138,6 +138,33 @@ parameters: - { tag: 'musllinux', arch: 'x86_64', platform: 'linux/amd64' } - { tag: 'musllinux', arch: 'aarch64', platform: 'linux/arm64' } + # ========================= + # ODBC PACKAGE CONFIGURATIONS (mssql-python-odbc) + # ========================= + # The standalone mssql-python-odbc package ships ONLY pre-built ODBC driver + # binaries (data) — there is NO compiled extension — so ONE py3-none- + # wheel per platform serves every supported Python version. Hence there is no + # per-Python matrix here: 7 wheels total (2 Windows + 1 macOS + 4 Linux). + # These stages build alongside the mssql-python wheels and are consolidated + # SEPARATELY (see the ConsolidateOdbc stage) so the release pipeline can publish + # mssql-python and mssql-python-odbc independently. + - name: odbcWindowsConfigs + type: object + default: + - { arch: 'x64' } + - { arch: 'arm64' } + - name: odbcMacosConfigs + type: object + default: + - { name: 'universal2' } + - name: odbcLinuxConfigs + type: object + default: + - { tag: 'manylinux_2_28', arch: 'x86_64', platform: 'linux/amd64' } + - { tag: 'manylinux_2_28', arch: 'aarch64', platform: 'linux/arm64' } + - { tag: 'musllinux', arch: 'x86_64', platform: 'linux/amd64' } + - { tag: 'musllinux', arch: 'aarch64', platform: 'linux/arm64' } + # ========================= # PIPELINE VARIABLES # ========================= @@ -338,8 +365,10 @@ extends: # ========================= # PIPELINE STAGES # ========================= - # Total stages: 9 Windows + 5 macOS + 4 Linux + 1 Consolidate = 19 stages - # Stages run in parallel (no dependencies between platform builds) + # mssql-python: 9 Windows + 5 macOS + 4 Linux + 1 Consolidate = 19 stages + # mssql-python-odbc: 2 Windows + 1 macOS + 4 Linux + 1 ConsolidateOdbc = 8 stages + # Total: 27 stages. All build stages run in parallel; each Consolidate* stage + # waits only on its own package's build stages. stages: # ========================= # WINDOWS BUILD STAGES @@ -466,3 +495,64 @@ extends: # Note: Symbol publishing handled directly in Windows build stages # PDB files uploaded to Microsoft Symbol Server for debugging + + # ========================================================================= + # ODBC PACKAGE BUILD STAGES (mssql-python-odbc) + # ========================================================================= + # 7 data-only wheels (no native compile, no pytest) built from setup_odbc.py. + # They run in parallel with the mssql-python stages and are consolidated by a + # SEPARATE ConsolidateOdbc stage so the release pipeline can ship each package + # independently. SDL "just works": these wheels contain the SAME ODBC driver + # binaries already scanned in mssql_python/libs by the main build — no new + # first-party native code is introduced. + # + # ODBC Windows stages (2): ODBC_Win_x64, ODBC_Win_arm64 + - ${{ each config in parameters.odbcWindowsConfigs }}: + - template: /OneBranchPipelines/stages/build-odbc-windows-stage.yml@self + parameters: + stageName: ODBC_Win_${{ config.arch }} + jobName: BuildWheel + architecture: ${{ config.arch }} + oneBranchType: '${{ variables.effectiveOneBranchType }}' + + # ODBC macOS stage (1): ODBC_MacOS_universal2 + - ${{ each config in parameters.odbcMacosConfigs }}: + - template: /OneBranchPipelines/stages/build-odbc-macos-stage.yml@self + parameters: + stageName: ODBC_MacOS_${{ config.name }} + jobName: BuildWheel + oneBranchType: '${{ variables.effectiveOneBranchType }}' + + # ODBC Linux stages (4): manylinux_2_28 x86_64/aarch64, musllinux x86_64/aarch64 + - ${{ each config in parameters.odbcLinuxConfigs }}: + - template: /OneBranchPipelines/stages/build-odbc-linux-stage.yml@self + parameters: + stageName: ODBC_Linux_${{ config.tag }}_${{ config.arch }} + jobName: BuildWheel + linuxTag: ${{ config.tag }} + arch: ${{ config.arch }} + dockerPlatform: ${{ config.platform }} + oneBranchType: '${{ variables.effectiveOneBranchType }}' + + # ========================================================================= + # CONSOLIDATE ODBC STAGE + # ========================================================================= + # Collects the 7 mssql-python-odbc wheels into a single dist/ folder and + # publishes them as `drop_ConsolidateOdbc_ConsolidateArtifacts` (distinct from + # the mssql-python `drop_Consolidate_ConsolidateArtifacts`). The release + # pipeline selects one artifact based on its `releasePackage` parameter. + - stage: ConsolidateOdbc + displayName: 'Consolidate All ODBC Artifacts' + dependsOn: + - ODBC_Win_x64 + - ODBC_Win_arm64 + - ODBC_MacOS_universal2 + - ODBC_Linux_manylinux_2_28_x86_64 + - ODBC_Linux_manylinux_2_28_aarch64 + - ODBC_Linux_musllinux_x86_64 + - ODBC_Linux_musllinux_aarch64 + jobs: + - template: /OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml@self + parameters: + oneBranchType: '${{ variables.effectiveOneBranchType }}' + expectedWheelCount: 7 diff --git a/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml b/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml index 0225d34a2..42db132ff 100644 --- a/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml +++ b/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml @@ -64,5 +64,8 @@ jobs: displayName: 'Publish Consolidated ODBC Artifacts' inputs: targetPath: '$(ob_outputDirectory)' - artifact: 'drop_Consolidate_ConsolidateArtifacts' + # Distinct name so it does not collide with the mssql-python consolidate + # artifact (drop_Consolidate_ConsolidateArtifacts) in the same build run. + # Matches the OneBranch auto-name for a stage named `ConsolidateOdbc`. + artifact: 'drop_ConsolidateOdbc_ConsolidateArtifacts' publishLocation: 'pipeline' diff --git a/OneBranchPipelines/official-release-odbc-pipeline.yml b/OneBranchPipelines/official-release-odbc-pipeline.yml deleted file mode 100644 index ef06b0895..000000000 --- a/OneBranchPipelines/official-release-odbc-pipeline.yml +++ /dev/null @@ -1,239 +0,0 @@ -# ========================================================================================= -# OneBranch Official Release Pipeline for mssql-python-odbc -# ========================================================================================= -# Downloads the consolidated wheels from the ODBC build pipeline and publishes them to -# PyPI via ESRP, using the SAME shared ESRP identity as mssql-python (variable group -# 'ESRP Federated Creds (AME)'). The EsrpRelease task is identical to the mssql-python -# release (same inputs and FolderLocation); the only differences are the build pipeline -# source and that this package has no debug symbols to publish. -# -# This pipeline is ALWAYS Official. It is runnable the moment ESRP onboarding of the -# `mssql-python-odbc` project name lands and the two infra values below are set. -# ========================================================================================= - -name: $(Year:YY)$(DayOfYear)$(Rev:.r)-Release-ODBC - -# Manual trigger only — releases should be deliberate. -trigger: none -pr: none - -parameters: - - name: releaseToPyPI - displayName: 'Release to PyPI (Production)' - type: boolean - default: false # Safety: default false to prevent accidental releases. - -variables: - - template: /OneBranchPipelines/variables/common-variables.yml@self - - template: /OneBranchPipelines/variables/onebranch-variables.yml@self - # Shared ESRP identity — same group used by the mssql-python release pipeline. - - group: 'ESRP Federated Creds (AME)' - # Infra setup: set to the ADO build definition ID of 'Build-Release-ODBC-Pipeline' - # once it is registered (the mssql-python build definition is 2199). Left as 0 so the - # pipeline compiles; the download tasks will fail at runtime until this is set. - - name: odbcBuildDefinitionId - value: '0' - -resources: - repositories: - - repository: templates - type: git - name: 'OneBranch.Pipelines/GovernedTemplates' - ref: 'refs/heads/main' - - # Reference to the ODBC build pipeline (select a run in the UI when releasing). - pipelines: - - pipeline: buildPipeline - source: 'Build-Release-ODBC-Pipeline' # Infra setup: match the registered build pipeline name. - trigger: none - -extends: - template: 'v2/OneBranch.Official.CrossPlat.yml@templates' - - parameters: - featureFlags: - WindowsHostVersion: - Version: '2022' - - globalSdl: - baseline: - baselineFile: $(Build.SourcesDirectory)\.gdn\.gdnbaselines - suppressionSet: default - suppression: - suppressionFile: $(Build.SourcesDirectory)\.gdn\.gdnsuppress - suppressionSet: default - - # Artifacts were already scanned during the build; this pipeline only republishes. - binskim: - enabled: false - justificationForDisabling: 'Release pipeline only republishes pre-scanned wheels; no binaries are built here.' - credscan: - enabled: true - policheck: - enabled: true - break: true - exclusionFile: '$(REPO_ROOT)/.config/PolicheckExclusions.xml' - publishLogs: - enabled: true - tsa: - enabled: true - configFile: '$(REPO_ROOT)/.config/tsaoptions.json' - - stages: - - stage: ReleasePackages - displayName: 'Release mssql-python-odbc to PyPI' - - jobs: - # Job 1: Download + validate the consolidated wheels (custom pool). - - job: PrepAndValidate - displayName: 'Download and Validate Artifacts' - - pool: - type: windows - isCustom: true - name: Python-1ES-pool - demands: - - imageOverride -equals PYTHON-1ES-MMS2022 - - variables: - ob_outputDirectory: '$(Build.ArtifactStagingDirectory)' - - steps: - - task: PowerShell@2 - displayName: 'Preflight: verify odbcBuildDefinitionId is configured' - inputs: - targetType: 'inline' - script: | - if ('$(odbcBuildDefinitionId)' -eq '0' -or [string]::IsNullOrWhiteSpace('$(odbcBuildDefinitionId)')) { - Write-Error "odbcBuildDefinitionId is still the '0' placeholder. Set it to the registered 'Build-Release-ODBC-Pipeline' build definition ID before releasing, otherwise the artifact download below fails cryptically." - exit 1 - } - Write-Host "odbcBuildDefinitionId = $(odbcBuildDefinitionId)" - - - task: DownloadPipelineArtifact@2 - displayName: 'Download Consolidated ODBC Artifacts' - inputs: - buildType: 'specific' - project: '$(System.TeamProject)' - definition: $(odbcBuildDefinitionId) - buildVersionToDownload: 'specific' - buildId: $(resources.pipeline.buildPipeline.runID) - artifactName: 'drop_Consolidate_ConsolidateArtifacts' - targetPath: '$(Build.SourcesDirectory)/artifacts' - - - task: PowerShell@2 - displayName: 'List and Verify Downloaded Wheels' - inputs: - targetType: 'inline' - script: | - $wheelsPath = "$(Build.SourcesDirectory)/artifacts/dist" - if (-not (Test-Path $wheelsPath)) { - Write-Error "Wheel directory not found at: $wheelsPath" - exit 1 - } - $wheels = Get-ChildItem -Path $wheelsPath -Filter "*.whl" - Write-Host "Total wheel files found: $($wheels.Count)" - - $allValid = $true - Add-Type -AssemblyName System.IO.Compression.FileSystem - foreach ($wheel in $wheels) { - try { - $zip = [System.IO.Compression.ZipFile]::OpenRead($wheel.FullName) - $entryCount = $zip.Entries.Count - $zip.Dispose() - Write-Host " OK $($wheel.Name) ($entryCount entries)" - } catch { - Write-Error " BAD $($wheel.Name): $_" - $allValid = $false - } - } - if (-not $allValid) { - Write-Error "One or more wheel files are corrupted" - exit 1 - } - if ($wheels.Count -ne 7) { - Write-Error "Expected 7 ODBC wheels but found $($wheels.Count)" - exit 1 - } - Write-Host "All wheels verified successfully!" - - # Dry-run summary (when release is disabled). - - ${{ if eq(parameters.releaseToPyPI, false) }}: - - task: PowerShell@2 - displayName: 'Dry Run - Release Skipped' - inputs: - targetType: 'inline' - script: | - Write-Host "====================================" - Write-Host "DRY RUN MODE - No Release Performed" - Write-Host "Package: mssql-python-odbc" - Write-Host "====================================" - Write-Host "To perform an actual release:" - Write-Host "1. Set 'releaseToPyPI' parameter to true" - Write-Host "2. Re-run pipeline" - - # Job 2: ESRP Release (releaseJob on 1ES hosted pool — required by OneBranch policy). - # EsrpRelease is not allowed in custom pools; must use templateContext.type: releaseJob. - - ${{ if eq(parameters.releaseToPyPI, true) }}: - - job: PyPIRelease - displayName: 'ESRP Release to PyPI' - dependsOn: PrepAndValidate - - templateContext: - type: releaseJob - isProduction: true - - pool: - type: windows - - variables: - ob_outputDirectory: '$(Build.ArtifactStagingDirectory)' - WindowsContainerImage: 'onebranch.azurecr.io/windows/ltsc2022/vse2022:latest' - - steps: - - task: DownloadPipelineArtifact@2 - displayName: 'Download Artifacts from ODBC Build Pipeline' - inputs: - buildType: 'specific' - project: '$(System.TeamProject)' - definition: $(odbcBuildDefinitionId) - buildVersionToDownload: 'specific' - buildId: $(resources.pipeline.buildPipeline.runID) - artifactName: 'drop_Consolidate_ConsolidateArtifacts' - targetPath: '$(Build.SourcesDirectory)/artifacts' - - # ESRP release task — identical to the mssql-python release pipeline - # (same shared ESRP identity and PyPI publish inputs). - - task: EsrpRelease@9 - displayName: 'ESRP Release to PyPI' - inputs: - connectedservicename: '$(ESRPConnectedServiceName)' - usemanagedidentity: true - keyvaultname: '$(AuthAKVName)' - signcertname: '$(AuthSignCertName)' - clientid: '$(EsrpClientId)' - Intent: 'PackageDistribution' - ContentType: 'PyPI' - ContentSource: 'Folder' - FolderLocation: '$(Build.SourcesDirectory)/artifacts/dist' - WaitForReleaseCompletion: true - Owners: '$(owner)' - Approvers: '$(approver)' - ServiceEndpointUrl: 'https://api.esrp.microsoft.com' - MainPublisher: 'ESRPRELPACMAN' - DomainTenantId: '$(DomainTenantId)' - - - task: PowerShell@2 - displayName: 'Release Summary' - inputs: - targetType: 'inline' - script: | - Write-Host "====================================" - Write-Host "ESRP Release Completed" - Write-Host "Package: mssql-python-odbc" - Write-Host "Target: PyPI" - Write-Host "====================================" - Write-Host "Next steps:" - Write-Host "1. Verify release in the ESRP portal" - Write-Host "2. Wait for approval workflow completion" - Write-Host "3. Verify on PyPI: https://pypi.org/project/mssql-python-odbc/" diff --git a/OneBranchPipelines/official-release-pipeline.yml b/OneBranchPipelines/official-release-pipeline.yml index a5ec227f6..731df538e 100644 --- a/OneBranchPipelines/official-release-pipeline.yml +++ b/OneBranchPipelines/official-release-pipeline.yml @@ -1,5 +1,7 @@ -# OneBranch Official Release Pipeline for mssql-python -# Downloads wheel and symbol artifacts from build pipeline, publishes symbols, and releases wheels to PyPI via ESRP +# OneBranch Official Release Pipeline (mssql-python and mssql-python-odbc) +# Selects the package via the `releasePackage` parameter, downloads that package's +# consolidated wheels (and symbols for mssql-python) from the build pipeline +# (definition 2199), publishes symbols, and releases wheels to PyPI via ESRP. # This pipeline is ALWAYS Official - no NonOfficial option name: $(Year:YY)$(DayOfYear)$(Rev:.r)-Release @@ -10,6 +12,16 @@ pr: none # Parameters for release pipeline parameters: + # Which package to release. Both are produced by the same build pipeline + # (definition 2199); this switches the consolidated artifact and messaging. + - name: releasePackage + displayName: 'Package to Release' + type: string + values: + - 'mssql-python' + - 'mssql-python-odbc' + default: 'mssql-python' + - name: publishSymbols displayName: 'Publish Symbols to Symbol Servers' type: boolean @@ -30,6 +42,16 @@ variables: - group: 'ESRP Federated Creds (AME)' # Contains ESRP signing credentials - group: 'Symbols Publishing' # Contains SymbolServer, SymbolTokenUri variables + # Select which consolidated artifact to download/publish based on the target + # package. Both are produced by the same build pipeline (definition 2199): + # mssql-python -> drop_Consolidate_ConsolidateArtifacts + # mssql-python-odbc -> drop_ConsolidateOdbc_ConsolidateArtifacts + - name: consolidatedArtifactName + ${{ if eq(parameters.releasePackage, 'mssql-python-odbc') }}: + value: 'drop_ConsolidateOdbc_ConsolidateArtifacts' + ${{ else }}: + value: 'drop_Consolidate_ConsolidateArtifacts' + # OneBranch resources resources: repositories: @@ -117,7 +139,7 @@ extends: definition: 2199 # Build-Release-Package-Pipeline definition ID buildVersionToDownload: 'specific' buildId: $(resources.pipeline.buildPipeline.runID) # Use the build run selected in UI - artifactName: 'drop_Consolidate_ConsolidateArtifacts' # Consolidated artifact with dist/ and symbols/ + artifactName: '${{ variables.consolidatedArtifactName }}' # mssql-python or mssql-python-odbc consolidated wheels targetPath: '$(Build.SourcesDirectory)/artifacts' # Step 2: List downloaded artifacts for verification @@ -173,13 +195,15 @@ extends: Write-Host "Symbols: $(if ($symbols) { $symbols.Count } else { 0 }) files" Write-Host "=====================================" - # Step 3: Validate mssql-py-core is a stable version (no dev/alpha/beta/rc) - - task: PowerShell@2 - displayName: 'Validate mssql-py-core is a stable version' - inputs: - targetType: 'filePath' - filePath: '$(Build.SourcesDirectory)\OneBranchPipelines\scripts\validate-release-versions.ps1' - arguments: '-VersionFile "$(Build.SourcesDirectory)\artifacts\dist\mssql-py-core.version"' + # Step 3: Validate mssql-py-core is a stable version (mssql-python only; + # mssql-python-odbc ships no mssql-py-core.version file) + - ${{ if eq(parameters.releasePackage, 'mssql-python') }}: + - task: PowerShell@2 + displayName: 'Validate mssql-py-core is a stable version' + inputs: + targetType: 'filePath' + filePath: '$(Build.SourcesDirectory)\OneBranchPipelines\scripts\validate-release-versions.ps1' + arguments: '-VersionFile "$(Build.SourcesDirectory)\artifacts\dist\mssql-py-core.version"' # Step 4: Verify wheel integrity - task: PowerShell@2 @@ -215,8 +239,8 @@ extends: Write-Host "`nAll wheels verified successfully!" - # Step 5: Publish Symbols (if enabled and symbols exist) - - ${{ if eq(parameters.publishSymbols, true) }}: + # Step 5: Publish Symbols (mssql-python only; mssql-python-odbc has no PDBs) + - ${{ if and(eq(parameters.publishSymbols, true), eq(parameters.releasePackage, 'mssql-python')) }}: - template: /OneBranchPipelines/steps/symbol-publishing-step.yml@self parameters: SymbolsFolder: '$(Build.SourcesDirectory)/symbols' @@ -231,7 +255,7 @@ extends: Write-Host "====================================" Write-Host "DRY RUN MODE - No Release Performed" Write-Host "====================================" - Write-Host "Package: mssql-python" + Write-Host "Package: ${{ parameters.releasePackage }}" Write-Host "" Write-Host "Actions performed:" Write-Host "- Downloaded wheels from build pipeline" @@ -273,7 +297,7 @@ extends: definition: 2199 buildVersionToDownload: 'specific' buildId: $(resources.pipeline.buildPipeline.runID) - artifactName: 'drop_Consolidate_ConsolidateArtifacts' + artifactName: '${{ variables.consolidatedArtifactName }}' targetPath: '$(Build.SourcesDirectory)/artifacts' - task: EsrpRelease@9 @@ -303,7 +327,7 @@ extends: Write-Host "====================================" Write-Host "ESRP Release Completed" Write-Host "====================================" - Write-Host "Package: mssql-python" + Write-Host "Package: ${{ parameters.releasePackage }}" Write-Host "Target: PyPI" Write-Host "Owners: $(owner)" Write-Host "Approvers: $(approver)" @@ -313,6 +337,6 @@ extends: Write-Host "Next steps:" Write-Host "1. Verify release in ESRP portal" Write-Host "2. Wait for approval workflow completion" - Write-Host "3. Verify package on PyPI: https://pypi.org/project/mssql-python/" + Write-Host "3. Verify package on PyPI: https://pypi.org/project/${{ parameters.releasePackage }}/" Write-Host "4. Verify symbols in SqlClientDrivers org (if published)" Write-Host "=====================================" From d85fd92bb3d7ec75e47325f30491d629f7273272 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Wed, 8 Jul 2026 11:50:04 +0530 Subject: [PATCH 05/13] Scope consolidate downloads by package to prevent cross-contamination Def 2199 now builds mssql-python and mssql-python-odbc in one run. Both consolidate jobs used DownloadPipelineArtifact@2 with buildType: current, which downloads every artifact in the run and copies all *.whl into dist/. In the merged run this caused: (1) the main Consolidate to sweep the 7 odbc wheels into the mssql-python release (count is warning-only, so silent), and (2) ConsolidateOdbc to see 34 wheels while expecting 7 and hard-fail. Add itemPattern to scope each download to its own package's build-stage artifacts (main: drop_Win_*/drop_MacOS_*/drop_Linux_*; odbc: drop_ODBC_*), giving clean partitioning with no wheel leakage. --- .../jobs/consolidate-artifacts-job.yml | 13 ++++++++++--- .../jobs/consolidate-odbc-artifacts-job.yml | 6 +++++- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/OneBranchPipelines/jobs/consolidate-artifacts-job.yml b/OneBranchPipelines/jobs/consolidate-artifacts-job.yml index 478803aa3..336c736a5 100644 --- a/OneBranchPipelines/jobs/consolidate-artifacts-job.yml +++ b/OneBranchPipelines/jobs/consolidate-artifacts-job.yml @@ -29,13 +29,20 @@ jobs: - checkout: self fetchDepth: 1 - # Download ALL artifacts from current build - # Matrix jobs publish as: Windows_, macOS_, Linux_ - # This downloads all of them automatically (27 total artifacts) + # Download only the mssql-python platform-stage artifacts from the current build. + # The build definition now ALSO produces mssql-python-odbc artifacts (drop_ODBC_*) + # in the SAME run, so we scope this download with itemPattern to the mssql-python + # stages (drop_Win_*, drop_MacOS_*, drop_Linux_*). Without this, the odbc wheels + # would be swept into this package's dist/ and shipped to the mssql-python PyPI + # project. The odbc wheels are consolidated separately (ConsolidateOdbc stage). - task: DownloadPipelineArtifact@2 displayName: 'Download All Platform Artifacts' inputs: buildType: 'current' + itemPattern: | + drop_Win_*/** + drop_MacOS_*/** + drop_Linux_*/** targetPath: '$(Pipeline.Workspace)/all-artifacts' # Consolidate all wheels into single dist/ directory diff --git a/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml b/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml index 42db132ff..f0d8a0715 100644 --- a/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml +++ b/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml @@ -32,11 +32,15 @@ jobs: - checkout: self fetchDepth: 1 - # Download ALL artifacts from the current build (one per platform stage). + # Download only the mssql-python-odbc platform-stage artifacts (drop_ODBC_*) from + # the current build. The same run also builds the mssql-python wheels (drop_Win_*, + # drop_MacOS_*, drop_Linux_*); itemPattern scopes this download to the odbc stages + # so ONLY the 7 odbc wheels are consolidated here (not the mssql-python wheels). - task: DownloadPipelineArtifact@2 displayName: 'Download All ODBC Platform Artifacts' inputs: buildType: 'current' + itemPattern: 'drop_ODBC_*/**' targetPath: '$(Pipeline.Workspace)/all-artifacts' - bash: | From e3cf2a9a56827bbe9891b2c67d8ff2b0c819b6b1 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Wed, 8 Jul 2026 13:16:59 +0530 Subject: [PATCH 06/13] Correct stale mssql-python wheel count in consolidate diagnostic (27 -> 34) The matrix is 9 Windows + 5 macOS + 4 Linux stages x 5 Python versions (cp310-cp314) = 34 wheels across 18 stages. The diagnostic said 27 (7+4+16), stale from an older 3.10-3.13 / 7-Windows matrix. Count check remains warning-only (no behavior change). --- .../jobs/consolidate-artifacts-job.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/OneBranchPipelines/jobs/consolidate-artifacts-job.yml b/OneBranchPipelines/jobs/consolidate-artifacts-job.yml index 336c736a5..52eb43381 100644 --- a/OneBranchPipelines/jobs/consolidate-artifacts-job.yml +++ b/OneBranchPipelines/jobs/consolidate-artifacts-job.yml @@ -1,7 +1,7 @@ # Consolidate Artifacts Job Template # Downloads artifacts from all platform build stages and consolidates into single dist/ folder -# Works with individual stage artifacts (15 stages total: 7 Windows + 4 macOS + 4 Linux) -# Each Linux stage builds 4 Python versions, resulting in 27 total wheels +# Works with individual stage artifacts (18 stages total: 9 Windows + 5 macOS + 4 Linux) +# Each Linux stage builds 5 Python versions (3.10-3.14), resulting in 34 total wheels parameters: - name: oneBranchType type: string @@ -75,12 +75,12 @@ jobs: echo "" WHEEL_COUNT=$(ls -1 $(ob_outputDirectory)/dist/*.whl 2>/dev/null | wc -l) echo "Total wheel count: $WHEEL_COUNT" - echo "Expected: 27 wheels (7 Windows + 4 macOS + 16 Linux)" + echo "Expected: 34 wheels (9 Windows + 5 macOS + 20 Linux)" - if [ "$WHEEL_COUNT" -ne 27 ]; then - echo "WARNING: Expected 27 wheels but found $WHEEL_COUNT" + if [ "$WHEEL_COUNT" -ne 34 ]; then + echo "WARNING: Expected 34 wheels but found $WHEEL_COUNT" else - echo "SUCCESS: All 27 wheels consolidated!" + echo "SUCCESS: All 34 wheels consolidated!" fi displayName: 'Consolidate wheels from all platforms' From a4b4e59fc4f9ed09ffa5e79481875339a52159e7 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Wed, 8 Jul 2026 13:49:09 +0530 Subject: [PATCH 07/13] Harden odbc release path: pin guard + wheel-content assertions official-release-pipeline.yml: gate 'Published symbols' dry-run/summary text to mssql-python so odbc runs no longer claim symbol publishing that never happens; add a mssql-python-only step asserting the wheel metadata pins mssql-python-odbc==18.6.0 (fails by design if released from a pre-pin build). build-odbc-{linux,macos,windows}-stage.yml: assert each built wheel actually contains the ODBC driver binary (msodbcsql*), guarding against the .gitignore '*.so' trap / silent sync_libs() skip shipping an empty data wheel that still passes twine check. --- .../official-release-pipeline.yml | 48 ++++++++++++++++++- .../stages/build-odbc-linux-stage.yml | 22 +++++++++ .../stages/build-odbc-macos-stage.yml | 21 ++++++++ .../stages/build-odbc-windows-stage.yml | 22 +++++++++ 4 files changed, 111 insertions(+), 2 deletions(-) diff --git a/OneBranchPipelines/official-release-pipeline.yml b/OneBranchPipelines/official-release-pipeline.yml index 731df538e..7cb529796 100644 --- a/OneBranchPipelines/official-release-pipeline.yml +++ b/OneBranchPipelines/official-release-pipeline.yml @@ -205,6 +205,49 @@ extends: filePath: '$(Build.SourcesDirectory)\OneBranchPipelines\scripts\validate-release-versions.ps1' arguments: '-VersionFile "$(Build.SourcesDirectory)\artifacts\dist\mssql-py-core.version"' + # Guard: the released mssql-python wheel MUST depend on the pinned + # mssql-python-odbc package. Catches releasing from a PRE-pin build + # (e.g. the wrong build run), which would ship a driver-less package + # that breaks 'pip install mssql-python'. By design this FAILS on a + # pre-pin build. Update $expectedVersion when the odbc pin bumps. + - task: PowerShell@2 + displayName: 'Validate mssql-python-odbc pin in wheel metadata' + inputs: + targetType: 'inline' + script: | + $ErrorActionPreference = 'Stop' + $expectedVersion = '18.6.0' + $wheels = Get-ChildItem -Path "$(Build.SourcesDirectory)/artifacts/dist" -Filter "mssql_python-*.whl" + if ($wheels.Count -eq 0) { + Write-Error "No mssql_python-*.whl found in artifacts/dist to validate the odbc pin." + exit 1 + } + Add-Type -AssemblyName System.IO.Compression.FileSystem + $wheel = $wheels[0].FullName + Write-Host "Inspecting wheel metadata: $wheel" + $zip = [System.IO.Compression.ZipFile]::OpenRead($wheel) + $metaEntry = $zip.Entries | Where-Object { $_.FullName -match '\.dist-info/METADATA$' } | Select-Object -First 1 + if (-not $metaEntry) { + $zip.Dispose() + Write-Error "METADATA not found inside $wheel" + exit 1 + } + $reader = New-Object System.IO.StreamReader($metaEntry.Open()) + $metadata = $reader.ReadToEnd() + $reader.Dispose() + $zip.Dispose() + $odbcLines = ($metadata -split "`n") | Where-Object { $_ -match 'Requires-Dist:\s*mssql[-_]python[-_]odbc' } + if ($odbcLines) { + Write-Host "odbc dependency in metadata:" + $odbcLines | ForEach-Object { Write-Host " $($_.Trim())" } + } + $pinRegex = 'Requires-Dist:\s*mssql[-_]python[-_]odbc\s*==\s*' + [regex]::Escape($expectedVersion) + if ($metadata -notmatch $pinRegex) { + Write-Error "mssql-python wheel does not pin mssql-python-odbc==$expectedVersion. Are you releasing from the POST-pin build? mssql-python must depend on the pinned odbc package before release." + exit 1 + } + Write-Host "OK: wheel depends on mssql-python-odbc==$expectedVersion" + # Step 4: Verify wheel integrity - task: PowerShell@2 displayName: 'Verify Wheel Integrity' @@ -261,7 +304,7 @@ extends: Write-Host "- Downloaded wheels from build pipeline" Write-Host "- Verified wheel integrity" Write-Host "- Downloaded symbols from build pipeline" - if ("${{ parameters.publishSymbols }}" -eq "True") { + if ("${{ parameters.publishSymbols }}" -eq "True" -and "${{ parameters.releasePackage }}" -eq "mssql-python") { Write-Host "- Published symbols to SqlClientDrivers org" } Write-Host "" @@ -331,7 +374,8 @@ extends: Write-Host "Target: PyPI" Write-Host "Owners: $(owner)" Write-Host "Approvers: $(approver)" - Write-Host "Symbols Published: ${{ parameters.publishSymbols }}" + $symbolsPublished = ("${{ parameters.publishSymbols }}" -eq "True") -and ("${{ parameters.releasePackage }}" -eq "mssql-python") + Write-Host "Symbols Published: $symbolsPublished" Write-Host "=====================================" Write-Host "" Write-Host "Next steps:" diff --git a/OneBranchPipelines/stages/build-odbc-linux-stage.yml b/OneBranchPipelines/stages/build-odbc-linux-stage.yml index d0a9883cd..969c0b8b3 100644 --- a/OneBranchPipelines/stages/build-odbc-linux-stage.yml +++ b/OneBranchPipelines/stages/build-odbc-linux-stage.yml @@ -136,6 +136,28 @@ stages: cp -v $(Build.SourcesDirectory)/dist/*.whl $(ob_outputDirectory)/wheels/ displayName: 'Stage wheel artifact' + # Guard against the .gitignore '*.so' trap or a silent sync_libs() skip + # shipping a data wheel with NO driver binary (twine check would still pass). + # The driver filename contains 'msodbcsql' on every platform. + - script: | + set -eu + wheel_dir="$(ob_outputDirectory)/wheels" + matches=$(ls "$wheel_dir"/*.whl 2>/dev/null || true) + if [ -z "$matches" ]; then + echo "ERROR: no wheels found to verify in $wheel_dir" >&2 + exit 1 + fi + for whl in $matches; do + echo "Verifying ODBC driver binary present in: $whl" + if python3 -m zipfile -l "$whl" | grep -qi 'msodbcsql'; then + echo "OK: $whl contains the ODBC driver binary" + else + echo "ERROR: ODBC driver binary (msodbcsql*) missing from $whl" >&2 + exit 1 + fi + done + displayName: 'Assert wheel contains ODBC driver binary' + - script: | docker rm -f build-$(LINUX_TAG)-$(ARCH) || true displayName: 'Stop and remove build container' diff --git a/OneBranchPipelines/stages/build-odbc-macos-stage.yml b/OneBranchPipelines/stages/build-odbc-macos-stage.yml index 9b47388e3..c1e7541b1 100644 --- a/OneBranchPipelines/stages/build-odbc-macos-stage.yml +++ b/OneBranchPipelines/stages/build-odbc-macos-stage.yml @@ -79,6 +79,27 @@ stages: TargetFolder: '$(ob_outputDirectory)/wheels' displayName: 'Stage wheel artifact' + # Guard against a data wheel that packaged no driver binary (twine check + # would still pass). The driver filename contains 'msodbcsql' on every platform. + - script: | + set -eu + wheel_dir="$(ob_outputDirectory)/wheels" + matches=$(ls "$wheel_dir"/*.whl 2>/dev/null || true) + if [ -z "$matches" ]; then + echo "ERROR: no wheels found to verify in $wheel_dir" >&2 + exit 1 + fi + for whl in $matches; do + echo "Verifying ODBC driver binary present in: $whl" + if python -m zipfile -l "$whl" | grep -qi 'msodbcsql'; then + echo "OK: $whl contains the ODBC driver binary" + else + echo "ERROR: ODBC driver binary (msodbcsql*) missing from $whl" >&2 + exit 1 + fi + done + displayName: 'Assert wheel contains ODBC driver binary' + - task: PublishPipelineArtifact@1 displayName: 'Publish ODBC macOS Artifact' inputs: diff --git a/OneBranchPipelines/stages/build-odbc-windows-stage.yml b/OneBranchPipelines/stages/build-odbc-windows-stage.yml index 57de838ed..dea809ed0 100644 --- a/OneBranchPipelines/stages/build-odbc-windows-stage.yml +++ b/OneBranchPipelines/stages/build-odbc-windows-stage.yml @@ -92,6 +92,28 @@ stages: TargetFolder: '$(ob_outputDirectory)\wheels' displayName: 'Stage wheel artifact' + # Guard against a data wheel that packaged no driver binary (twine check + # would still pass). The driver DLL name contains 'msodbcsql' on every platform. + - powershell: | + $ErrorActionPreference = "Stop" + $wheels = Get-ChildItem -Path "$(ob_outputDirectory)\wheels" -Filter "*.whl" + if ($wheels.Count -eq 0) { + Write-Error "No wheels found to verify in $(ob_outputDirectory)\wheels" + exit 1 + } + Add-Type -AssemblyName System.IO.Compression.FileSystem + foreach ($whl in $wheels) { + $zip = [System.IO.Compression.ZipFile]::OpenRead($whl.FullName) + $hasDriver = $zip.Entries | Where-Object { $_.FullName -match 'msodbcsql' } + $zip.Dispose() + if (-not $hasDriver) { + Write-Error "ODBC driver binary (msodbcsql*) missing from $($whl.Name)" + exit 1 + } + Write-Host "OK: $($whl.Name) contains the ODBC driver binary" + } + displayName: 'Assert wheel contains ODBC driver binary' + # OneBranch requires artifact naming: drop__. - task: PublishPipelineArtifact@1 displayName: 'Publish ODBC Windows Artifact' From b88e169f006f16bb26f59efc13be907bd11fc93f Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Wed, 8 Jul 2026 14:24:25 +0530 Subject: [PATCH 08/13] Hard-fail mssql-python consolidate on wheel-count mismatch Flip the 34-wheel count check from WARNING-only to a hard exit 1, symmetric with the odbc consolidate job's count guard. A mismatch means a build stage silently produced a partial wheel set (e.g. a missing Python version) or the matrix changed without updating the expected count -- previously this shipped silently. --- OneBranchPipelines/jobs/consolidate-artifacts-job.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/OneBranchPipelines/jobs/consolidate-artifacts-job.yml b/OneBranchPipelines/jobs/consolidate-artifacts-job.yml index 52eb43381..40cda12b0 100644 --- a/OneBranchPipelines/jobs/consolidate-artifacts-job.yml +++ b/OneBranchPipelines/jobs/consolidate-artifacts-job.yml @@ -77,8 +77,13 @@ jobs: echo "Total wheel count: $WHEEL_COUNT" echo "Expected: 34 wheels (9 Windows + 5 macOS + 20 Linux)" + # Hard-fail on a wheel-count mismatch (symmetric with the odbc consolidate job). + # A mismatch means a build stage silently produced a partial set (e.g. a missing + # Python version) or the build matrix changed without updating this count. + # Update the expected 34 whenever the Windows/macOS/Linux build matrix changes. if [ "$WHEEL_COUNT" -ne 34 ]; then - echo "WARNING: Expected 34 wheels but found $WHEEL_COUNT" + echo "ERROR: Expected 34 wheels but found $WHEEL_COUNT" >&2 + exit 1 else echo "SUCCESS: All 34 wheels consolidated!" fi From d1b3d467028a4986a3558ce4653a002322484fff Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Fri, 10 Jul 2026 14:04:13 +0530 Subject: [PATCH 09/13] odbc: add mssql-python-odbc selector to dummy release pipeline Mirrors the official-release-pipeline odbc changes: releasePackage parameter, consolidatedArtifactName selection for both artifact downloads, and gating the mssql-py-core version validation + symbol publishing to mssql-python only. Stays on Maven ContentType so it never publishes to PyPI. --- OneBranchPipelines/dummy-release-pipeline.yml | 48 ++++++++++++++----- 1 file changed, 35 insertions(+), 13 deletions(-) diff --git a/OneBranchPipelines/dummy-release-pipeline.yml b/OneBranchPipelines/dummy-release-pipeline.yml index f9e548fb0..28751e845 100644 --- a/OneBranchPipelines/dummy-release-pipeline.yml +++ b/OneBranchPipelines/dummy-release-pipeline.yml @@ -12,6 +12,16 @@ pr: none # Parameters for DUMMY release pipeline parameters: + # Which package to test-release. Both are produced by the same build pipeline + # (definition 2199); this switches the consolidated artifact and messaging. + - name: releasePackage + displayName: '[TEST] Package to Release' + type: string + values: + - 'mssql-python' + - 'mssql-python-odbc' + default: 'mssql-python' + - name: publishSymbols displayName: '[TEST] Publish Symbols to Symbol Servers' type: boolean @@ -32,6 +42,16 @@ variables: - group: 'ESRP Federated Creds (AME)' # Contains ESRP signing credentials - group: 'Symbols Publishing' # Contains SymbolServer, SymbolTokenUri variables + # Select which consolidated artifact to download based on the target package. + # Both are produced by the same build pipeline (definition 2199): + # mssql-python -> drop_Consolidate_ConsolidateArtifacts + # mssql-python-odbc -> drop_ConsolidateOdbc_ConsolidateArtifacts + - name: consolidatedArtifactName + ${{ if eq(parameters.releasePackage, 'mssql-python-odbc') }}: + value: 'drop_ConsolidateOdbc_ConsolidateArtifacts' + ${{ else }}: + value: 'drop_Consolidate_ConsolidateArtifacts' + # OneBranch resources resources: repositories: @@ -114,7 +134,7 @@ extends: definition: 2199 # Build-Release-Package-Pipeline definition ID buildVersionToDownload: 'specific' buildId: $(resources.pipeline.buildPipeline.runID) # Use the build run selected in UI - artifactName: 'drop_Consolidate_ConsolidateArtifacts' # Consolidated artifact with dist/ and symbols/ + artifactName: '${{ variables.consolidatedArtifactName }}' # mssql-python or mssql-python-odbc consolidated wheels targetPath: '$(Build.SourcesDirectory)/artifacts' # Step 2: List downloaded artifacts for verification @@ -175,13 +195,15 @@ extends: Write-Host "Symbols: $(if ($symbols) { $symbols.Count } else { 0 }) files" Write-Host "=====================================" - # Step 3: Validate mssql-py-core is a stable version (no dev/alpha/beta/rc) - - task: PowerShell@2 - displayName: '[TEST] Validate mssql-py-core is a stable version' - inputs: - targetType: 'filePath' - filePath: '$(Build.SourcesDirectory)\OneBranchPipelines\scripts\validate-release-versions.ps1' - arguments: '-VersionFile "$(Build.SourcesDirectory)\artifacts\dist\mssql-py-core.version"' + # Step 3: Validate mssql-py-core is a stable version (mssql-python only; + # the mssql-python-odbc package ships no mssql-py-core.version file) + - ${{ if eq(parameters.releasePackage, 'mssql-python') }}: + - task: PowerShell@2 + displayName: '[TEST] Validate mssql-py-core is a stable version' + inputs: + targetType: 'filePath' + filePath: '$(Build.SourcesDirectory)\OneBranchPipelines\scripts\validate-release-versions.ps1' + arguments: '-VersionFile "$(Build.SourcesDirectory)\artifacts\dist\mssql-py-core.version"' # Step 4: Verify wheel integrity - task: PowerShell@2 @@ -217,8 +239,8 @@ extends: Write-Host "`nAll wheels verified successfully!" - # Step 5: Publish Symbols (if enabled and symbols exist) - - ${{ if eq(parameters.publishSymbols, true) }}: + # Step 5: Publish Symbols (if enabled and symbols exist; mssql-python only) + - ${{ if and(eq(parameters.publishSymbols, true), eq(parameters.releasePackage, 'mssql-python')) }}: - template: /OneBranchPipelines/steps/symbol-publishing-step.yml@self parameters: SymbolsFolder: '$(Build.SourcesDirectory)/symbols' @@ -233,7 +255,7 @@ extends: Write-Host "====================================" Write-Host "⚠️ TEST PIPELINE - DRY RUN MODE ⚠️" Write-Host "====================================" - Write-Host "Package: mssql-python (TEST)" + Write-Host "Package: ${{ parameters.releasePackage }} (TEST)" Write-Host "" Write-Host "Actions performed:" Write-Host "✓ Downloaded wheels from build pipeline" @@ -280,7 +302,7 @@ extends: definition: 2199 buildVersionToDownload: 'specific' buildId: $(resources.pipeline.buildPipeline.runID) - artifactName: 'drop_Consolidate_ConsolidateArtifacts' + artifactName: '${{ variables.consolidatedArtifactName }}' targetPath: '$(Build.SourcesDirectory)/artifacts' # ⚠️ IMPORTANT: Uses Maven ContentType for testing - NOT PyPI! @@ -313,7 +335,7 @@ extends: Write-Host "====================================" Write-Host "⚠️ TEST PIPELINE - DUMMY RELEASE COMPLETED ⚠️" Write-Host "====================================" - Write-Host "Package: mssql-python (TEST)" + Write-Host "Package: ${{ parameters.releasePackage }} (TEST)" Write-Host "ContentType: Maven (NOT PyPI - Safe for Testing)" Write-Host "Owners: $(owner)" Write-Host "Approvers: $(approver)" From 9f93c36073dabd47297265c91527f26efcee4bcf Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Fri, 10 Jul 2026 15:00:10 +0530 Subject: [PATCH 10/13] odbc: align dummy release pipeline mssql-python handling with official Adds the mssql-python-odbc pin guard (gated to mssql-python, matching official-release-pipeline) and gates the 'symbols published' summary messaging on releasePackage==mssql-python so the dummy pipeline exercises the exact same mssql-python validation/messaging as the official pipeline for both packages. --- OneBranchPipelines/dummy-release-pipeline.yml | 51 +++++++++++++++++-- 1 file changed, 47 insertions(+), 4 deletions(-) diff --git a/OneBranchPipelines/dummy-release-pipeline.yml b/OneBranchPipelines/dummy-release-pipeline.yml index 28751e845..c98c351ff 100644 --- a/OneBranchPipelines/dummy-release-pipeline.yml +++ b/OneBranchPipelines/dummy-release-pipeline.yml @@ -196,7 +196,7 @@ extends: Write-Host "=====================================" # Step 3: Validate mssql-py-core is a stable version (mssql-python only; - # the mssql-python-odbc package ships no mssql-py-core.version file) + # mssql-python-odbc ships no mssql-py-core.version file) - ${{ if eq(parameters.releasePackage, 'mssql-python') }}: - task: PowerShell@2 displayName: '[TEST] Validate mssql-py-core is a stable version' @@ -205,6 +205,49 @@ extends: filePath: '$(Build.SourcesDirectory)\OneBranchPipelines\scripts\validate-release-versions.ps1' arguments: '-VersionFile "$(Build.SourcesDirectory)\artifacts\dist\mssql-py-core.version"' + # Guard: the released mssql-python wheel MUST depend on the pinned + # mssql-python-odbc package. Catches releasing from a PRE-pin build + # (e.g. the wrong build run), which would ship a driver-less package + # that breaks 'pip install mssql-python'. By design this FAILS on a + # pre-pin build. Update $expectedVersion when the odbc pin bumps. + - task: PowerShell@2 + displayName: '[TEST] Validate mssql-python-odbc pin in wheel metadata' + inputs: + targetType: 'inline' + script: | + $ErrorActionPreference = 'Stop' + $expectedVersion = '18.6.0' + $wheels = Get-ChildItem -Path "$(Build.SourcesDirectory)/artifacts/dist" -Filter "mssql_python-*.whl" + if ($wheels.Count -eq 0) { + Write-Error "No mssql_python-*.whl found in artifacts/dist to validate the odbc pin." + exit 1 + } + Add-Type -AssemblyName System.IO.Compression.FileSystem + $wheel = $wheels[0].FullName + Write-Host "Inspecting wheel metadata: $wheel" + $zip = [System.IO.Compression.ZipFile]::OpenRead($wheel) + $metaEntry = $zip.Entries | Where-Object { $_.FullName -match '\.dist-info/METADATA$' } | Select-Object -First 1 + if (-not $metaEntry) { + $zip.Dispose() + Write-Error "METADATA not found inside $wheel" + exit 1 + } + $reader = New-Object System.IO.StreamReader($metaEntry.Open()) + $metadata = $reader.ReadToEnd() + $reader.Dispose() + $zip.Dispose() + $odbcLines = ($metadata -split "`n") | Where-Object { $_ -match 'Requires-Dist:\s*mssql[-_]python[-_]odbc' } + if ($odbcLines) { + Write-Host "odbc dependency in metadata:" + $odbcLines | ForEach-Object { Write-Host " $($_.Trim())" } + } + $pinRegex = 'Requires-Dist:\s*mssql[-_]python[-_]odbc\s*==\s*' + [regex]::Escape($expectedVersion) + if ($metadata -notmatch $pinRegex) { + Write-Error "mssql-python wheel does not pin mssql-python-odbc==$expectedVersion. Are you releasing from the POST-pin build? mssql-python must depend on the pinned odbc package before release." + exit 1 + } + Write-Host "OK: wheel depends on mssql-python-odbc==$expectedVersion" + # Step 4: Verify wheel integrity - task: PowerShell@2 displayName: '[TEST] Verify Wheel Integrity' @@ -261,7 +304,7 @@ extends: Write-Host "✓ Downloaded wheels from build pipeline" Write-Host "✓ Verified wheel integrity" Write-Host "✓ Downloaded symbols from build pipeline" - if ("${{ parameters.publishSymbols }}" -eq "True") { + if ("${{ parameters.publishSymbols }}" -eq "True" -and "${{ parameters.releasePackage }}" -eq "mssql-python") { Write-Host "✓ Published symbols to SqlClientDrivers org" } Write-Host "✗ ESRP dummy release NOT performed (parameter disabled)" @@ -339,7 +382,7 @@ extends: Write-Host "ContentType: Maven (NOT PyPI - Safe for Testing)" Write-Host "Owners: $(owner)" Write-Host "Approvers: $(approver)" - Write-Host "Symbols Published: ${{ parameters.publishSymbols }}" + Write-Host "Symbols Published: $(if ('${{ parameters.publishSymbols }}' -eq 'True' -and '${{ parameters.releasePackage }}' -eq 'mssql-python') { 'True' } else { 'False' })" Write-Host "=====================================" Write-Host "" Write-Host "⚠️ IMPORTANT: This was a DUMMY release using Maven ContentType" @@ -348,7 +391,7 @@ extends: Write-Host "What was tested:" Write-Host "✓ Artifact download from build pipeline" Write-Host "✓ Wheel integrity verification" - if ("${{ parameters.publishSymbols }}" -eq "True") { + if ("${{ parameters.publishSymbols }}" -eq "True" -and "${{ parameters.releasePackage }}" -eq "mssql-python") { Write-Host "✓ Symbol publishing to SqlClientDrivers org" } Write-Host "✓ ESRP release workflow (Maven ContentType)" From 2d93fe07234bf24f30353fd7e56cd6f754a437f9 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Fri, 10 Jul 2026 15:28:24 +0530 Subject: [PATCH 11/13] odbc: bump mssql-python-odbc pin to 18.6.2 in release pipelines --- OneBranchPipelines/dummy-release-pipeline.yml | 2 +- OneBranchPipelines/official-release-pipeline.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/OneBranchPipelines/dummy-release-pipeline.yml b/OneBranchPipelines/dummy-release-pipeline.yml index c98c351ff..6cbb7c44b 100644 --- a/OneBranchPipelines/dummy-release-pipeline.yml +++ b/OneBranchPipelines/dummy-release-pipeline.yml @@ -216,7 +216,7 @@ extends: targetType: 'inline' script: | $ErrorActionPreference = 'Stop' - $expectedVersion = '18.6.0' + $expectedVersion = '18.6.2' $wheels = Get-ChildItem -Path "$(Build.SourcesDirectory)/artifacts/dist" -Filter "mssql_python-*.whl" if ($wheels.Count -eq 0) { Write-Error "No mssql_python-*.whl found in artifacts/dist to validate the odbc pin." diff --git a/OneBranchPipelines/official-release-pipeline.yml b/OneBranchPipelines/official-release-pipeline.yml index 7cb529796..8937fd0f6 100644 --- a/OneBranchPipelines/official-release-pipeline.yml +++ b/OneBranchPipelines/official-release-pipeline.yml @@ -216,7 +216,7 @@ extends: targetType: 'inline' script: | $ErrorActionPreference = 'Stop' - $expectedVersion = '18.6.0' + $expectedVersion = '18.6.2' $wheels = Get-ChildItem -Path "$(Build.SourcesDirectory)/artifacts/dist" -Filter "mssql_python-*.whl" if ($wheels.Count -eq 0) { Write-Error "No mssql_python-*.whl found in artifacts/dist to validate the odbc pin." From a4e39687fc1930f1c35f17367bb1fe6391e1fe93 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Fri, 10 Jul 2026 15:17:36 +0530 Subject: [PATCH 12/13] build: add buildPackage selector to build pipeline Gate mssql-python and mssql-python-odbc stage groups behind a buildPackage parameter (mssql-python default) so only the selected package builds, halving run time and enabling concurrent per-package builds. Mirrors the releasePackage selector pattern in the release pipelines. --- .../build-release-package-pipeline.yml | 216 ++++++++++-------- 1 file changed, 121 insertions(+), 95 deletions(-) diff --git a/OneBranchPipelines/build-release-package-pipeline.yml b/OneBranchPipelines/build-release-package-pipeline.yml index 984e52bcc..29e735080 100644 --- a/OneBranchPipelines/build-release-package-pipeline.yml +++ b/OneBranchPipelines/build-release-package-pipeline.yml @@ -69,6 +69,22 @@ parameters: displayName: 'Run SDL Security Tasks' type: boolean default: true + + # Which package to build. Both are produced from this same pipeline but are + # fully independent stage groups, so building only one halves the run time and + # lets you run two builds (one per package) concurrently. + # mssql-python -> Windows/macOS/Linux wheels + Consolidate + # mssql-python-odbc -> ODBC data wheels + ConsolidateOdbc + # both -> everything (previous default behaviour) + # Scheduled (daily) builds use this default too, so they build mssql-python only. + - name: buildPackage + displayName: 'Package to Build' + type: string + values: + - 'mssql-python' + - 'mssql-python-odbc' + - 'both' + default: 'mssql-python' # ========================= # PLATFORM CONFIGURATIONS @@ -384,16 +400,18 @@ extends: # 5. Builds wheel # 6. Publishes artifacts (wheels + PYD + PDB) # 7. ESRP malware scanning - - ${{ each config in parameters.windowsConfigs }}: - - template: /OneBranchPipelines/stages/build-windows-single-stage.yml@self - parameters: - stageName: Win_py${{ config.pyVer }}_${{ config.arch }} - jobName: BuildWheel - # Convert pyVer '310' → pythonVersion '3.10' - pythonVersion: ${{ format('{0}.{1}', substring(config.pyVer, 0, 1), substring(config.pyVer, 1, 2)) }} - shortPyVer: ${{ config.pyVer }} - architecture: ${{ config.arch }} - oneBranchType: '${{ variables.effectiveOneBranchType }}' + # Gate: build mssql-python stages unless an odbc-only build was requested + - ${{ if ne(parameters.buildPackage, 'mssql-python-odbc') }}: + - ${{ each config in parameters.windowsConfigs }}: + - template: /OneBranchPipelines/stages/build-windows-single-stage.yml@self + parameters: + stageName: Win_py${{ config.pyVer }}_${{ config.arch }} + jobName: BuildWheel + # Convert pyVer '310' → pythonVersion '3.10' + pythonVersion: ${{ format('{0}.{1}', substring(config.pyVer, 0, 1), substring(config.pyVer, 1, 2)) }} + shortPyVer: ${{ config.pyVer }} + architecture: ${{ config.arch }} + oneBranchType: '${{ variables.effectiveOneBranchType }}' # ========================= # MACOS BUILD STAGES @@ -410,15 +428,16 @@ extends: # 6. Builds wheel # 7. Publishes artifacts (wheels + .so) # 8. ESRP malware scanning - - ${{ each config in parameters.macosConfigs }}: - - template: /OneBranchPipelines/stages/build-macos-single-stage.yml@self - parameters: - stageName: MacOS_py${{ config.pyVer }} - jobName: BuildWheel - # Convert pyVer '310' → pythonVersion '3.10' - pythonVersion: ${{ format('{0}.{1}', substring(config.pyVer, 0, 1), substring(config.pyVer, 1, 2)) }} - shortPyVer: ${{ config.pyVer }} - oneBranchType: '${{ variables.effectiveOneBranchType }}' + - ${{ if ne(parameters.buildPackage, 'mssql-python-odbc') }}: + - ${{ each config in parameters.macosConfigs }}: + - template: /OneBranchPipelines/stages/build-macos-single-stage.yml@self + parameters: + stageName: MacOS_py${{ config.pyVer }} + jobName: BuildWheel + # Convert pyVer '310' → pythonVersion '3.10' + pythonVersion: ${{ format('{0}.{1}', substring(config.pyVer, 0, 1), substring(config.pyVer, 1, 2)) }} + shortPyVer: ${{ config.pyVer }} + oneBranchType: '${{ variables.effectiveOneBranchType }}' # ========================= # LINUX BUILD STAGES @@ -440,15 +459,16 @@ extends: # d. Runs pytest against SQL Server # 4. Publishes artifacts (all 5 wheels) # 5. Component Governance + AntiMalware scanning - - ${{ each config in parameters.linuxConfigs }}: - - template: /OneBranchPipelines/stages/build-linux-single-stage.yml@self - parameters: - stageName: Linux_${{ config.tag }}_${{ config.arch }} - jobName: BuildWheels - linuxTag: ${{ config.tag }} - arch: ${{ config.arch }} - dockerPlatform: ${{ config.platform }} - oneBranchType: '${{ variables.effectiveOneBranchType }}' + - ${{ if ne(parameters.buildPackage, 'mssql-python-odbc') }}: + - ${{ each config in parameters.linuxConfigs }}: + - template: /OneBranchPipelines/stages/build-linux-single-stage.yml@self + parameters: + stageName: Linux_${{ config.tag }}_${{ config.arch }} + jobName: BuildWheels + linuxTag: ${{ config.tag }} + arch: ${{ config.arch }} + dockerPlatform: ${{ config.platform }} + oneBranchType: '${{ variables.effectiveOneBranchType }}' # ========================= # CONSOLIDATE STAGE @@ -462,36 +482,37 @@ extends: # - dist/bindings/macOS/*.so (macOS universal2 binaries) # - dist/bindings/Linux/*.so (Linux native extensions) # This stage also runs final BinSkim scan on all binaries - - stage: Consolidate - displayName: 'Consolidate All Artifacts' - dependsOn: - # Windows dependencies (9 stages) - - Win_py310_x64 - - Win_py311_x64 - - Win_py312_x64 - - Win_py313_x64 - - Win_py314_x64 - - Win_py311_arm64 - - Win_py312_arm64 - - Win_py313_arm64 - - Win_py314_arm64 - # macOS dependencies (5 stages) - - MacOS_py310 - - MacOS_py311 - - MacOS_py312 - - MacOS_py313 - - MacOS_py314 - # Linux dependencies (4 stages) - - Linux_manylinux_2_28_x86_64 - - Linux_manylinux_2_28_aarch64 - - Linux_musllinux_x86_64 - - Linux_musllinux_aarch64 - jobs: - - template: /OneBranchPipelines/jobs/consolidate-artifacts-job.yml@self - parameters: - # CRITICAL: Use effectiveOneBranchType to ensure scheduled builds run as 'Official' - # Using parameters.oneBranchType would break scheduled builds (they'd run as 'NonOfficial') - oneBranchType: '${{ variables.effectiveOneBranchType }}' + - ${{ if ne(parameters.buildPackage, 'mssql-python-odbc') }}: + - stage: Consolidate + displayName: 'Consolidate All Artifacts' + dependsOn: + # Windows dependencies (9 stages) + - Win_py310_x64 + - Win_py311_x64 + - Win_py312_x64 + - Win_py313_x64 + - Win_py314_x64 + - Win_py311_arm64 + - Win_py312_arm64 + - Win_py313_arm64 + - Win_py314_arm64 + # macOS dependencies (5 stages) + - MacOS_py310 + - MacOS_py311 + - MacOS_py312 + - MacOS_py313 + - MacOS_py314 + # Linux dependencies (4 stages) + - Linux_manylinux_2_28_x86_64 + - Linux_manylinux_2_28_aarch64 + - Linux_musllinux_x86_64 + - Linux_musllinux_aarch64 + jobs: + - template: /OneBranchPipelines/jobs/consolidate-artifacts-job.yml@self + parameters: + # CRITICAL: Use effectiveOneBranchType to ensure scheduled builds run as 'Official' + # Using parameters.oneBranchType would break scheduled builds (they'd run as 'NonOfficial') + oneBranchType: '${{ variables.effectiveOneBranchType }}' # Note: Symbol publishing handled directly in Windows build stages # PDB files uploaded to Microsoft Symbol Server for debugging @@ -507,32 +528,36 @@ extends: # first-party native code is introduced. # # ODBC Windows stages (2): ODBC_Win_x64, ODBC_Win_arm64 - - ${{ each config in parameters.odbcWindowsConfigs }}: - - template: /OneBranchPipelines/stages/build-odbc-windows-stage.yml@self - parameters: - stageName: ODBC_Win_${{ config.arch }} - jobName: BuildWheel - architecture: ${{ config.arch }} - oneBranchType: '${{ variables.effectiveOneBranchType }}' + # Gate: build mssql-python-odbc stages unless an mssql-python-only build was requested + - ${{ if ne(parameters.buildPackage, 'mssql-python') }}: + - ${{ each config in parameters.odbcWindowsConfigs }}: + - template: /OneBranchPipelines/stages/build-odbc-windows-stage.yml@self + parameters: + stageName: ODBC_Win_${{ config.arch }} + jobName: BuildWheel + architecture: ${{ config.arch }} + oneBranchType: '${{ variables.effectiveOneBranchType }}' # ODBC macOS stage (1): ODBC_MacOS_universal2 - - ${{ each config in parameters.odbcMacosConfigs }}: - - template: /OneBranchPipelines/stages/build-odbc-macos-stage.yml@self - parameters: - stageName: ODBC_MacOS_${{ config.name }} - jobName: BuildWheel - oneBranchType: '${{ variables.effectiveOneBranchType }}' + - ${{ if ne(parameters.buildPackage, 'mssql-python') }}: + - ${{ each config in parameters.odbcMacosConfigs }}: + - template: /OneBranchPipelines/stages/build-odbc-macos-stage.yml@self + parameters: + stageName: ODBC_MacOS_${{ config.name }} + jobName: BuildWheel + oneBranchType: '${{ variables.effectiveOneBranchType }}' # ODBC Linux stages (4): manylinux_2_28 x86_64/aarch64, musllinux x86_64/aarch64 - - ${{ each config in parameters.odbcLinuxConfigs }}: - - template: /OneBranchPipelines/stages/build-odbc-linux-stage.yml@self - parameters: - stageName: ODBC_Linux_${{ config.tag }}_${{ config.arch }} - jobName: BuildWheel - linuxTag: ${{ config.tag }} - arch: ${{ config.arch }} - dockerPlatform: ${{ config.platform }} - oneBranchType: '${{ variables.effectiveOneBranchType }}' + - ${{ if ne(parameters.buildPackage, 'mssql-python') }}: + - ${{ each config in parameters.odbcLinuxConfigs }}: + - template: /OneBranchPipelines/stages/build-odbc-linux-stage.yml@self + parameters: + stageName: ODBC_Linux_${{ config.tag }}_${{ config.arch }} + jobName: BuildWheel + linuxTag: ${{ config.tag }} + arch: ${{ config.arch }} + dockerPlatform: ${{ config.platform }} + oneBranchType: '${{ variables.effectiveOneBranchType }}' # ========================================================================= # CONSOLIDATE ODBC STAGE @@ -541,18 +566,19 @@ extends: # publishes them as `drop_ConsolidateOdbc_ConsolidateArtifacts` (distinct from # the mssql-python `drop_Consolidate_ConsolidateArtifacts`). The release # pipeline selects one artifact based on its `releasePackage` parameter. - - stage: ConsolidateOdbc - displayName: 'Consolidate All ODBC Artifacts' - dependsOn: - - ODBC_Win_x64 - - ODBC_Win_arm64 - - ODBC_MacOS_universal2 - - ODBC_Linux_manylinux_2_28_x86_64 - - ODBC_Linux_manylinux_2_28_aarch64 - - ODBC_Linux_musllinux_x86_64 - - ODBC_Linux_musllinux_aarch64 - jobs: - - template: /OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml@self - parameters: - oneBranchType: '${{ variables.effectiveOneBranchType }}' - expectedWheelCount: 7 + - ${{ if ne(parameters.buildPackage, 'mssql-python') }}: + - stage: ConsolidateOdbc + displayName: 'Consolidate All ODBC Artifacts' + dependsOn: + - ODBC_Win_x64 + - ODBC_Win_arm64 + - ODBC_MacOS_universal2 + - ODBC_Linux_manylinux_2_28_x86_64 + - ODBC_Linux_manylinux_2_28_aarch64 + - ODBC_Linux_musllinux_x86_64 + - ODBC_Linux_musllinux_aarch64 + jobs: + - template: /OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml@self + parameters: + oneBranchType: '${{ variables.effectiveOneBranchType }}' + expectedWheelCount: 7 From 9d3097c8331d5caed76a8ade8a2d28baded2bdba Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Fri, 10 Jul 2026 15:21:13 +0530 Subject: [PATCH 13/13] build: nightly build always builds both packages Add effectiveBuildPackage variable that forces 'both' on scheduled (Build.Reason=Schedule) runs while manual/PR runs honour the buildPackage parameter. Mirrors the effectiveOneBranchType pattern so the nightly keeps refreshing every artifact. --- .../build-release-package-pipeline.yml | 27 ++++++++++++------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/OneBranchPipelines/build-release-package-pipeline.yml b/OneBranchPipelines/build-release-package-pipeline.yml index 29e735080..c485cc8ee 100644 --- a/OneBranchPipelines/build-release-package-pipeline.yml +++ b/OneBranchPipelines/build-release-package-pipeline.yml @@ -76,7 +76,8 @@ parameters: # mssql-python -> Windows/macOS/Linux wheels + Consolidate # mssql-python-odbc -> ODBC data wheels + ConsolidateOdbc # both -> everything (previous default behaviour) - # Scheduled (daily) builds use this default too, so they build mssql-python only. + # Scheduled (daily) builds ignore this and always build 'both' (see + # effectiveBuildPackage below) so the nightly keeps producing every artifact. - name: buildPackage displayName: 'Package to Build' type: string @@ -192,6 +193,14 @@ variables: value: 'Official' ${{ else }}: value: '${{ parameters.oneBranchType }}' + + # Determine which package(s) to build: scheduled (nightly) builds always build + # 'both' so every artifact keeps refreshing; manual/PR builds honour the parameter. + - name: effectiveBuildPackage + ${{ if eq(variables['Build.Reason'], 'Schedule') }}: + value: 'both' + ${{ else }}: + value: '${{ parameters.buildPackage }}' # Variable template imports # Each file provides specific variable groups: @@ -401,7 +410,7 @@ extends: # 6. Publishes artifacts (wheels + PYD + PDB) # 7. ESRP malware scanning # Gate: build mssql-python stages unless an odbc-only build was requested - - ${{ if ne(parameters.buildPackage, 'mssql-python-odbc') }}: + - ${{ if ne(variables.effectiveBuildPackage, 'mssql-python-odbc') }}: - ${{ each config in parameters.windowsConfigs }}: - template: /OneBranchPipelines/stages/build-windows-single-stage.yml@self parameters: @@ -428,7 +437,7 @@ extends: # 6. Builds wheel # 7. Publishes artifacts (wheels + .so) # 8. ESRP malware scanning - - ${{ if ne(parameters.buildPackage, 'mssql-python-odbc') }}: + - ${{ if ne(variables.effectiveBuildPackage, 'mssql-python-odbc') }}: - ${{ each config in parameters.macosConfigs }}: - template: /OneBranchPipelines/stages/build-macos-single-stage.yml@self parameters: @@ -459,7 +468,7 @@ extends: # d. Runs pytest against SQL Server # 4. Publishes artifacts (all 5 wheels) # 5. Component Governance + AntiMalware scanning - - ${{ if ne(parameters.buildPackage, 'mssql-python-odbc') }}: + - ${{ if ne(variables.effectiveBuildPackage, 'mssql-python-odbc') }}: - ${{ each config in parameters.linuxConfigs }}: - template: /OneBranchPipelines/stages/build-linux-single-stage.yml@self parameters: @@ -482,7 +491,7 @@ extends: # - dist/bindings/macOS/*.so (macOS universal2 binaries) # - dist/bindings/Linux/*.so (Linux native extensions) # This stage also runs final BinSkim scan on all binaries - - ${{ if ne(parameters.buildPackage, 'mssql-python-odbc') }}: + - ${{ if ne(variables.effectiveBuildPackage, 'mssql-python-odbc') }}: - stage: Consolidate displayName: 'Consolidate All Artifacts' dependsOn: @@ -529,7 +538,7 @@ extends: # # ODBC Windows stages (2): ODBC_Win_x64, ODBC_Win_arm64 # Gate: build mssql-python-odbc stages unless an mssql-python-only build was requested - - ${{ if ne(parameters.buildPackage, 'mssql-python') }}: + - ${{ if ne(variables.effectiveBuildPackage, 'mssql-python') }}: - ${{ each config in parameters.odbcWindowsConfigs }}: - template: /OneBranchPipelines/stages/build-odbc-windows-stage.yml@self parameters: @@ -539,7 +548,7 @@ extends: oneBranchType: '${{ variables.effectiveOneBranchType }}' # ODBC macOS stage (1): ODBC_MacOS_universal2 - - ${{ if ne(parameters.buildPackage, 'mssql-python') }}: + - ${{ if ne(variables.effectiveBuildPackage, 'mssql-python') }}: - ${{ each config in parameters.odbcMacosConfigs }}: - template: /OneBranchPipelines/stages/build-odbc-macos-stage.yml@self parameters: @@ -548,7 +557,7 @@ extends: oneBranchType: '${{ variables.effectiveOneBranchType }}' # ODBC Linux stages (4): manylinux_2_28 x86_64/aarch64, musllinux x86_64/aarch64 - - ${{ if ne(parameters.buildPackage, 'mssql-python') }}: + - ${{ if ne(variables.effectiveBuildPackage, 'mssql-python') }}: - ${{ each config in parameters.odbcLinuxConfigs }}: - template: /OneBranchPipelines/stages/build-odbc-linux-stage.yml@self parameters: @@ -566,7 +575,7 @@ extends: # publishes them as `drop_ConsolidateOdbc_ConsolidateArtifacts` (distinct from # the mssql-python `drop_Consolidate_ConsolidateArtifacts`). The release # pipeline selects one artifact based on its `releasePackage` parameter. - - ${{ if ne(parameters.buildPackage, 'mssql-python') }}: + - ${{ if ne(variables.effectiveBuildPackage, 'mssql-python') }}: - stage: ConsolidateOdbc displayName: 'Consolidate All ODBC Artifacts' dependsOn: