hero123457/uhd
1
0
Fork 0
mirror of https://github.com/EttusResearch/uhd.git synced 2026-10-25 17:36:10 +00:00
Code Issues Projects Releases Wiki Activity
d1be63490e
uhd/.ci/uhd-build-docker-container.yml
Marian Koop 6efbd7986b ci: docker: Re-add python 3.13 targets
2026-08-20 02:03:09 +02:00

483 lines
20 KiB
YAML
Raw Blame History

#
# Copyright 2021 Ettus Research, a National Instruments Brand
#
# SPDX-License-Identifier: LGPL-3.0-or-later
#
trigger:
branches:
include:
- master
paths:
include:
- .ci/docker
- .ci/uhd-build-docker-container.yml
pr:
branches:
include:
- master
paths:
include:
- .ci/docker
- .ci/uhd-build-docker-container.yml
schedules:
- cron: "0 18 * * SAT"
displayName: Weekly Docker Build master branch
branches:
include:
- master
always: true
parameters:
- name: build_linux
type: boolean
displayName: Build Linux images
default: true
- name: build_windows
type: boolean
displayName: Build Windows images
default: true
- name: prune_docker_images
type: boolean
displayName: Prune dangling Docker images before Windows build
default: false
- name: use_vcpkg_cache
type: boolean
displayName: Use vcpkg binary cache for Windows build
default: true
variables:
- template: uhd-pipeline-vars.yml
- ${{ if parameters.build_linux }}:
- name: linuxDockerImages
value:
"{'Fedora-42-builder':{'buildOSName':'fedora42','dockerImageName':'uhd-builder-fedora42','releaseName':'42'},
'Fedora-43-builder':{'buildOSName':'fedora43','dockerImageName':'uhd-builder-fedora43','releaseName':'43'},
'Fedora-44-builder':{'buildOSName':'fedora44','dockerImageName':'uhd-builder-fedora44','releaseName':'44'},
'Ubuntu-2004-builder':{'buildOSName':'ubuntu2004','dockerImageName':'uhd-builder-ubuntu2004','releaseName':'focal'},
'Ubuntu-2204-builder':{'buildOSName':'ubuntu2204','dockerImageName':'uhd-builder-ubuntu2204','releaseName':'jammy'},
'Ubuntu-2404-builder':{'buildOSName':'ubuntu2404','dockerImageName':'uhd-builder-ubuntu2404','releaseName':'noble'},
'Ubuntu-2510-builder':{'buildOSName':'ubuntu2510','dockerImageName':'uhd-builder-ubuntu2510','releaseName':'questing'},
'Ubuntu-2604-builder':{'buildOSName':'ubuntu2604','dockerImageName':'uhd-builder-ubuntu2604','releaseName':'resolute'}}"
- ${{ else }}:
- name: linuxDockerImages
value:
- ${{ if parameters.build_windows }}:
- name: winDockerImages
value:
"{'VS2022-x64-py310-builder':{'buildOSName':'win-vs22-v143-x64-py310','dockerImageName':'uhd-builder-vs2022-v143-x64-py310','cmakeCompiler':'Visual Studio 17 2022','cmakeArch':'x64','vsArch':'x64','vsYear':'2022','pythonVersion':'python310'},
'VS2022-x64-py312-builder':{'buildOSName':'win-vs22-v143-x64-py312','dockerImageName':'uhd-builder-vs2022-v143-x64-py312','cmakeCompiler':'Visual Studio 17 2022','cmakeArch':'x64','vsArch':'x64','vsYear':'2022','pythonVersion':'python312'},
'VS2022-x64-py313-builder':{'buildOSName':'win-vs22-v143-x64-py313','dockerImageName':'uhd-builder-vs2022-v143-x64-py313','cmakeCompiler':'Visual Studio 17 2022','cmakeArch':'x64','vsArch':'x64','vsYear':'2022','pythonVersion':'python313'},
'VS2026-x64-py310-builder':{'buildOSName':'win-vs26-v145-x64-py310','dockerImageName':'uhd-builder-vs2026-v145-x64-py310','cmakeCompiler':'Visual Studio 18 2026','cmakeArch':'x64','vsArch':'x64','vsYear':'2026','pythonVersion':'python310'},
'VS2026-x64-py312-builder':{'buildOSName':'win-vs26-v145-x64-py312','dockerImageName':'uhd-builder-vs2026-v145-x64-py312','cmakeCompiler':'Visual Studio 18 2026','cmakeArch':'x64','vsArch':'x64','vsYear':'2026','pythonVersion':'python312'}}"
# temporarily exclude py313 Windows builder
# 'VS2026-x64-py313-builder':{'buildOSName':'win-vs26-v145-x64-py313','dockerImageName':'uhd-builder-vs2026-v145-x64-py313','cmakeCompiler':'Visual Studio 18 2026','cmakeArch':'x64','vsArch':'x64','vsYear':'2026','pythonVersion':'python313'}
- name: python310_version
value: '3.10.11'
- name: python312_version
value: '3.12.10'
- name: python313_version
value: '3.13.14'
- ${{ else }}:
- name: winDockerImages
value:
- name: macOSBuilders
value:
"{'macOS-15-x64':{'buildOSName':'macOS-15-x64', 'macOSVersionDemand':'15', 'macOSArchDemand': 'X64'}}"
- name: winDockerImagesPythonBuild
# Match with one of the images in winDockerImages
value:
"{}"
- ${{ if or(eq(variables['Build.SourceBranch'], 'refs/heads/master'), startsWith(variables['Build.SourceBranch'], 'refs/heads/UHD-')) }}:
- name: dockerImagePrefix
value: 'rnd-docker-ci/ni/rfsdr/'
- ${{ else }}:
- name: dockerImagePrefix
value: 'rnd-docker-pre/ni/rfsdr/'
- ${{ if or(eq(variables['Build.Reason'], 'PullRequest'), eq(variables['Build.Reason'], 'Manual')) }}:
- name: isPRorManualBuild
value: 'true'
- ${{ else }}:
- name: isPRorManualBuild
value: 'false'
jobs:
- job: set_uhd_docker_image_version
displayName: Set UHD Docker Image Version
pool:
name: Drivers-NIBuildFarm-RFMIBUILD
demands:
- agent.os -equals Linux
steps:
- checkout: self
clean: true
- script: |
echo pip_index_url: $PIP_INDEX_URL
echo pip_index_host: $PIP_INDEX_HOST
if [[ -z "$PIP_INDEX_URL" || -z "$PIP_INDEX_HOST" ]]
then
echo "Error: One of the variables pip_index_url or pip_index_host is not set"
echo "They must be set as variables in the pipeline UI and point to the correct PyPI server"
exit 1
fi
displayName: Check if UI variables are set
- script: |
cd $(Build.SourcesDirectory)
uhd_version_major=$(sed -nE "s/^set\(UHD_VERSION_MAJOR\s+([0-9]+)\)/\1/p" host/cmake/Modules/UHDVersion.cmake)
uhd_version_api=$(sed -nE "s/^set\(UHD_VERSION_API\s+([0-9]+)\)/\1/p" host/cmake/Modules/UHDVersion.cmake)
uhd_version_abi=$(sed -nE "s/^set\(UHD_VERSION_ABI\s+([0-9]+)\)/\1/p" host/cmake/Modules/UHDVersion.cmake)
if [[ -z $uhd_version_major || -z $uhd_version_api || -z $uhd_version_abi ]]
then
echo "Error parsing UHDVersion.cmake for UHD Version"
exit 1
fi
uhd_docker_image_version=${uhd_version_major}.${uhd_version_api}.${uhd_version_abi}_$(Build.BuildNumber)
echo "Using UHD version string: $uhd_docker_image_version"
echo "##vso[task.setvariable variable=uhdDockerImageVersion;isOutput=true;]$uhd_docker_image_version"
name: set_uhd_version
condition: succeeded()
- job: build_image_linux
dependsOn: set_uhd_docker_image_version
displayName: Build Docker Image
variables:
uhdDockerImageVersion: $[ dependencies.set_uhd_docker_image_version.outputs['set_uhd_version.uhdDockerImageVersion'] ]
pool:
name: Drivers-NIBuildFarm-RFMIBUILD
demands:
- agent.os -equals Linux
- docker
condition: and(succeeded(), ${{ parameters.build_linux }})
strategy:
matrix: $[ variables.linuxDockerImages ]
steps:
- checkout: self
clean: true
- task: Docker@2
displayName: Build Image
inputs:
command: 'build'
Dockerfile: '$(Build.SourcesDirectory)/.ci/docker/$(dockerImageName).Dockerfile'
${{ if or(eq(variables['Build.SourceBranch'], 'refs/heads/master'), startsWith(variables['Build.SourceBranch'], 'refs/heads/UHD-')) }}:
containerRegistry: rnd-docker-niartifacts-sdr-ci-publish
${{ else }}:
containerRegistry: rnd-docker-niartifacts-pre-publish
repository: '$(dockerImagePrefix)$(dockerImageName)'
tags: $(uhdDockerImageVersion)
arguments: --no-cache --pull --build-arg PIP_INDEX_HOST=$(pip_index_host) --build-arg PIP_INDEX_URL=$(pip_index_url)
- task: Docker@2
displayName: Push Image
inputs:
command: 'push'
${{ if or(eq(variables['Build.SourceBranch'], 'refs/heads/master'), startsWith(variables['Build.SourceBranch'], 'refs/heads/UHD-')) }}:
containerRegistry: rnd-docker-niartifacts-sdr-ci-publish
${{ else }}:
containerRegistry: rnd-docker-niartifacts-pre-publish
repository: '$(dockerImagePrefix)$(dockerImageName)'
tags: $(uhdDockerImageVersion)
- job: test_image_linux
displayName: Test Docker Image
dependsOn:
- build_image_linux
- set_uhd_docker_image_version
variables:
uhdDockerImageVersion: $[ dependencies.set_uhd_docker_image_version.outputs['set_uhd_version.uhdDockerImageVersion'] ]
pool:
name: Drivers-NIBuildFarm-RFMIBUILD
demands:
- agent.os -equals Linux
- docker
strategy:
matrix: $[ variables.linuxDockerImages ]
container:
image: "$(dockerImagePrefix)$(dockerImageName):$(uhdDockerImageVersion)"
${{ if or(eq(variables['Build.SourceBranch'], 'refs/heads/master'), startsWith(variables['Build.SourceBranch'], 'refs/heads/UHD-')) }}:
endpoint: rnd-docker-niartifacts-ci-readonly
${{ else }}:
endpoint: rnd-docker-niartifacts-pre-readonly
workspace:
clean: outputs
steps:
- checkout: self
clean: true
- script: |
cd $(Build.BinariesDirectory)
mkdir -p uhddev/build
cd uhddev/build
cmake $(Build.SourcesDirectory)/host
displayName: Test image with cmake uhd
- script: |
cd $(Build.BinariesDirectory)
mkdir -p uhddev/build-ninja
cd uhddev/build-ninja
cmake -G Ninja $(Build.SourcesDirectory)/host
displayName: Test image with cmake ninja uhd
- job: build_image_win
dependsOn: set_uhd_docker_image_version
displayName: Build Docker Image Windows
timeoutInMinutes: 600
variables:
uhdDockerImageVersion: $[ dependencies.set_uhd_docker_image_version.outputs['set_uhd_version.uhdDockerImageVersion'] ]
VCPKG_BINARY_CACHE_DIR: $(Build.SourcesDirectory)\.cache\vcpkg
VCPKG_BINARY_CACHE_CONTAINER_DIR: 'C:\vcpkg-binary-cache'
pool:
name: Drivers-NIBuildFarm-RFMIBUILD
demands:
- agent.os -equals Windows_NT
- docker
- large_disk -equals 1
- high_disk_io -equals 1
- high_cpu_per_thread -equals 1
- cpu_host_type -equals Intel_x86-64-v4
condition: and(succeeded(), ${{ parameters.build_windows }})
strategy:
matrix: $[ variables.winDockerImages ]
steps:
- checkout: self
clean: true
- script: |
REM Set DOCKER_CONFIG based on vsYear
if "$(vsYear)"=="2026" (
echo ##vso[task.setvariable variable=DOCKER_CONFIG]uhd-builder-vs2026-v145-x64
) else if "$(vsYear)"=="2022" (
echo ##vso[task.setvariable variable=DOCKER_CONFIG]uhd-builder-vs2022-v143-x64
) else (
echo ERROR: Unknown vsYear value "$(vsYear)".
exit /b 1
)
REM Set PYTHON_VERSION_ARG based on pythonVersion
if "$(pythonVersion)"=="python310" (
echo ##vso[task.setvariable variable=PYTHON_VERSION_ARG]--build-arg PYTHON_VERSION=$(python310_version)
) else if "$(pythonVersion)"=="python312" (
echo ##vso[task.setvariable variable=PYTHON_VERSION_ARG]--build-arg PYTHON_VERSION=$(python312_version)
) else if "$(pythonVersion)"=="python313" (
echo ##vso[task.setvariable variable=PYTHON_VERSION_ARG]--build-arg PYTHON_VERSION=$(python313_version)
) else (
echo WARNING: pythonVersion is not defined or unknown value "$(pythonVersion)".
echo ##vso[task.setvariable variable=PYTHON_VERSION_ARG]
)
REM Create vcpkg cache directory if it does not exist (e.g. on a cache miss).
REM Docker COPY fails if the source path is absent from the build context.
REM mkdir creates the full path including intermediate directories in one call.
if not exist "$(VCPKG_BINARY_CACHE_DIR)" (
mkdir "$(VCPKG_BINARY_CACHE_DIR)"
)
displayName: Set dynamic build variables
- script: |
call $(Build.SourcesDirectory)\.ci\utils\extract-vs-version.bat "$(cmakeCompiler)"
displayName: Extract VS version info
- powershell: |
$(Build.SourcesDirectory)\.ci\docker\scripts\check-url.ps1 -Url "$(vsToolsUrl)"
displayName: Verify expected VS Tools Installer URL
- ${{ if parameters.use_vcpkg_cache }}:
- powershell: |
$now = Get-Date
$calendar = [System.Globalization.CultureInfo]::InvariantCulture.Calendar
$week = $calendar.GetWeekOfYear(
$now,
[System.Globalization.CalendarWeekRule]::FirstFourDayWeek,
[System.DayOfWeek]::Monday
)
$cacheTimestamp = "{0}w{1:D2}" -f $now.Year, $week
Write-Host "CacheTimestamp=$cacheTimestamp"
Write-Host "##vso[task.setvariable variable=CacheTimestamp]$cacheTimestamp"
displayName: Set CacheTimestamp
- task: Cache@2
inputs:
key: |
"vcpkg" | "$(vsYear)" | "$(CacheTimestamp)"
${{ if eq(variables['isPRorManualBuild'], 'true') }}:
restoreKeys: |
"vcpkg" | "$(vsYear)"
path: $(VCPKG_BINARY_CACHE_DIR)
displayName: Cache vcpkg binary packages
- script: |
echo Pruning dangling images and stopped containers to reclaim disk space before build.
docker container prune -f
REM Pass 1: remove old uhd-builder images from previous runs (largest space consumers).
for /f "tokens=3" %%i in ('docker images -a --format table ^| findstr "uhd-builder"') do docker rmi %%i >nul 2>&1
REM Pass 2: remove any <none> / <untagged> dangling layers left behind after pass 1.
for /f "tokens=3" %%i in ('docker images -a --format table ^| findstr /c:"<none>" /c:"<untagged>"') do docker rmi %%i >nul 2>&1
docker images --format table
REM Always exit with success (0) so this cleanup step never fails the pipeline.
exit /b 0
displayName: Prune dangling Docker images
condition: and(succeeded(), or(${{ parameters.prune_docker_images }}, ne(variables['isPRorManualBuild'], 'true')))
- task: Docker@2
displayName: Build Image
retryCountOnTaskFailure: 3
inputs:
command: 'build'
Dockerfile: '$(Build.SourcesDirectory)/.ci/docker/$(DOCKER_CONFIG).Dockerfile'
${{ if or(eq(variables['Build.SourceBranch'], 'refs/heads/master'), startsWith(variables['Build.SourceBranch'], 'refs/heads/UHD-')) }}:
containerRegistry: rnd-docker-niartifacts-sdr-ci-publish
${{ else }}:
containerRegistry: rnd-docker-niartifacts-pre-publish
repository: '$(dockerImagePrefix)$(dockerImageName)'
tags: $(uhdDockerImageVersion)
buildContext: $(Build.SourcesDirectory)
arguments: >
--no-cache --pull
--build-arg VCPKG_BINARY_CACHE_DIR=$(VCPKG_BINARY_CACHE_CONTAINER_DIR)
--build-arg VS_BUILD_TOOLS_URL=$(vsToolsUrl)
--build-arg PIP_INDEX_HOST=$(pip_index_host)
--build-arg PIP_INDEX_URL=$(pip_index_url)
--build-arg VCPKG_MANIFEST_FILE=$(vcpkgManifestFile)
$(PYTHON_VERSION_ARG)
- script: |
docker images --format table
displayName: List Docker images
- ${{ if parameters.use_vcpkg_cache }}:
- script: |
set IMAGE_ID=
REM Match image by name:version in plain docker images output, independent of registry hostname prefix.
REM docker images --format with Go templates is unreliable on Windows Docker.
REM Standard output columns: REPOSITORY TAG IMAGE-ID CREATED SIZE -> tokens=3 gives IMAGE-ID.
for /f "tokens=3" %%i in ('docker images --format table ^| findstr /i "$(dockerImageName)" ^| findstr /i "$(uhdDockerImageVersion)"') do set IMAGE_ID=%%i
if not defined IMAGE_ID (
echo WARNING: Could not find locally built image for $(dockerImagePrefix)$(dockerImageName):$(uhdDockerImageVersion)
echo WARNING: Skipping vcpkg cache extraction and preventing cache publish for this run.
if exist "$(VCPKG_BINARY_CACHE_DIR)" (
rmdir /s /q "$(VCPKG_BINARY_CACHE_DIR)"
)
exit /b 0
)
echo Using local IMAGE_ID=%IMAGE_ID%
docker rm -f vcpkgcache >nul 2>&1
docker create --name vcpkgcache ^
%IMAGE_ID%
if errorlevel 1 (
echo WARNING: Failed to create temporary container for cache extraction.
echo WARNING: Skipping vcpkg cache extraction and preventing cache publish for this run.
if exist "$(VCPKG_BINARY_CACHE_DIR)" (
rmdir /s /q "$(VCPKG_BINARY_CACHE_DIR)"
)
exit /b 0
)
docker images --format table
if exist "$(VCPKG_BINARY_CACHE_DIR)" (
rmdir /s /q "$(VCPKG_BINARY_CACHE_DIR)"
)
docker cp vcpkgcache:$(VCPKG_BINARY_CACHE_CONTAINER_DIR) "$(VCPKG_BINARY_CACHE_DIR)"
if errorlevel 1 (
echo WARNING: Failed to copy vcpkg cache from built image.
echo WARNING: Skipping vcpkg cache publish for this run.
docker rm -f vcpkgcache >nul 2>&1
if exist "$(VCPKG_BINARY_CACHE_DIR)" (
rmdir /s /q "$(VCPKG_BINARY_CACHE_DIR)"
)
exit /b 0
)
docker rm vcpkgcache
displayName: Extract vcpkg cache
- task: Docker@2
displayName: Push Image
inputs:
command: 'push'
${{ if or(eq(variables['Build.SourceBranch'], 'refs/heads/master'), startsWith(variables['Build.SourceBranch'], 'refs/heads/UHD-')) }}:
containerRegistry: rnd-docker-niartifacts-sdr-ci-publish
${{ else }}:
containerRegistry: rnd-docker-niartifacts-pre-publish
repository: '$(dockerImagePrefix)$(dockerImageName)'
tags: $(uhdDockerImageVersion)
- task: PublishPipelineArtifact@1
inputs:
targetPath: $(VCPKG_BINARY_CACHE_DIR)
artifactName: vcpkg-binary-cache-$(dockerImageName))-run$(Build.BuildId)
displayName: Publish vcpkg binary cache as artifact
condition: eq(variables['System.Debug'], 'true')
- job: test_image_win
displayName: Test Docker Image Windows
timeoutInMinutes: 400
dependsOn:
- build_image_win
- set_uhd_docker_image_version
variables:
uhdDockerImageVersion: $[ dependencies.set_uhd_docker_image_version.outputs['set_uhd_version.uhdDockerImageVersion'] ]
pool:
name: Drivers-NIBuildFarm-RFMIBUILD
demands:
- agent.os -equals Windows_NT
- docker
- large_disk -equals 1
- high_disk_io -equals 1
- high_cpu_per_thread -equals 1
- cpu_host_type -equals Intel_x86-64-v4
strategy:
matrix: $[ variables.winDockerImages ]
container:
image: "$(dockerImagePrefix)$(dockerImageName):$(uhdDockerImageVersion)"
options: --cpu-count 6 --memory 12G
${{ if or(eq(variables['Build.SourceBranch'], 'refs/heads/master'), startsWith(variables['Build.SourceBranch'], 'refs/heads/UHD-')) }}:
endpoint: rnd-docker-niartifacts-ci-readonly
${{ else }}:
endpoint: rnd-docker-niartifacts-pre-readonly
workspace:
clean: outputs
steps:
- checkout: self
clean: true
- script: |
mkdir c:\uhddev\build
cd c:\uhddev\build
cmake $(Build.SourcesDirectory)/host ^
-DVCPKG_TARGET_TRIPLET=uhd-$(vsArch)-windows-static-md ^
-DCMAKE_TOOLCHAIN_FILE=%VCPKG_INSTALL_DIR%\scripts\buildsystems\vcpkg.cmake ^
-DVCPKG_INSTALLED_DIR=c:\uhd-vcpkg\vcpkg_installed ^
-DVCPKG_MANIFEST_DIR=c:\uhd-vcpkg ^
-G "$(cmakeCompiler)" -A $(cmakeArch)
displayName: Test image with cmake uhd
- script: |
call $(Build.SourcesDirectory)\.ci\utils\extract-vs-version.bat "$(cmakeCompiler)"
cd c:\uhddev\build
"%vsToolsInstallDir%\VC\Auxiliary\Build\vcvarsall.bat" $(vsArch)
displayName: Load Visual C++ Env
- job: export_build_number
dependsOn: set_uhd_docker_image_version
displayName: Export Image List to Artifacts
variables:
uhdDockerImageVersion: $[ dependencies.set_uhd_docker_image_version.outputs['set_uhd_version.uhdDockerImageVersion'] ]
pool:
name: Drivers-NIBuildFarm-RFMIBUILD
demands: sh
steps:
- checkout: self
clean: true
sparseCheckoutDirectories: .ci
- script: |
cd $(Build.ArtifactStagingDirectory)
touch $(UHDPipelineDockerImageArtifactName)
echo "$(uhdDockerImageVersion)" > $(UHDPipelineDockerImageArtifactName)
echo "${{ variables.dockerImagePrefix }}" >> $(UHDPipelineDockerImageArtifactName)
echo "${{ variables.linuxDockerImages }}" >> $(UHDPipelineDockerImageArtifactName)
echo "${{ variables.winDockerImages }}" >> $(UHDPipelineDockerImageArtifactName)
echo "${{ variables.macOSBuilders }}" >> $(UHDPipelineDockerImageArtifactName)
echo "${{ variables.winDockerImagesPythonBuild }}" >> $(UHDPipelineDockerImageArtifactName)
displayName: Generate created image list
- script: |
cd $(Build.SourcesDirectory)
python3 .ci/utils/tests/test_derive_docker_image_matrix.py --artifact-file "$(Build.ArtifactStagingDirectory)/$(UHDPipelineDockerImageArtifactName)"
displayName: Validate generated image list with matrix unit test
- task: PublishPipelineArtifact@1
inputs:
targetPath: $(Build.ArtifactStagingDirectory)
artifactName: $(UHDPipelineDockerImageArtifactName)
displayName: Upload image list artifact
Reference in New Issue View Git Blame Copy Permalink