Skip to content

Commit

Permalink
Remove Windows ARM32 from nuget packaging pipelines (microsoft#19049)
Browse files Browse the repository at this point in the history 
### Description
1. Remove Windows ARM32 from nuget  packaging pipelines

2. Add missing component-governance-component-detection-steps.yml to
some build jobs.

### Motivation and Context
Stop supporting Windows ARM32 to align with [Windows's support
policy](https://learn.microsoft.com/en-us/windows/arm/arm32-to-arm64).
Users who need this feature still can build the DLLs from source.
However, later on we will remove that support too.
  • Loading branch information
@snnn
snnn authored Jan 9, 2024
1 parent 7cb8b20 commit ab897a4
Show file tree
Hide file tree
Showing 4 changed files with 49 additions and 132 deletions.
66 changes: 3 additions & 63 deletions .pipelines/OneBranch.Nuget-WindowsAI-Pipeline.Official.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,10 +53,6 @@ extends:
BuildArch: x86
PythonPackageName: pythonx86

- template: .pipelines/windowsai-steps.yml@self
parameters:
BuildArch: arm

- template: .pipelines/windowsai-steps.yml@self
parameters:
BuildArch: arm64
Expand All @@ -72,11 +68,6 @@ extends:
PythonPackageName: pythonx86
Runtime: static

- template: .pipelines/windowsai-steps.yml@self
parameters:
BuildArch: arm
Runtime: static

- template: .pipelines/windowsai-steps.yml@self
parameters:
BuildArch: arm64
Expand All @@ -94,11 +85,9 @@ extends:
dependsOn:
- Windows_Packaging_x64_dynamic
- Windows_Packaging_x86_dynamic
- Windows_Packaging_arm_dynamic
- Windows_Packaging_arm64_dynamic
- Windows_Packaging_x64_static
- Windows_Packaging_x86_static
- Windows_Packaging_arm_static
- Windows_Packaging_arm64_static
condition: succeeded()
steps:
Expand All @@ -120,12 +109,6 @@ extends:
artifactName: 'drop_Windows_Build_Windows_Packaging_arm64_dynamic'
targetPath: '$(Build.BinariesDirectory)/nuget-artifact-arm64'

- task: DownloadPipelineArtifact@0
displayName: 'Download Pipeline Artifact - NuGet DirectML arm'
inputs:
artifactName: 'drop_Windows_Build_Windows_Packaging_arm_dynamic'
targetPath: '$(Build.BinariesDirectory)/nuget-artifact-arm'

- task: DownloadPipelineArtifact@0
displayName: 'Download Pipeline Artifact - NuGet DirectML x64 StaticRuntime'
inputs:
Expand All @@ -144,12 +127,6 @@ extends:
artifactName: 'drop_Windows_Build_Windows_Packaging_arm64_static'
targetPath: '$(Build.BinariesDirectory)/nuget-artifact-arm64-static-runtime'

- task: DownloadPipelineArtifact@0
displayName: 'Download Pipeline Artifact - NuGet DirectML arm StaticRuntime'
inputs:
artifactName: 'drop_Windows_Build_Windows_Packaging_arm_static'
targetPath: '$(Build.BinariesDirectory)/nuget-artifact-arm-static-runtime'

- task: PowerShell@2
displayName: 'Bundle NuGet and other binaries'
inputs:
Expand Down Expand Up @@ -194,17 +171,7 @@ extends:
$arm64_static_runtime_nupkg_unzipped_directory = [System.IO.Path]::Combine($arm64_static_runtime_nupkg_unzipped_directory_root, 'binaries', [System.IO.Path]::GetFileNameWithoutExtension($arm64_static_runtime_nuget_package))
[System.IO.Compression.ZipFile]::ExtractToDirectory($arm64_static_runtime_nuget_package, $arm64_static_runtime_nupkg_unzipped_directory)
$nupkgs = (Get-ChildItem ..\nuget-artifact-arm -Filter Microsoft.AI.MachineLearning*.nupkg -Recurse)
$arm_nuget_package = $nupkgs[0].FullName
$arm_nupkg_unzipped_directory_root = $nupkgs[0].Directory.FullName
$arm_nupkg_unzipped_directory = [System.IO.Path]::Combine($arm_nupkg_unzipped_directory_root, 'binaries', [System.IO.Path]::GetFileNameWithoutExtension($arm_nuget_package))
[System.IO.Compression.ZipFile]::ExtractToDirectory($arm_nuget_package, $arm_nupkg_unzipped_directory)
$nupkgs = (Get-ChildItem ..\nuget-artifact-arm-static-runtime -Filter Microsoft.AI.MachineLearning*.nupkg -Recurse)
$arm_static_runtime_nuget_package = $nupkgs[0].FullName
$arm_static_runtime_nupkg_unzipped_directory_root = $nupkgs[0].Directory.FullName
$arm_static_runtime_nupkg_unzipped_directory = [System.IO.Path]::Combine($arm_static_runtime_nupkg_unzipped_directory_root, 'binaries', [System.IO.Path]::GetFileNameWithoutExtension($arm_static_runtime_nuget_package))
[System.IO.Compression.ZipFile]::ExtractToDirectory($arm_static_runtime_nuget_package, $arm_static_runtime_nupkg_unzipped_directory)
$x64_static_runtime_path_old = [System.IO.Path]::Combine($x64_static_runtime_nupkg_unzipped_directory, 'runtimes', 'win-x64', '_native')
$x64_static_runtime_path_new = [System.IO.Path]::Combine($x64_nupkg_unzipped_directory, 'runtimes', 'win-x64', '_native', 'static')
Expand All @@ -216,10 +183,7 @@ extends:
$arm64_runtime_path_new = [System.IO.Path]::Combine($x64_nupkg_unzipped_directory, 'runtimes', 'win-arm64', '_native')
$arm64_static_runtime_path_old = [System.IO.Path]::Combine($arm64_static_runtime_nupkg_unzipped_directory, 'runtimes', 'win-arm64', '_native')
$arm64_static_runtime_path_new = [System.IO.Path]::Combine($x64_nupkg_unzipped_directory, 'runtimes', 'win-arm64', '_native', 'static')
$arm_runtime_path_old = [System.IO.Path]::Combine($arm_nupkg_unzipped_directory, 'runtimes', 'win-arm', '_native')
$arm_runtime_path_new = [System.IO.Path]::Combine($x64_nupkg_unzipped_directory, 'runtimes', 'win-arm', '_native')
$arm_static_runtime_path_old = [System.IO.Path]::Combine($arm_static_runtime_nupkg_unzipped_directory, 'runtimes', 'win-arm', '_native')
$arm_static_runtime_path_new = [System.IO.Path]::Combine($x64_nupkg_unzipped_directory, 'runtimes', 'win-arm', '_native', 'static')
$uap_build_path_old = [System.IO.Path]::Combine($x64_static_runtime_nupkg_unzipped_directory, 'build', 'native')
$uap_build_path_new = [System.IO.Path]::Combine($x64_nupkg_unzipped_directory, 'build', 'uap10.0')
Expand All @@ -228,8 +192,6 @@ extends:
New-Item -Path $x86_static_runtime_path_new -ItemType Directory
New-Item -Path $arm64_runtime_path_new -ItemType Directory
New-Item -Path $arm64_static_runtime_path_new -ItemType Directory
New-Item -Path $arm_runtime_path_new -ItemType Directory
New-Item -Path $arm_static_runtime_path_new -ItemType Directory
Copy-Item ([System.IO.Path]::Combine($x86_runtime_path_old, 'onnxruntime.dll')) $x86_runtime_path_new
Copy-Item ([System.IO.Path]::Combine($x86_runtime_path_old, 'onnxruntime.lib')) $x86_runtime_path_new
Expand All @@ -241,11 +203,6 @@ extends:
Copy-Item ([System.IO.Path]::Combine($arm64_runtime_path_old, 'microsoft.ai.machinelearning.dll')) $arm64_runtime_path_new
Copy-Item ([System.IO.Path]::Combine($arm64_runtime_path_old, 'microsoft.ai.machinelearning.lib')) $arm64_runtime_path_new
Copy-Item ([System.IO.Path]::Combine($arm_runtime_path_old, 'onnxruntime.dll')) $arm_runtime_path_new
Copy-Item ([System.IO.Path]::Combine($arm_runtime_path_old, 'onnxruntime.lib')) $arm_runtime_path_new
Copy-Item ([System.IO.Path]::Combine($arm_runtime_path_old, 'microsoft.ai.machinelearning.dll')) $arm_runtime_path_new
Copy-Item ([System.IO.Path]::Combine($arm_runtime_path_old, 'microsoft.ai.machinelearning.lib')) $arm_runtime_path_new
Copy-Item ([System.IO.Path]::Combine($x64_static_runtime_path_old, 'onnxruntime.dll')) ([System.IO.Path]::Combine($x64_static_runtime_path_new, 'onnxruntime.dll'))
Copy-Item ([System.IO.Path]::Combine($x64_static_runtime_path_old, 'onnxruntime.lib')) ([System.IO.Path]::Combine($x64_static_runtime_path_new, 'onnxruntime.lib'))
Copy-Item ([System.IO.Path]::Combine($x64_static_runtime_path_old, 'microsoft.ai.machinelearning.dll')) ([System.IO.Path]::Combine($x64_static_runtime_path_new, 'microsoft.ai.machinelearning.dll'))
Expand All @@ -261,11 +218,6 @@ extends:
Copy-Item ([System.IO.Path]::Combine($arm64_static_runtime_path_old, 'microsoft.ai.machinelearning.dll')) ([System.IO.Path]::Combine($arm64_static_runtime_path_new, 'microsoft.ai.machinelearning.dll'))
Copy-Item ([System.IO.Path]::Combine($arm64_static_runtime_path_old, 'microsoft.ai.machinelearning.lib')) ([System.IO.Path]::Combine($arm64_static_runtime_path_new, 'microsoft.ai.machinelearning.lib'))
Copy-Item ([System.IO.Path]::Combine($arm_static_runtime_path_old, 'onnxruntime.dll')) ([System.IO.Path]::Combine($arm_static_runtime_path_new, 'onnxruntime.dll'))
Copy-Item ([System.IO.Path]::Combine($arm_static_runtime_path_old, 'onnxruntime.lib')) ([System.IO.Path]::Combine($arm_static_runtime_path_new, 'onnxruntime.lib'))
Copy-Item ([System.IO.Path]::Combine($arm_static_runtime_path_old, 'microsoft.ai.machinelearning.dll')) ([System.IO.Path]::Combine($arm_static_runtime_path_new, 'microsoft.ai.machinelearning.dll'))
Copy-Item ([System.IO.Path]::Combine($arm_static_runtime_path_old, 'microsoft.ai.machinelearning.lib')) ([System.IO.Path]::Combine($arm_static_runtime_path_new, 'microsoft.ai.machinelearning.lib'))
Copy-Item -Recurse $uap_build_path_old $uap_build_path_new
$merged_nuget_path = [System.IO.Path]::Combine($Env:BUILD_ARTIFACTSTAGINGDIRECTORY, 'merged')
Expand Down Expand Up @@ -304,32 +256,20 @@ extends:
$arm64_nupkg_unzipped_directory = [System.IO.Path]::Combine($arm64_nupkg_unzipped_directory_root, 'symbols', [System.IO.Path]::GetFileNameWithoutExtension($arm64_nuget_package))
[System.IO.Compression.ZipFile]::ExtractToDirectory($arm64_nuget_package, $arm64_nupkg_unzipped_directory)
$nupkgs = (Get-ChildItem ..\nuget-artifact-arm -Filter Microsoft.AI.MachineLearning*.snupkg -Recurse)
$arm_nuget_package = $nupkgs[0].FullName
$arm_nupkg_unzipped_directory_root = $nupkgs[0].Directory.FullName
$arm_nupkg_unzipped_directory = [System.IO.Path]::Combine($arm_nupkg_unzipped_directory_root, 'symbols', [System.IO.Path]::GetFileNameWithoutExtension($arm_nuget_package))
[System.IO.Compression.ZipFile]::ExtractToDirectory($arm_nuget_package, $arm_nupkg_unzipped_directory)
$x86_runtime_path_old = [System.IO.Path]::Combine($x86_nupkg_unzipped_directory, 'runtimes', 'win-x86', '_native')
$x86_runtime_path_new = [System.IO.Path]::Combine($x64_nupkg_unzipped_directory, 'runtimes', 'win-x86', '_native')
$arm64_runtime_path_old = [System.IO.Path]::Combine($arm64_nupkg_unzipped_directory, 'runtimes', 'win-arm64', '_native')
$arm64_runtime_path_new = [System.IO.Path]::Combine($x64_nupkg_unzipped_directory, 'runtimes', 'win-arm64', '_native')
$arm_runtime_path_old = [System.IO.Path]::Combine($arm_nupkg_unzipped_directory, 'runtimes', 'win-arm', '_native')
$arm_runtime_path_new = [System.IO.Path]::Combine($x64_nupkg_unzipped_directory, 'runtimes', 'win-arm', '_native')
New-Item -Path $x86_runtime_path_new -ItemType Directory
New-Item -Path $arm64_runtime_path_new -ItemType Directory
New-Item -Path $arm_runtime_path_new -ItemType Directory
Copy-Item ([System.IO.Path]::Combine($x86_runtime_path_old, 'onnxruntime.pdb')) $x86_runtime_path_new
Copy-Item ([System.IO.Path]::Combine($x86_runtime_path_old, 'microsoft.ai.machinelearning.pdb')) $x86_runtime_path_new
Copy-Item ([System.IO.Path]::Combine($arm64_runtime_path_old, 'onnxruntime.pdb')) $arm64_runtime_path_new
Copy-Item ([System.IO.Path]::Combine($arm64_runtime_path_old, 'microsoft.ai.machinelearning.pdb')) $arm64_runtime_path_new
Copy-Item ([System.IO.Path]::Combine($arm_runtime_path_old, 'onnxruntime.pdb')) $arm_runtime_path_new
Copy-Item ([System.IO.Path]::Combine($arm_runtime_path_old, 'microsoft.ai.machinelearning.pdb')) $arm_runtime_path_new
$merged_nuget_path = [System.IO.Path]::Combine($Env:BUILD_ARTIFACTSTAGINGDIRECTORY, 'merged')
if (!(Test-Path $merged_nuget_path)) {
New-Item -Path $merged_nuget_path -ItemType Directory
Expand Down
85 changes: 42 additions & 43 deletions tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,9 @@ stages:
vmImage: ubuntu-latest
steps:
- checkout: none
- task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
displayName: 'Clean Agent Directories'
condition: always()
- bash: |
# Do not output ##vso[] commands with `set -x` or they may be parsed again and include a trailing quote.
set +x
Expand All @@ -105,6 +108,10 @@ stages:
echo "##vso[task.setvariable variable=ReleaseVersionSuffix;isOutput=true]"
fi
name: Set_Release_Version_Suffix
- template: templates/component-governance-component-detection-steps.yml
parameters :
condition : 'succeeded'


- stage: Debug
dependsOn: Setup
Expand All @@ -116,7 +123,14 @@ stages:
MyVar: $[stageDependencies.Setup.Set_Variables.outputs['Set_Release_Version_Suffix.ReleaseVersionSuffix']]
steps:
- checkout: none
- task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
displayName: 'Clean Agent Directories'
condition: always()
- bash: echo $(MyVar)
- template: templates/component-governance-component-detection-steps.yml
parameters :
condition : 'succeeded'


- stage: Download_Java_Tools
dependsOn: []
Expand All @@ -126,6 +140,9 @@ stages:
vmImage: ubuntu-latest
steps:
- checkout: none
- task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
displayName: 'Clean Agent Directories'
condition: always()
- task: CmdLine@2
displayName: Download Java Tools
inputs:
Expand All @@ -141,6 +158,9 @@ stages:
inputs:
targetPath: '$(Agent.TempDirectory)/java-tools'
artifact: 'onnxruntime-java-tools'
- template: templates/component-governance-component-detection-steps.yml
parameters :
condition : 'succeeded'

- template: templates/c-api-cpu.yml
parameters:
Expand Down Expand Up @@ -525,6 +545,9 @@ stages:
submodules: false
- checkout: manylinux # due to checkout multiple repos, the root directory is $(Build.SourcesDirectory)/manylinux
submodules: false
- task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
displayName: 'Clean Agent Directories'
condition: always()

- script: |
set -e -x
Expand Down Expand Up @@ -603,6 +626,10 @@ stages:
inputs:
targetPath: '$(Build.ArtifactStagingDirectory)/onnxruntime-linux-x64-gpu-$(OnnxRuntimeVersion).tgz'
artifactName: 'onnxruntime-linux-x64-gpu'
- template: templates/component-governance-component-detection-steps.yml
parameters :
condition : 'succeeded'


- stage: Windows_Packaging_combined_GPU
dependsOn:
Expand All @@ -619,6 +646,10 @@ stages:
- checkout: self # due to checkout multiple repos, the root directory is $(Build.SourcesDirectory)/onnxruntime
- checkout: onnxruntime-inference-examples # due to checkout multiple repos, the root directory is $(Build.SourcesDirectory)/onnxruntime-inference-examples
submodules: false
- task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
displayName: 'Clean Agent Directories'
condition: always()

- script: dir $(Build.SourcesDirectory)
- task: BatchScript@1
displayName: 'setup env'
Expand Down Expand Up @@ -688,7 +719,9 @@ stages:
inputs:
artifactName: 'onnxruntime-win-x64-gpu'
targetPath: '$(Build.ArtifactStagingDirectory)'

- template: templates/component-governance-component-detection-steps.yml
parameters :
condition : 'succeeded'

- stage: NuGet_Packaging_GPU
dependsOn:
Expand Down Expand Up @@ -1246,45 +1279,21 @@ stages:
mkdir $(Build.ArtifactStagingDirectory)\testdata
copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\custom_op_library.* $(Build.ArtifactStagingDirectory)\testdata
- template: nuget/templates/dml-vs-2022.yml
parameters:
AgentPool : 'onnxruntime-Win-CPU-2022'
IsReleaseBuild: ${{ parameters.IsReleaseBuild }}
ArtifactName: 'drop-win-dml-arm-zip'
StageName: 'Windows_CI_GPU_DML_Dev_arm'
BuildCommand: --build_dir $(Build.BinariesDirectory) --arm --skip_submodule_sync --build_shared_lib --enable_onnx_tests --enable_wcos --use_telemetry --use_dml --use_winml --cmake_generator "Visual Studio 17 2022"
BuildArch: 'x64'
EnvSetupScript: 'setup_env.bat'
sln_platform: 'arm'
DoDebugBuild: 'false'
DoNugetPack : 'true'
DoCompliance: ${{ parameters.DoCompliance }}
DoEsrp: ${{ parameters.DoEsrp }}
RunTests: 'false'
BuildNodejs: 'false'
NuPackScript: |
msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /p:TargetArchitecture=arm /t:CreatePackage /p:OrtPackageId=Microsoft.ML.OnnxRuntime.DirectML /p:IsReleaseBuild=${{ parameters.IsReleaseBuild }}
cd $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\
ren Microsoft.ML.OnnxRuntime.DirectML.* win-dml-arm.zip
copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\win-dml-arm.zip $(Build.ArtifactStagingDirectory)
mkdir $(Build.ArtifactStagingDirectory)\testdata
copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\custom_op_library.* $(Build.ArtifactStagingDirectory)\testdata
- stage: NuGet_Packaging_DML
dependsOn:
- Windows_CI_GPU_DML_Dev
- Windows_CI_GPU_DML_Dev_x86
- Windows_CI_GPU_DML_Dev_arm64
- Windows_CI_GPU_DML_Dev_arm
condition: succeeded()
jobs:
- job:
workspace:
clean: all
pool: 'onnxruntime-Win2022-GPU-T4'

pool: 'onnxruntime-Win2022-GPU-dml-A10'
steps:

- task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
displayName: 'Clean Agent Directories'
condition: always()
- task: DownloadPipelineArtifact@0
displayName: 'Download Pipeline Artifact - NuGet DirectML'
inputs:
Expand All @@ -1303,12 +1312,6 @@ stages:
artifactName: 'drop-win-dml-arm64-zip'
targetPath: '$(Build.BinariesDirectory)/nuget-artifact-dml'

- task: DownloadPipelineArtifact@0
displayName: 'Download Pipeline Artifact - NuGet DirectML arm'
inputs:
artifactName: 'drop-win-dml-arm-zip'
targetPath: '$(Build.BinariesDirectory)/nuget-artifact-dml'

- script: |
pushd $(Build.BinariesDirectory)\nuget-artifact-dml
dir
Expand Down Expand Up @@ -1339,13 +1342,6 @@ stages:
move win-arm64\runtimes\win-arm64\native\onnxruntime.lib %%~ni\runtimes\win-arm64\native\onnxruntime.lib
move win-arm64\runtimes\win-arm64\native\onnxruntime.pdb %%~ni\runtimes\win-arm64\native\onnxruntime.pdb
unzip win-dml-arm.zip -d win-arm
mkdir %%~ni\runtimes\win-arm
mkdir %%~ni\runtimes\win-arm\native
move win-arm\runtimes\win-arm\native\onnxruntime.dll %%~ni\runtimes\win-arm\native\onnxruntime.dll
move win-arm\runtimes\win-arm\native\onnxruntime.lib %%~ni\runtimes\win-arm\native\onnxruntime.lib
move win-arm\runtimes\win-arm\native\onnxruntime.pdb %%~ni\runtimes\win-arm\native\onnxruntime.pdb
pushd %%~ni
zip -r ..\%%~ni.zip .
Expand All @@ -1368,11 +1364,14 @@ stages:
PackageType: 'nuget'
PackagePath: '$(Build.ArtifactStagingDirectory)'
PackageName: 'Microsoft.ML.OnnxRuntime.DirectML*nupkg'
PlatformsSupported: 'win-x64,win-x86,win-arm64,win-arm'
PlatformsSupported: 'win-x64,win-x86,win-arm64'
VerifyNugetSigning: ${{ parameters.DoEsrp }}

- task: PublishPipelineArtifact@0
displayName: 'Publish Pipeline NuGet Artifact'
inputs:
artifactName: 'drop-signed-nuget-dml'
targetPath: '$(Build.ArtifactStagingDirectory)'
- template: templates/component-governance-component-detection-steps.yml
parameters :
condition : 'succeeded'
Loading

0 comments on commit ab897a4

Please sign in to comment.