[release/4.0] Update dependencies from dotnet/arcade (#7327)

[release/4.0] Update dependencies from dotnet/arcade - fixing net version in sample
dotnet · Dec 5, 2024 · d92c0b3 · d92c0b3
1 parent 26bb7cb
commit d92c0b3
Show file tree

Hide file tree

Showing 15 changed files with 98 additions and 141 deletions.
diff --git a/docs/samples/Microsoft.ML.Samples.OneDal/Microsoft.ML.Samples.OneDal.csproj b/docs/samples/Microsoft.ML.Samples.OneDal/Microsoft.ML.Samples.OneDal.csproj
@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
     <!--This ensures that we can never make the mistake of adding this as a friend assembly. Please don't remove.-->
     <PublicSign>false</PublicSign>
   </PropertyGroup>

diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml
@@ -7,38 +7,38 @@
     </Dependency>
   </ProductDependencies>
   <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="10.0.0-beta.24504.4">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="9.0.0-beta.24572.2">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>f209a925b15bc66ecb9a8825bd9595937bbe3aa1</Sha>
+      <Sha>b41381d5cd633471265e9cd72e933a7048e03062</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.Build.Tasks.Feed" Version="10.0.0-beta.24504.4">
+    <Dependency Name="Microsoft.DotNet.Build.Tasks.Feed" Version="9.0.0-beta.24572.2">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>f209a925b15bc66ecb9a8825bd9595937bbe3aa1</Sha>
+      <Sha>b41381d5cd633471265e9cd72e933a7048e03062</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.SignTool" Version="10.0.0-beta.24504.4">
+    <Dependency Name="Microsoft.DotNet.SignTool" Version="9.0.0-beta.24572.2">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>f209a925b15bc66ecb9a8825bd9595937bbe3aa1</Sha>
+      <Sha>b41381d5cd633471265e9cd72e933a7048e03062</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="10.0.0-beta.24504.4">
+    <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="9.0.0-beta.24572.2">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>f209a925b15bc66ecb9a8825bd9595937bbe3aa1</Sha>
+      <Sha>b41381d5cd633471265e9cd72e933a7048e03062</Sha>
     </Dependency>
     <!-- Stay on package 8.0 until we stop testing for net6.0
     <Dependency Name="Microsoft.DotNet.RemoteExecutor" Version="9.0.0-beta.24212.4">
       <Uri>https://github.com/dotnet/arcade</Uri>
       <Sha>812d978c303174dc1aa305d7359e79053d7d4971</Sha>
     </Dependency> -->
-    <Dependency Name="Microsoft.DotNet.SwaggerGenerator.MSBuild" Version="10.0.0-beta.24504.4">
+    <Dependency Name="Microsoft.DotNet.SwaggerGenerator.MSBuild" Version="9.0.0-beta.24572.2">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>f209a925b15bc66ecb9a8825bd9595937bbe3aa1</Sha>
+      <Sha>b41381d5cd633471265e9cd72e933a7048e03062</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.XUnitExtensions" Version="10.0.0-beta.24504.4">
+    <Dependency Name="Microsoft.DotNet.XUnitExtensions" Version="9.0.0-beta.24572.2">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>f209a925b15bc66ecb9a8825bd9595937bbe3aa1</Sha>
+      <Sha>b41381d5cd633471265e9cd72e933a7048e03062</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.XliffTasks" Version="10.0.0-beta.24504.4">
+    <Dependency Name="Microsoft.DotNet.XliffTasks" Version="9.0.0-beta.24572.2">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>f209a925b15bc66ecb9a8825bd9595937bbe3aa1</Sha>
+      <Sha>b41381d5cd633471265e9cd72e933a7048e03062</Sha>
     </Dependency>
   </ToolsetDependencies>
 </Dependencies>
diff --git a/eng/Versions.props b/eng/Versions.props
@@ -91,7 +91,7 @@
     <MicrosoftCodeAnalysisTestingVersion>1.1.2-beta1.23431.1</MicrosoftCodeAnalysisTestingVersion>
     <MicrosoftDotNetPlatformAbstractionsVersion>5.0.0-preview.5.20278.1</MicrosoftDotNetPlatformAbstractionsVersion>
     <MicrosoftDotNetRemoteExecutorVersion>8.0.0-beta.24525.2</MicrosoftDotNetRemoteExecutorVersion>
-    <MicrosoftDotNetXUnitExtensionsVersion>10.0.0-beta.24504.4</MicrosoftDotNetXUnitExtensionsVersion>
+    <MicrosoftDotNetXUnitExtensionsVersion>9.0.0-beta.24572.2</MicrosoftDotNetXUnitExtensionsVersion>
     <MicrosoftExtensionsDependencyModelVersion>8.0.1</MicrosoftExtensionsDependencyModelVersion>
     <MicrosoftMLOnnxTestModelsVersion>0.0.6-test</MicrosoftMLOnnxTestModelsVersion>
     <MicrosoftMLTensorFlowTestModelsVersion>0.0.13-test</MicrosoftMLTensorFlowTestModelsVersion>

diff --git a/eng/common/core-templates/job/job.yml b/eng/common/core-templates/job/job.yml
@@ -19,7 +19,6 @@ parameters:
   # publishing defaults
   artifacts: ''
   enableMicrobuild: false
-  enableMicrobuildForMacAndLinux: false
   enablePublishBuildArtifacts: false
   enablePublishBuildAssets: false
   enablePublishTestResults: false
@@ -135,26 +134,11 @@ jobs:
           signType: $(_SignType)
           zipSources: false
           feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json
-          ${{ if and(eq(parameters.enableMicrobuildForMacAndLinux, 'true'), ne(variables['Agent.Os'], 'Windows_NT')) }}:
-            azureSubscription: 'MicroBuild Signing Task (DevDiv)'
         env:
           TeamName: $(_TeamName)
           MicroBuildOutputFolderOverride: '$(Agent.TempDirectory)'
-          SYSTEM_ACCESSTOKEN: $(System.AccessToken)
         continueOnError: ${{ parameters.continueOnError }}
-        condition: and(
-          succeeded(),
-          or(
-            and(
-              eq(variables['Agent.Os'], 'Windows_NT'),
-              in(variables['_SignType'], 'real', 'test')
-            ),
-            and(
-              ${{ eq(parameters.enableMicrobuildForMacAndLinux, true) }},
-              ne(variables['Agent.Os'], 'Windows_NT'),
-              eq(variables['_SignType'], 'real')
-            )
-          ))
+        condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT'))
 
   - ${{ if and(eq(parameters.runAsPublic, 'false'), eq(variables['System.TeamProject'], 'internal')) }}:
     - task: NuGetAuthenticate@1
@@ -187,19 +171,7 @@ jobs:
     - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
       - task: MicroBuildCleanup@1
         displayName: Execute Microbuild cleanup tasks
-        condition: and(
-          always(),
-          or(
-            and(
-              eq(variables['Agent.Os'], 'Windows_NT'),
-              in(variables['_SignType'], 'real', 'test')
-            ),
-            and(
-              ${{ eq(parameters.enableMicrobuildForMacAndLinux, true) }},
-              ne(variables['Agent.Os'], 'Windows_NT'),
-              eq(variables['_SignType'], 'real')
-            )
-          ))
+        condition: and(always(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT'))
         continueOnError: ${{ parameters.continueOnError }}
         env:
           TeamName: $(_TeamName)

diff --git a/eng/common/core-templates/job/source-index-stage1.yml b/eng/common/core-templates/job/source-index-stage1.yml
@@ -1,5 +1,8 @@
 parameters:
   runAsPublic: false
+  sourceIndexUploadPackageVersion: 2.0.0-20240522.1
+  sourceIndexProcessBinlogPackageVersion: 1.0.1-20240522.1
+  sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json
   sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci"
   preSteps: []
   binlogPath: artifacts/log/Debug/Build.binlog
@@ -13,6 +16,12 @@ jobs:
   dependsOn: ${{ parameters.dependsOn }}
   condition: ${{ parameters.condition }}
   variables:
+  - name: SourceIndexUploadPackageVersion
+    value: ${{ parameters.sourceIndexUploadPackageVersion }}
+  - name: SourceIndexProcessBinlogPackageVersion
+    value: ${{ parameters.sourceIndexProcessBinlogPackageVersion }}
+  - name: SourceIndexPackageSource
+    value: ${{ parameters.sourceIndexPackageSource }}
   - name: BinlogPath
     value: ${{ parameters.binlogPath }}
   - template: /eng/common/core-templates/variables/pool-providers.yml
@@ -25,20 +34,48 @@ jobs:
     pool:
       ${{ if eq(variables['System.TeamProject'], 'public') }}:
         name: $(DncEngPublicBuildPool)
-        image: windows.vs2022.amd64.open
+        image: 1es-windows-2022-open
+        os: windows
       ${{ if eq(variables['System.TeamProject'], 'internal') }}:
         name: $(DncEngInternalBuildPool)
-        image: windows.vs2022.amd64
+        image: 1es-windows-2022
+        os: windows
 
   steps:
   - ${{ if eq(parameters.is1ESPipeline, '') }}:
     - 'Illegal entry point, is1ESPipeline is not defined. Repository yaml should not directly reference templates in core-templates folder.': error
 
   - ${{ each preStep in parameters.preSteps }}:
     - ${{ preStep }}
+
+  - task: UseDotNet@2
+    displayName: Use .NET 8 SDK
+    inputs:
+      packageType: sdk
+      version: 8.0.x
+      installationPath: $(Agent.TempDirectory)/dotnet
+      workingDirectory: $(Agent.TempDirectory)
+
+  - script: |
+      $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(sourceIndexProcessBinlogPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools
+      $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(sourceIndexUploadPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools
+    displayName: Download Tools
+    # Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk.
+    workingDirectory: $(Agent.TempDirectory)
+
   - script: ${{ parameters.sourceIndexBuildCommand }}
     displayName: Build Repository
 
-  - template: /eng/common/core-templates/steps/source-index-stage1-publish.yml
-    parameters:
-      binLogPath: ${{ parameters.binLogPath }}
+  - script: $(Agent.TempDirectory)/.source-index/tools/BinLogToSln -i $(BinlogPath) -r $(Build.SourcesDirectory) -n $(Build.Repository.Name) -o .source-index/stage1output
+    displayName: Process Binlog into indexable sln
+
+  - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+    - task: AzureCLI@2
+      displayName: Log in to Azure and upload stage1 artifacts to source index
+      inputs:
+        azureSubscription: 'SourceDotNet Stage1 Publish'
+        addSpnToEnvironment: true
+        scriptType: 'ps'
+        scriptLocation: 'inlineScript'
+        inlineScript: |
+          $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) -s netsourceindexstage1 -b stage1
diff --git a/eng/common/core-templates/steps/get-delegation-sas.yml b/eng/common/core-templates/steps/get-delegation-sas.yml
@@ -31,7 +31,16 @@ steps:
       # Calculate the expiration of the SAS token and convert to UTC
       $expiry = (Get-Date).AddHours(${{ parameters.expiryInHours }}).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ")
 
-      $sas = az storage container generate-sas --account-name ${{ parameters.storageAccount }} --name ${{ parameters.container }} --permissions ${{ parameters.permissions }} --expiry $expiry --auth-mode login --as-user -o tsv
+      # Temporarily work around a helix issue where SAS tokens with / in them will cause incorrect downloads
+      # of correlation payloads. https://github.com/dotnet/dnceng/issues/3484
+      $sas = ""
+      do {
+        $sas = az storage container generate-sas --account-name ${{ parameters.storageAccount }} --name ${{ parameters.container }} --permissions ${{ parameters.permissions }} --expiry $expiry --auth-mode login --as-user -o tsv
+        if ($LASTEXITCODE -ne 0) {
+          Write-Error "Failed to generate SAS token."
+          exit 1
+        }
+      } while($sas.IndexOf('/') -ne -1)
 
       if ($LASTEXITCODE -ne 0) {
         Write-Error "Failed to generate SAS token."

diff --git a/eng/common/core-templates/steps/publish-logs.yml b/eng/common/core-templates/steps/publish-logs.yml
@@ -34,9 +34,7 @@ steps:
       '$(akams-client-id)'
       '$(microsoft-symbol-server-pat)'
       '$(symweb-symbol-server-pat)'
-      '$(dnceng-symbol-server-pat)'
       '$(dn-bot-all-orgs-build-rw-code-rw)'
-      '$(System.AccessToken)'
       ${{parameters.CustomSensitiveDataList}}
   continueOnError: true
   condition: always()
@@ -47,7 +45,6 @@ steps:
     SourceFolder: '$(Build.SourcesDirectory)/PostBuildLogs'
     Contents: '**'
     TargetFolder: '$(Build.ArtifactStagingDirectory)/PostBuildLogs'
-  condition: always()
 
 - template: /eng/common/core-templates/steps/publish-build-artifacts.yml
   parameters:

diff --git a/eng/common/core-templates/steps/source-index-stage1-publish.yml b/eng/common/core-templates/steps/source-index-stage1-publish.yml
diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh
@@ -66,7 +66,6 @@ __UbuntuPackages+=" libcurl4-openssl-dev"
 __UbuntuPackages+=" libkrb5-dev"
 __UbuntuPackages+=" libssl-dev"
 __UbuntuPackages+=" zlib1g-dev"
-__UbuntuPackages+=" libbrotli-dev"
 
 __AlpinePackages+=" curl-dev"
 __AlpinePackages+=" krb5-dev"
@@ -92,18 +91,18 @@ __HaikuPackages="gcc_syslibs"
 __HaikuPackages+=" gcc_syslibs_devel"
 __HaikuPackages+=" gmp"
 __HaikuPackages+=" gmp_devel"
-__HaikuPackages+=" icu[0-9]+"
-__HaikuPackages+=" icu[0-9]*_devel"
+__HaikuPackages+=" icu66"
+__HaikuPackages+=" icu66_devel"
 __HaikuPackages+=" krb5"
 __HaikuPackages+=" krb5_devel"
 __HaikuPackages+=" libiconv"
 __HaikuPackages+=" libiconv_devel"
-__HaikuPackages+=" llvm[0-9]*_libunwind"
-__HaikuPackages+=" llvm[0-9]*_libunwind_devel"
+__HaikuPackages+=" llvm12_libunwind"
+__HaikuPackages+=" llvm12_libunwind_devel"
 __HaikuPackages+=" mpfr"
 __HaikuPackages+=" mpfr_devel"
-__HaikuPackages+=" openssl3"
-__HaikuPackages+=" openssl3_devel"
+__HaikuPackages+=" openssl"
+__HaikuPackages+=" openssl_devel"
 __HaikuPackages+=" zlib"
 __HaikuPackages+=" zlib_devel"
 
@@ -497,7 +496,7 @@ if [[ "$__CodeName" == "alpine" ]]; then
     arch="$(uname -m)"
 
     ensureDownloadTool
-
+    
     if [[ "$__hasWget" == 1 ]]; then
         wget -P "$__ApkToolsDir" "https://gitlab.alpinelinux.org/api/v4/projects/5/packages/generic/v$__ApkToolsVersion/$arch/apk.static"
     else
@@ -682,7 +681,7 @@ elif [[ "$__CodeName" == "haiku" ]]; then
 
     ensureDownloadTool
 
-    echo "Downloading Haiku package tools"
+    echo "Downloading Haiku package tool"
     git clone https://github.com/haiku/haiku-toolchains-ubuntu --depth 1 "$__RootfsDir/tmp/script"
     if [[ "$__hasWget" == 1 ]]; then
         wget -O "$__RootfsDir/tmp/download/hosttools.zip" "$("$__RootfsDir/tmp/script/fetch.sh" --hosttools)"
@@ -692,42 +691,34 @@ elif [[ "$__CodeName" == "haiku" ]]; then
 
     unzip -o "$__RootfsDir/tmp/download/hosttools.zip" -d "$__RootfsDir/tmp/bin"
 
-    HaikuBaseUrl="https://eu.hpkg.haiku-os.org/haiku/master/$__HaikuArch/current"
-    HaikuPortsBaseUrl="https://eu.hpkg.haiku-os.org/haikuports/master/$__HaikuArch/current"
-
-    echo "Downloading HaikuPorts package repository index..."
-    if [[ "$__hasWget" == 1 ]]; then
-        wget -P "$__RootfsDir/tmp/download" "$HaikuPortsBaseUrl/repo"
-    else
-        curl -SLO --create-dirs --output-dir "$__RootfsDir/tmp/download" "$HaikuPortsBaseUrl/repo"
-    fi
+    DepotBaseUrl="https://depot.haiku-os.org/__api/v2/pkg/get-pkg"
+    HpkgBaseUrl="https://eu.hpkg.haiku-os.org/haiku/master/$__HaikuArch/current"
 
+    # Download Haiku packages
     echo "Downloading Haiku packages"
     read -ra array <<<"$__HaikuPackages"
     for package in "${array[@]}"; do
         echo "Downloading $package..."
-        hpkgFilename="$(LD_LIBRARY_PATH="$__RootfsDir/tmp/bin" "$__RootfsDir/tmp/bin/package_repo" list -f "$__RootfsDir/tmp/download/repo" |
-            grep -E "${package}-" | sort -V | tail -n 1 | xargs)"
-        if [ -z "$hpkgFilename" ]; then
-            >&2 echo "ERROR: package $package missing."
-            exit 1
-        fi
-        echo "Resolved filename: $hpkgFilename..."
-        hpkgDownloadUrl="$HaikuPortsBaseUrl/packages/$hpkgFilename"
+        # API documented here: https://github.com/haiku/haikudepotserver/blob/master/haikudepotserver-api2/src/main/resources/api2/pkg.yaml#L60
+        # The schema here: https://github.com/haiku/haikudepotserver/blob/master/haikudepotserver-api2/src/main/resources/api2/pkg.yaml#L598
         if [[ "$__hasWget" == 1 ]]; then
+            hpkgDownloadUrl="$(wget -qO- --post-data '{"name":"'"$package"'","repositorySourceCode":"haikuports_'$__HaikuArch'","versionType":"LATEST","naturalLanguageCode":"en"}' \
+                --header 'Content-Type:application/json' "$DepotBaseUrl" | jq -r '.result.versions[].hpkgDownloadURL')"
             wget -P "$__RootfsDir/tmp/download" "$hpkgDownloadUrl"
         else
+            hpkgDownloadUrl="$(curl -sSL -XPOST --data '{"name":"'"$package"'","repositorySourceCode":"haikuports_'$__HaikuArch'","versionType":"LATEST","naturalLanguageCode":"en"}' \
+                --header 'Content-Type:application/json' "$DepotBaseUrl" | jq -r '.result.versions[].hpkgDownloadURL')"
             curl -SLO --create-dirs --output-dir "$__RootfsDir/tmp/download" "$hpkgDownloadUrl"
         fi
     done
     for package in haiku haiku_devel; do
         echo "Downloading $package..."
         if [[ "$__hasWget" == 1 ]]; then
-            hpkgVersion="$(wget -qO- "$HaikuBaseUrl" | sed -n 's/^.*version: "\([^"]*\)".*$/\1/p')"
-            wget -P "$__RootfsDir/tmp/download" "$HaikuBaseUrl/packages/$package-$hpkgVersion-1-$__HaikuArch.hpkg"
+            hpkgVersion="$(wget -qO- "$HpkgBaseUrl" | sed -n 's/^.*version: "\([^"]*\)".*$/\1/p')"
+            wget -P "$__RootfsDir/tmp/download" "$HpkgBaseUrl/packages/$package-$hpkgVersion-1-$__HaikuArch.hpkg"
         else
-            hpkgVersion="$(curl -sSL "$HaikuBaseUrl" | sed -n 's/^.*version: "\([^"]*\)".*$/\1/p')"
-            curl -SLO --create-dirs --output-dir "$__RootfsDir/tmp/download" "$HaikuBaseUrl/packages/$package-$hpkgVersion-1-$__HaikuArch.hpkg"
+            hpkgVersion="$(curl -sSL "$HpkgBaseUrl" | sed -n 's/^.*version: "\([^"]*\)".*$/\1/p')"
+            curl -SLO --create-dirs --output-dir "$__RootfsDir/tmp/download" "$HpkgBaseUrl/packages/$package-$hpkgVersion-1-$__HaikuArch.hpkg"
         fi
     done
 

diff --git a/eng/common/sdk-task.ps1 b/eng/common/sdk-task.ps1
@@ -64,7 +64,7 @@ try {
       $GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.5`" }") -MemberType NoteProperty
     }
     if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) {
-      $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "17.10.0-pre.4.0" -MemberType NoteProperty
+      $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "17.12.0" -MemberType NoteProperty
     }
     if ($GlobalJson.tools."xcopy-msbuild".Trim() -ine "none") {
         $xcopyMSBuildToolsFolder = InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true