Update common Docker engineering infrastructure with latest (dotnet#3555

)
anluoma · Mar 14, 2022 · ffac3ff · ffac3ff
1 parent cb79325
commit ffac3ff
Show file tree

Hide file tree

Showing 5 changed files with 70 additions and 34 deletions.
diff --git a/eng/common/templates/jobs/build-images.yml b/eng/common/templates/jobs/build-images.yml
@@ -25,6 +25,7 @@ jobs:
   variables:
     imageBuilderDockerRunExtraOptions: $(build.imageBuilderDockerRunExtraOptions)
     versionsRepoPath: versions
+    sbomDirectory: $(Build.ArtifactStagingDirectory)/sbom
     ${{ if eq(parameters.noCache, false) }}:
       versionsBasePath: $(versionsRepoPath)/
       pipelineDisabledCache: false
@@ -95,7 +96,7 @@ jobs:
 
       echo "##vso[task.setvariable variable=imageBuilderBuildArgs]$imageBuilderBuildArgs"
     displayName: Set Image Builder Build Args
-  - script: >
+  - powershell: >
       $(runImageBuilderCmd) build
       --manifest $(manifest)
       $(imageBuilderPaths)
@@ -105,14 +106,54 @@ jobs:
       --retry
       --source-repo $(publicGitRepoUri)
       --get-installed-pkgs-path $(baseContainerRepoPath)/$(engCommonRelativePath)/package-scripts/get-installed-packages.sh
+      --digests-out-var 'builtImages'
       $(manifestVariables)
       $(imageBuilderBuildArgs)
+    name: BuildImages
     displayName: Build Images
   - publish: $(Build.ArtifactStagingDirectory)/$(legName)-image-info.json
     artifact: $(legName)-image-info-$(System.JobAttempt)
     displayName: Publish Image Info File Artifact
+  - ${{ if eq(variables['System.TeamProject'], parameters.internalProjectName) }}:
+      # Define the task here to load it into the agent so that we can invoke the tool manually
+    - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0
+      inputs:
+        BuildDropPath: $(Build.ArtifactStagingDirectory)
+      displayName: Load Manifest Generator
+      condition: and(succeeded(), ne(variables['BuildImages.builtImages'], ''), eq(variables.architecture, 'amd64'))
+    - powershell: |
+        $images = "$(BuildImages.builtImages)"
+        if (-not $images) { return 0 }
+        $taskDir = $(Get-ChildItem -Recurse -Directory -Filter "ManifestGeneratorTask*" -Path '$(Agent.WorkFolder)').FullName
+        $manifestToolDllPath = $(Get-ChildItem -Recurse -File -Filter "Microsoft.ManifestTool.dll" -Path $taskDir).FullName
+        $dotnetDir = $(Get-ChildItem -Recurse -Directory -Filter "dotnet-*" -Path $taskDir).FullName
+
+        # Call the manifest tool for each image to produce seperate SBOMs
+        # Manifest tool docs: https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/secure-supply-chain/custom-sbom-generation-workflows
+        $images -Split ',' | ForEach-Object {
+          echo "Generating SBOM for $_";
+          $formattedImageName = $_.Replace('$(acr.server)/$(stagingRepoPrefix)', "").Replace('/', '_').Replace(':', '_');
+          $sbomChildDir = "$(sbomDirectory)/$formattedImageName";
+          New-Item -Type Directory -Path $sbomChildDir > $null;
+          & "$dotnetDir/dotnet" "$manifestToolDllPath" `
+            Generate `
+            -BuildDropPath '$(Build.ArtifactStagingDirectory)' `
+            -BuildComponentPath '$(Agent.BuildDirectory)' `
+            -PackageName '.NET' `
+            -PackageVersion '$(Build.BuildNumber)' `
+            -ManifestDirPath $sbomChildDir `
+            -DockerImagesToScan $_ `
+            -Verbosity Information 
+        }
+      displayName: Generate SBOMs
+      condition: and(succeeded(), ne(variables['BuildImages.builtImages'], ''), eq(variables.architecture, 'amd64'))
   - ${{ if eq(variables['System.TeamProject'], parameters.publicProjectName) }}:
     - template: ${{ format('../steps/test-images-{0}-client.yml', parameters.dockerClientOS) }}
       parameters:
         condition: ne(variables.testScriptPath, '')
   - template: ${{ format('../steps/cleanup-docker-{0}.yml', parameters.dockerClientOS) }}
+  - ${{ if eq(variables['System.TeamProject'], parameters.internalProjectName) }}:
+    - publish: $(sbomDirectory)
+      artifact: $(legName)-sboms
+      displayName: Publish SBOM
+      condition: and(succeeded(), ne(variables['BuildImages.builtImages'], ''), eq(variables.architecture, 'amd64'))
diff --git a/eng/common/templates/jobs/post-build.yml b/eng/common/templates/jobs/post-build.yml
@@ -4,14 +4,31 @@ parameters:
 jobs:
 - job: Build
   pool: ${{ parameters.pool }}
+  variables:
+    imageInfosSubDir: "/image-infos"
+    sbomSubDir: "/sbom"
   steps:
   - template: ../steps/init-docker-linux.yml
   - template: ../steps/download-build-artifact.yml
     parameters:
       targetPath: $(Build.ArtifactStagingDirectory)
-  - pwsh: |
+  - powershell: |
+      # Move all image-info artifacts to their own directory
+      New-Item -ItemType Directory -Path $(Build.ArtifactStagingDirectory)$(imageInfosSubDir)
+      Get-ChildItem -Directory -Filter "*-image-info-*" $(Build.ArtifactStagingDirectory) |
+        Move-Item -Verbose -Destination $(Build.ArtifactStagingDirectory)$(imageInfosSubDir)
+    displayName: Collect Image Info Files
+  - powershell: |
+      # Move the contents of all the SBOM artifact directories to a single location
+      New-Item -ItemType Directory -Path $(Build.ArtifactStagingDirectory)$(sbomSubDir)
+      Get-ChildItem -Directory -Filter "*-sboms" $(Build.ArtifactStagingDirectory) |
+        ForEach-Object {
+          Get-ChildItem $_ -Directory | Move-Item -Verbose -Destination $(Build.ArtifactStagingDirectory)$(sbomSubDir)
+        }
+    displayName: Consolidate SBOMs to Single Directory
+  - powershell: |
       # Deletes the artifacts from all the unsuccessful jobs
-      Get-ChildItem $(Build.ArtifactStagingDirectory) -Directory |
+      Get-ChildItem $(Build.ArtifactStagingDirectory)$(imageInfosSubDir) -Directory |
           ForEach-Object {
               [pscustomobject]@{
                   # Parse the artifact name to separate the base of the name from the job attempt number 
@@ -32,10 +49,13 @@ jobs:
   - script: >
       $(runImageBuilderCmd) mergeImageInfo
       --manifest $(manifest)
-      $(artifactsPath)
-      $(artifactsPath)/image-info.json
+      $(artifactsPath)$(imageInfosSubDir)
+      $(artifactsPath)$(imageInfosSubDir)/image-info.json
       $(manifestVariables)
     displayName: Merge Image Info Files
-  - publish: $(Build.ArtifactStagingDirectory)/image-info.json
+  - publish: $(Build.ArtifactStagingDirectory)$(sbomSubDir)
+    artifact: sboms
+    displayName: Publish SBOM Artifact
+  - publish: $(Build.ArtifactStagingDirectory)$(imageInfosSubDir)/image-info.json
     artifact: image-info
     displayName: Publish Image Info File Artifact
diff --git a/eng/common/templates/stages/build-test-publish-repo.yml b/eng/common/templates/stages/build-test-publish-repo.yml
@@ -248,25 +248,6 @@ stages:
         testJobTimeout: ${{ parameters.windowsAmdTestJobTimeout }}
         internalProjectName: ${{ parameters.internalProjectName }}
 
-################################################################################
-# Generate SBOM
-################################################################################
-- ${{ if eq(variables['System.TeamProject'], parameters.internalProjectName) }}:
-  - stage: Generate_SBOM
-    dependsOn: Post_Build
-    condition: "
-      and(
-        contains(variables['stages'], 'sbom'),
-        or(
-          and(
-            succeeded(),
-            contains(variables['stages'], 'build')),
-          not(contains(variables['stages'], 'build'))))"
-    jobs:
-    - template: ../jobs/generate-sbom.yml
-      parameters:
-        pool: ${{ parameters.linuxAmd64Pool }}
-
 ################################################################################
 # Publish Images
 ################################################################################

diff --git a/eng/common/templates/stages/dotnet/build-test-publish-repo.yml b/eng/common/templates/stages/dotnet/build-test-publish-repo.yml
@@ -70,18 +70,12 @@ stages:
       ${{ if eq(variables['System.TeamProject'], parameters.publicProjectName) }}:
         name: DotNetCore-Docker-Public
       ${{ if eq(variables['System.TeamProject'], parameters.internalProjectName) }}:
-        name: DotNetCore-Docker
-      demands:
-      - Agent.OS -equals linux
-      - Agent.OSArchitecture -equals ARM64
+        name: Docker-Linux-Arm-Internal
     linuxArm32Pool:
       ${{ if eq(variables['System.TeamProject'], parameters.publicProjectName) }}:
         name: DotNetCore-Docker-Public
       ${{ if eq(variables['System.TeamProject'], parameters.internalProjectName) }}:
-        name: DotNetCore-Docker
-      demands:
-      - Agent.OS -equals linux
-      - Agent.OSArchitecture -equals ARM64
+        name: Docker-Linux-Arm-Internal
     windows2016Pool: Docker-2016-${{ variables['System.TeamProject'] }}
     windows1809Pool: Docker-1809-${{ variables['System.TeamProject'] }}
     windows20H2Pool: Docker-20H2-${{ variables['System.TeamProject'] }}

diff --git a/eng/common/templates/variables/docker-images.yml b/eng/common/templates/variables/docker-images.yml
@@ -1,5 +1,5 @@
 variables:
-  imageNames.imageBuilder: mcr.microsoft.com/dotnet-buildtools/image-builder:1630661
+  imageNames.imageBuilder: mcr.microsoft.com/dotnet-buildtools/image-builder:1656462
   imageNames.imageBuilder.withrepo: imagebuilder-withrepo:$(Build.BuildId)-$(System.JobId)
   imageNames.testRunner: mcr.microsoft.com/dotnet-buildtools/prereqs:debian-buster-slim-docker-testrunner-974165
   imageNames.testRunner.withrepo: testrunner-withrepo:$(Build.BuildId)-$(System.JobId)