Updating .sh files and other changes.

* Synced or added .sh files including'run-test.sh' even though we don't currently use it, we may need to at some point. * .sh files not tested on Linux. * Added the publish-packages files but did not test them. * Pulled in a few more changes to various files from PRs in corefx occuring within the last 24 hrs.
OliverKingOk · Apr 15, 2016 · aef3027 · aef3027
1 parent 5aea308
commit aef3027
Show file tree

Hide file tree

Showing 19 changed files with 472 additions and 121 deletions.
diff --git a/Packaging.props b/Packaging.props
@@ -29,13 +29,13 @@
 
   <!-- Add required legal files to packages -->
   <ItemGroup Condition="'$(MSBuildProjectExtension)' == '.pkgproj'">
-    <File Condition="Exists('$(PackageLicenseFile)')"
+    <File Condition="Exists('$(PackageLicenseFile)')" 
 	  	    Include="$(PackageLicenseFile)" >
-      <SkipPackageFileCheck>true</SkipPackageFileCheck>
-    </File>
+        <SkipPackageFileCheck>true</SkipPackageFileCheck>
+    </File>        
     <File Condition="Exists('$(PackageThirdPartyNoticesFile)')"
 	        Include="$(PackageThirdPartyNoticesFile)" >
-      <SkipPackageFileCheck>true</SkipPackageFileCheck>
-    </File>
+        <SkipPackageFileCheck>true</SkipPackageFileCheck>
+    </File>        
   </ItemGroup>
 </Project>
diff --git a/build-packages.sh b/build-packages.sh
@@ -0,0 +1,74 @@
+#!/usr/bin/env bash
+
+usage()
+{
+    echo "Builds the NuGet packages from the binaries that were built in the Build product binaries step."
+    echo "No option parameters."
+    exit 1
+}
+
+working_tree_root="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+build_packages_log=$working_tree_root/build-packages.log
+binclashlog=$working_tree_root/binclash.log
+binclashloggerdll=$working_tree_root/Tools/Microsoft.DotNet.Build.Tasks.dll
+RuntimeOS=ubuntu.$VERSION_ID
+
+# Use uname to determine what the OS is.
+OSName=$(uname -s)
+case $OSName in
+    Darwin)
+        # Darwin version can be three sets of digits (e.g. 10.10.3), we want just the first two
+        DarwinVersion=$(sw_vers -productVersion | awk 'match($0, /[0-9]+\.[0-9]+/) { print substr($0, RSTART, RLENGTH) }')
+        RuntimeOS=osx.$DarwinVersion
+        ;;
+
+    FreeBSD|NetBSD)
+        # TODO this doesn't seem correct
+        RuntimeOS=osx.10.10
+        ;;
+
+    Linux)
+        source /etc/os-release
+        VersionMajor=$(echo $VERSION_ID | awk 'match($0, /[0-9]+/) { print substr($0, RSTART, RLENGTH) }')
+        if [ "$ID" == "rhel" ]; then
+            RuntimeOS=rhel.$VersionMajor
+        elif [ "$ID" == "debian" ]; then
+            RuntimeOS=debian.$VersionMajor
+        elif [ "$ID" == "ubuntu" ]; then
+            RuntimeOS=ubuntu.$VERSION_ID
+        else
+            echo "Unsupported Linux distribution '$ID' detected. Configuring as if for Ubuntu."
+        fi
+        ;;
+
+    *)
+        echo "Unsupported OS '$OSName' detected. Configuring as if for Ubuntu."
+        ;;
+esac
+
+options="/m /nologo /v:minimal /clp:Summary /flp:v=diagnostic;Append;LogFile=$build_packages_log /l:BinClashLogger,$binclashloggerdll;LogFile=$binclashlog /p:FilterToOSGroup=$RuntimeOS"
+allargs="$@"
+
+echo -e "Running build-packages.sh $allargs" > $build_packages_log
+
+if [ "$allargs" == "-h" ] || [ "$allargs" == "--help" ]; then
+    usage
+fi
+
+# Ensure that MSBuild is available
+echo "Running init-tools.sh"
+$working_tree_root/init-tools.sh
+
+echo -e "\n$working_tree_root/Tools/corerun $working_tree_root/Tools/MSBuild.exe $working_tree_root/src/packages.builds $options $allargs" >> $build_packages_log
+$working_tree_root/Tools/corerun $working_tree_root/Tools/MSBuild.exe $working_tree_root/src/packages.builds $options $allargs
+
+
+if [ $? -ne 0 ]; then
+    echo -e "\nAn error occurred. Aborting build-packages.sh ." >> $build_packages_log
+    echo "ERROR: An error occurred while building packages, see $build_packages_log for more details."
+    exit 1
+fi
+
+echo "Done building packages."
+echo -e "\nDone building packages." >> $build_packages_log
+exit 0
diff --git a/build-tests.sh b/build-tests.sh
@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+
+usage()
+{
+    echo "Builds the tests that are in the repository."
+    echo "No option parameters."
+    exit 1
+}
+
+working_tree_root="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+build_tests_log=$working_tree_root/build-tests.log
+binclashlog=$working_tree_root/binclash.log
+binclashloggerdll=$working_tree_root/Tools/Microsoft.DotNet.Build.Tasks.dll
+
+options="/m /nologo /v:minimal /clp:Summary /flp:v=detailed;Append;LogFile=$build_tests_log /l:BinClashLogger,$binclashloggerdll;LogFile=$binclashlog"
+allargs="$@"
+
+echo -e "Running build-tests.sh $allargs" > $build_tests_log
+
+if [ "$allargs" == "-h" ] || [ "$allargs" == "--help" ]; then
+    usage
+fi
+
+# Ensure that MSBuild is available
+echo "Running init-tools.sh"
+$working_tree_root/init-tools.sh
+
+echo -e "\n$working_tree_root/Tools/corerun $working_tree_root/Tools/MSBuild.exe $working_tree_root/src/tests.builds $options $allargs" >> $build_tests_log
+$working_tree_root/Tools/corerun $working_tree_root/Tools/MSBuild.exe $working_tree_root/src/tests.builds $options $allargs
+
+
+if [ $? -ne 0 ]; then
+    echo -e "\nAn error occurred. Aborting build-tests.sh ." >> $build_tests_log
+    echo "ERROR: An error occurred while building tests, see $build_tests_log for more details."
+    exit 1
+fi
+
+echo "Done building tests."
+echo -e "\nDone building tests." >> $build_tests_log
+exit 0
diff --git a/build.sh b/build.sh
@@ -7,14 +7,15 @@ usage()
     echo "native - optional argument to build the native code"
     echo "The following arguments affect native builds only:"
     echo "BuildArch can be: x64, x86, arm, arm64"
-    echo "BuildType can be: Debug, Release"
+    echo "BuildType can be: debug, release"
     echo "clean - optional argument to force a clean build."
     echo "verbose - optional argument to enable verbose build output."
     echo "clangx.y - optional argument to build using clang version x.y."
     echo "platform can be: FreeBSD, Linux, NetBSD, OSX, Windows"
     echo "cross - optional argument to signify cross compilation,"
     echo "      - will use ROOTFS_DIR environment variable if set."
     echo "skiptests - skip the tests in the './bin/*/*Tests/' subdirectory."
+    echo "generateversion - if building native only, pass this in to get a version on the build output."
     echo "cmakeargs - user-settable additional arguments passed to CMake."
     exit 1
 }
@@ -99,6 +100,25 @@ prepare_native_build()
     if [ $__VerboseBuild == 1 ]; then
         export VERBOSE=1
     fi
+
+    # If managed build is supported, then generate version
+    if [ $__buildmanaged == true ]; then
+        __generateversionsource=true
+    fi
+
+    # Ensure tools are present if we will generate version.c
+    if [ $__generateversionsource == true ]; then
+        $__scriptpath/init-tools.sh
+    fi
+
+    # Generate version.c if specified, else have an empty one.
+    __versionSourceFile=$__scriptpath/bin/obj/version.c
+    if [ $__generateversionsource == true ]; then
+        $__scriptpath/Tools/corerun $__scriptpath/Tools/MSBuild.exe "$__scriptpath/build.proj" /t:GenerateVersionSourceFile /p:NativeVersionSourceFile=$__scriptpath/bin/obj/version.c /p:GenerateVersionSourceFile=true /v:minimal
+    else
+        __versionSourceLine="static char sccsid[] __attribute__((used)) = \"@(#)No version information produced\";"
+        echo $__versionSourceLine > $__versionSourceFile
+    fi
 }
 
 build_managed()
@@ -108,7 +128,7 @@ build_managed()
     __binclashlog=$__scriptpath/binclash.log
     __binclashloggerdll=$__scriptpath/Tools/Microsoft.DotNet.Build.Tasks.dll
 
-    $__scriptpath/Tools/corerun $__scriptpath/Tools/MSBuild.exe "$__buildproj" /nologo /verbosity:minimal "/fileloggerparameters:Verbosity=normal;LogFile=$__buildlog" "/l:BinClashLogger,$__binclashloggerdll;LogFile=$__binclashlog" /t:Build /p:ConfigurationGroup=$__BuildType /p:OSGroup=$__BuildOS /p:SkipTests=$__SkipTests /p:COMPUTERNAME=$(hostname) /p:USERNAME=$(id -un) /p:TestNugetRuntimeId=$__TestNugetRuntimeId $__UnprocessedBuildArgs
+    $__scriptpath/Tools/corerun $__scriptpath/Tools/MSBuild.exe "$__buildproj" /m /nologo /verbosity:minimal "/fileloggerparameters:Verbosity=normal;LogFile=$__buildlog" "/l:BinClashLogger,$__binclashloggerdll;LogFile=$__binclashlog" /t:Build /p:ConfigurationGroup=$__BuildType /p:OSGroup=$__BuildOS /p:SkipTests=$__SkipTests /p:COMPUTERNAME=$(hostname) /p:USERNAME=$(id -un) /p:TestNugetRuntimeId=$__TestNugetRuntimeId $__UnprocessedBuildArgs
     BUILDERRORLEVEL=$?
 
     echo
@@ -169,6 +189,7 @@ __nugetpath=$__packageroot/NuGet.exe
 __nugetconfig=$__sourceroot/NuGet.Config
 __rootbinpath="$__scriptpath/bin"
 __msbuildpackageid="Microsoft.Build.Mono.Debug"
+__generateversionsource=false
 __msbuildpackageversion="14.1.0.0-prerelease"
 __msbuildpath=$__packageroot/$__msbuildpackageid.$__msbuildpackageversion/lib/MSBuild.exe
 __buildmanaged=false
@@ -306,6 +327,9 @@ while :; do
         verbose)
             __VerboseBuild=1
             ;;
+        generateversion)
+            __generateversionsource=true
+            ;;
         clang3.5)
             __ClangMajorVersion=3
             __ClangMinorVersion=5

diff --git a/clean.sh b/clean.sh
@@ -28,13 +28,15 @@ check_exit_status()
 
 WorkingTreeRoot="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 CleanLog=$WorkingTreeRoot/clean.log
+
+options="/nologo /v:minimal /clp:Summary /flp:v=detailed;Append;LogFile=$CleanLog"
+unprocessedBuildArgs=
 CleanSuccessful=true
 CleanTargets=
 
-# Parse arguments
-
 echo "Running clean.sh $*" > $CleanLog
 
+# Parse arguments
 if [ $# == 0 ]
 then
     CleanTargets="Clean;"
@@ -67,10 +69,7 @@ do
         CleanTargets="Clean;CleanPackages;CleanPackagesCache;"
         ;;
         *)
-        echo "Unrecognized argument '$opt'"
-        echo "Use 'clean -h' for help."
-        exit 1
-        ;;
+        unprocessedBuildArgs="$unprocessedBuildArgs $1"
     esac
     shift
 done
@@ -85,8 +84,8 @@ then
     CleanTargetsTrimmed=${CleanTargets:0:${#CleanTargets}-1}
 
     echo "Running MSBuild target(s): $CleanTargetsTrimmed"
-    echo -e "\n$WorkingTreeRoot/Tools/corerun $WorkingTreeRoot/Tools/MSBuild.exe $WorkingTreeRoot/build.proj /t:$CleanTargetsTrimmed /nologo /verbosity:minimal /flp:v=detailed;Append;LogFile=$CleanLog" >> $CleanLog
-    $WorkingTreeRoot/Tools/corerun $WorkingTreeRoot/Tools/MSBuild.exe $WorkingTreeRoot/build.proj /t:$CleanTargetsTrimmed /nologo /verbosity:minimal "/flp:v=detailed;Append;LogFile=$CleanLog"
+    echo -e "\n$WorkingTreeRoot/Tools/corerun $WorkingTreeRoot/Tools/MSBuild.exe $WorkingTreeRoot/build.proj /t:$CleanTargetsTrimmed $options $unprocessedBuildArgs" >> $CleanLog
+    $WorkingTreeRoot/Tools/corerun $WorkingTreeRoot/Tools/MSBuild.exe $WorkingTreeRoot/build.proj /t:$CleanTargetsTrimmed $options $unprocessedBuildArgs
     check_exit_status
 fi
 

diff --git a/dir.props b/dir.props
@@ -1,3 +1,4 @@
+<?xml version="1.0" encoding="utf-8"?>
 <Project ToolsVersion="12.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <Import Condition="Exists('..\dir.props')" Project="..\dir.props" />
 
@@ -110,16 +111,23 @@
 
   <!-- list of nuget package sources passed to dnu -->
   <ItemGroup Condition="'$(ExcludeInternetFeeds)' != 'true'">
+    <!-- PackagesDrops is passed in via the commandline "/p:PackagesDrops=[drop location]" -->
+    <DnuSourceList Condition="'$(BuildTestsAgainstPackages)' == 'true'" Include="$(PackagesDrops)" />
     <!-- Need to escape double forward slash (%2F) or MSBuild will normalize to one slash on Unix. -->
-
     <DnuSourceList Include="https:%2F%2Fdotnet.myget.org/F/dotnet-core/api/v3/index.json" />
     <DnuSourceList Include="https:%2F%2Fwww.nuget.org/api/v2/" />
   </ItemGroup>
 
+  <!-- This is the directory where we dynamically generate project.json's for our test build package dependencies -->
+  <PropertyGroup Condition="'$(BuildTestsAgainstPackages)' == 'true'">
+      <GeneratedProjectJsonDir>$(ObjDir)generated</GeneratedProjectJsonDir>
+  </PropertyGroup>
+
   <!-- list of directories to perform batch restore -->
   <ItemGroup>
     <DnuRestoreDir Include="$(MSBuildThisFileDirectory)/src" />
     <DnuRestoreDir Include="$(MSBuildThisFileDirectory)/pkg" />
+    <DnuRestoreDir Condition="'$(BuildTestsAgainstPackages)' == 'true'" Include="$(GeneratedProjectJsonDir)" />
   </ItemGroup>
 
   <PropertyGroup>

diff --git a/dir.targets b/dir.targets
@@ -13,6 +13,10 @@
   <Target Name="RebuildAndTest" DependsOnTargets="Rebuild;Test" />
   <Target Name="Test" />
 
+  <!-- Targets will be overridden if buildagainstpackages.targets is imported. -->
+  <Target Name="GenerateTestProjectJson" />
+  <Target Name="GenerateAllTestProjectJsons" />
+
   <Import Project="$(ToolsDir)/Build.Common.targets" />
 
   <!-- permit a wrapping build system to contribute targets to this build -->

diff --git a/dir.traversal.targets b/dir.traversal.targets
@@ -178,6 +178,36 @@
     <Error Condition="'$(MSBuildLastTaskResult)'=='false'" />
   </Target>
 
+  <Target Name="GenerateAllTestProjectJsons">
+    <PropertyGroup>
+      <DefaultGenerateAllTestProjectJsonsTarget Condition="'$(DefaultGenerateAllTestProjectJsonsTarget)' == ''">GenerateAllTestProjectJsons</DefaultGenerateAllTestProjectJsonsTarget>
+    </PropertyGroup>
+
+    <ItemGroup>
+      <GeneratedProject Include="@(Project)">
+        <AdditionalProperties>GeneratedTargetGroup=%(Project.TargetGroup);GeneratedOSGroup=%(Project.OSGroup);%(Project.AdditionalProperties)</AdditionalProperties>
+      </GeneratedProject>
+    </ItemGroup>
+
+    <!-- To Serialize we use msbuild's batching functionality '%' to force it to batch all similar projects with the same identity
+         however since the project names are unique it will essentially force each to run in its own batch -->
+    <MSBuild Targets="$(DefaultGenerateAllTestProjectJsonsTarget)"
+             Projects="@(GeneratedProject)"
+             Condition="'$(SerializeProjects)' != 'true'"
+             Properties="GenerateAllTestProjectJsons=$(DefaultGenerateAllTestProjectJsonsTarget)"
+             BuildInParallel="true"
+             ContinueOnError="ErrorAndContinue" />
+
+    <MSBuild Targets="$(DefaultGenerateAllTestProjectJsonsTarget)"
+             Projects="@(GeneratedProject)"
+             Condition="'$(SerializeProjects)' == 'true' AND '%(Identity)' != ''"
+             Properties="GenerateAllTestProjectJsons=$(DefaultGenerateAllTestProjectJsonsTarget)"
+             ContinueOnError="ErrorAndContinue" />
+
+    <!-- Given we ErrorAndContinue we need to propagate the error if the overall task failed -->
+    <Error Condition="'$(MSBuildLastTaskResult)'=='false'" />
+  </Target> 
+
   <PropertyGroup>
     <TraversalBuildDependsOn>
       BuildAllProjects;
@@ -189,10 +219,20 @@
       $(TraversalCleanDependsOn);
     </TraversalCleanDependsOn>
 
+    <TraversalGenerateTestProjectJsonsDependsOn>
+      GenerateAllTestProjectJsons;
+      $(TraversalGenerateTestProjectJsonsDependsOn)
+    </TraversalGenerateTestProjectJsonsDependsOn>
+
     <TraversalRestorePackagesDependsOn>
       RestoreAllProjectPackages;
       $(TraversalRestorePackagesDependsOn)
     </TraversalRestorePackagesDependsOn>
+
+    <TraversalRestorePackagesDependsOn Condition="'$(_BuildAgainstPackages)' == 'true'">
+      $(TraversalGenerateTestProjectJsonsDependsOn);
+      $(TraversalRestorePackagesDependsOn)
+    </TraversalRestorePackagesDependsOn>
   </PropertyGroup>
 
   <Target Name="Build" DependsOnTargets="$(TraversalBuildDependsOn)" />
@@ -206,4 +246,7 @@
   <Target Name="RebuildAndTest" DependsOnTargets="Rebuild;Test" />
   <Target Name="Test" />
 
+  <!-- Target will be overridden if buildagainstpackages.targets is imported. -->
+  <Target Name="GenerateTestProjectJson" />
+  <Import Condition="'$(_BuildAgainstPackages)' == 'true'" Project="$(ToolsDir)/buildagainstpackages.targets" />
 </Project>
diff --git a/global.json b/global.json
@@ -0,0 +1,3 @@
+{
+  "projects": [ "./src", "./src/Common" ]
+}
diff --git a/init-tools.sh b/init-tools.sh
@@ -19,6 +19,7 @@ case $OSName in
     Darwin)
         OS=OSX
         __DOTNET_PKG=dotnet-dev-osx-x64
+        ulimit -n 2048
         ;;
 
     Linux)

diff --git a/publish-packages.cmd b/publish-packages.cmd
@@ -0,0 +1,33 @@
+@if "%_echo%" neq "on" echo off
+setlocal EnableDelayedExpansion
+
+set packagesLog=publish-packages.log
+echo Running publish-packages.cmd %* > %packagesLog%
+
+set options=/nologo /v:minimal /flp:v=detailed;Append;LogFile=%packagesLog%
+set allargs=%*
+
+if /I [%1] == [/?] goto Usage
+if /I [%1] == [/help] goto Usage
+
+REM ensure that msbuild is available
+echo Running init-tools.cmd
+call %~dp0init-tools.cmd
+
+echo msbuild.exe %~dp0src\publish.builds !options! !allargs! >> %packagesLog%
+call msbuild.exe %~dp0src\publish.builds !options! !allargs!
+if NOT [%ERRORLEVEL%]==[0] (
+  echo ERROR: An error occurred while publishing packages, see %packagesLog% for more details.
+  exit /b
+)
+
+echo Done publishing packages.
+exit /b
+
+:Usage
+echo.
+echo Publishes the NuGet packages to the specified location.
+echo For publishing to Azure the following properties are required.
+echo   /p:CloudDropAccountName="account name"
+echo   /p:CloudDropAccessToken="access token"
+exit /b
-Original file line number
+Diff line change
@@ Expand Up / @@ -19,6 +19,7 @@ case $OSName in @@
         Darwin)
             OS=OSX
             __DOTNET_PKG=dotnet-dev-osx-x64
+            ulimit -n 2048
             ;;
         Linux)
@@ Expand Down @@