Initial version of NBench

petabridge · Nov 16, 2015 · 3e94422 · 3e94422
1 parent bec0371
commit 3e94422
Show file tree

Hide file tree

Showing 117 changed files with 6,725 additions and 0 deletions.
diff --git a/.gitignore b/.gitignore
@@ -22,6 +22,9 @@ bld/
 [Bb]in/
 [Oo]bj/
 
+# FAKE output
+.fake/
+
 # Visual Studo 2015 cache/options directory
 .vs/
 
@@ -194,3 +197,6 @@ FakesAssemblies/
 
 # Visual Studio 6 workspace options file
 *.opt
+
+# NuGet 
+nuget.exe
diff --git a/.nuget/NuGet.Config b/.nuget/NuGet.Config
@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="utf-8"?>
+<configuration>
+  <solution>
+    <add key="disableSourceControlIntegration" value="true" />
+  </solution>
+</configuration>
diff --git a/NBench.sln b/NBench.sln
@@ -0,0 +1,64 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio 14
+VisualStudioVersion = 14.0.23107.0
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Benchmarks", "Benchmarks", "{C8104500-2C0F-46A2-AE84-A3D74CC55697}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{85C9F790-0322-4B2E-97EC-3581DA58A0AF}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NBench", "src\NBench\NBench.csproj", "{4E0B1852-7B6D-49E7-9939-A315D086B094}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NBench.Microbenchmarks", "benchmarks\NBench.Microbenchmarks\NBench.Microbenchmarks.csproj", "{FDA04BAB-34FD-4E5D-91F4-0503090973A6}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Tests", "Tests", "{1F1914B9-8F67-4B51-9738-CDE5AFB2F8DE}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NBench.Tests", "tests\NBench.Tests\NBench.Tests.csproj", "{5716EAF7-ABA1-434F-9AFA-51693A73C627}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NBench.Tests.End2End", "tests\NBench.Tests.End2End\NBench.Tests.End2End.csproj", "{B7F38539-67A1-42B8-A5E5-CB592B6E3233}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NBench.Runner", "src\NBench.Runner\NBench.Runner.csproj", "{5C586501-4E91-4007-B23F-02E20014DA51}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Build", "Build", "{97444074-C626-443F-BE96-374AD2E43604}"
+	ProjectSection(SolutionItems) = preProject
+		build.cmd = build.cmd
+		build.fsx = build.fsx
+		build.sh = build.sh
+	EndProjectSection
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Release|Any CPU = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{4E0B1852-7B6D-49E7-9939-A315D086B094}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{4E0B1852-7B6D-49E7-9939-A315D086B094}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{4E0B1852-7B6D-49E7-9939-A315D086B094}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{4E0B1852-7B6D-49E7-9939-A315D086B094}.Release|Any CPU.Build.0 = Release|Any CPU
+		{FDA04BAB-34FD-4E5D-91F4-0503090973A6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{FDA04BAB-34FD-4E5D-91F4-0503090973A6}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{FDA04BAB-34FD-4E5D-91F4-0503090973A6}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{FDA04BAB-34FD-4E5D-91F4-0503090973A6}.Release|Any CPU.Build.0 = Release|Any CPU
+		{5716EAF7-ABA1-434F-9AFA-51693A73C627}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{5716EAF7-ABA1-434F-9AFA-51693A73C627}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{5716EAF7-ABA1-434F-9AFA-51693A73C627}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{5716EAF7-ABA1-434F-9AFA-51693A73C627}.Release|Any CPU.Build.0 = Release|Any CPU
+		{B7F38539-67A1-42B8-A5E5-CB592B6E3233}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{B7F38539-67A1-42B8-A5E5-CB592B6E3233}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{B7F38539-67A1-42B8-A5E5-CB592B6E3233}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{B7F38539-67A1-42B8-A5E5-CB592B6E3233}.Release|Any CPU.Build.0 = Release|Any CPU
+		{5C586501-4E91-4007-B23F-02E20014DA51}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{5C586501-4E91-4007-B23F-02E20014DA51}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{5C586501-4E91-4007-B23F-02E20014DA51}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{5C586501-4E91-4007-B23F-02E20014DA51}.Release|Any CPU.Build.0 = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+	GlobalSection(NestedProjects) = preSolution
+		{FDA04BAB-34FD-4E5D-91F4-0503090973A6} = {C8104500-2C0F-46A2-AE84-A3D74CC55697}
+		{5716EAF7-ABA1-434F-9AFA-51693A73C627} = {1F1914B9-8F67-4B51-9738-CDE5AFB2F8DE}
+		{B7F38539-67A1-42B8-A5E5-CB592B6E3233} = {1F1914B9-8F67-4B51-9738-CDE5AFB2F8DE}
+	EndGlobalSection
+EndGlobal
diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md
@@ -0,0 +1,39 @@
+#### 0.0.1 November 3 2015
+In development
+
+#### 0.0.2 November 16 2015
+First bleeding-edge, alpha release of NBench.
+
+To write an NBench test, use the following syntax:
+
+```csharp
+public class SimpleCounterBenchmark
+    {
+        public const string CounterName = "DumbCounter";
+        private Counter _counter;
+
+        [PerfSetup]
+        public void SetUp(BenchmarkContext context)
+        {
+            _counter = context.GetCounter(CounterName);
+        }
+
+        /// <summary>
+        /// Run 3 tests, 1 second long each
+        /// </summary>
+        [PerformanceBenchmark(Description = "Simple iteration collection test", RunMode = RunType.Iterations, TestMode = TestType.Test, RunTimeMilliseconds = 1000, NumberOfIterations = 30)]
+        [CounterMeasurement(CounterName)]
+        [MemoryAssertion(MemoryMetric.TotalBytesAllocated, MustBe.LessThan, ByteConstants.EightKb)]
+        [GcTotalAssertion(GcMetric.TotalCollections, GcGeneration.Gen2, MustBe.ExactlyEqualTo, 0d)]
+        public void Run()
+        {
+            _counter.Increment();
+        }
+
+        [PerfCleanup]
+        public void CleanUp(BenchmarkContext context)
+        {
+            //no-op
+        }
+    }
+```
diff --git a/benchmarks/NBench.Microbenchmarks/App.config b/benchmarks/NBench.Microbenchmarks/App.config
@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<configuration>
+    <startup> 
+        <supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.5" />
+    </startup>
+</configuration>
diff --git a/benchmarks/NBench.Microbenchmarks/NBench.Microbenchmarks.csproj b/benchmarks/NBench.Microbenchmarks/NBench.Microbenchmarks.csproj
@@ -0,0 +1,73 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="14.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
+  <PropertyGroup>
+    <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
+    <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
+    <ProjectGuid>{FDA04BAB-34FD-4E5D-91F4-0503090973A6}</ProjectGuid>
+    <OutputType>Exe</OutputType>
+    <AppDesignerFolder>Properties</AppDesignerFolder>
+    <RootNamespace>NBench.Microbenchmarks</RootNamespace>
+    <AssemblyName>NBench.Microbenchmarks</AssemblyName>
+    <TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
+    <FileAlignment>512</FileAlignment>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
+    <PlatformTarget>AnyCPU</PlatformTarget>
+    <DebugSymbols>true</DebugSymbols>
+    <DebugType>full</DebugType>
+    <Optimize>false</Optimize>
+    <OutputPath>bin\Debug\</OutputPath>
+    <DefineConstants>DEBUG;TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
+    <PlatformTarget>AnyCPU</PlatformTarget>
+    <DebugType>pdbonly</DebugType>
+    <Optimize>true</Optimize>
+    <OutputPath>bin\Release\</OutputPath>
+    <DefineConstants>TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+  </PropertyGroup>
+  <ItemGroup>
+    <Reference Include="BenchmarkDotNet, Version=0.7.8.0, Culture=neutral, processorArchitecture=MSIL">
+      <HintPath>..\..\packages\BenchmarkDotNet.0.7.8\lib\net40\BenchmarkDotNet.dll</HintPath>
+      <Private>True</Private>
+    </Reference>
+    <Reference Include="System" />
+    <Reference Include="System.Core" />
+    <Reference Include="System.Xml.Linq" />
+    <Reference Include="System.Data.DataSetExtensions" />
+    <Reference Include="Microsoft.CSharp" />
+    <Reference Include="System.Data" />
+    <Reference Include="System.Net.Http" />
+    <Reference Include="System.Xml" />
+  </ItemGroup>
+  <ItemGroup>
+    <Compile Include="Program.cs" />
+    <Compile Include="Properties\AssemblyInfo.cs" />
+    <Compile Include="SDK\Sdk_ActionBenchmarkInvoker.cs" />
+    <Compile Include="SDK\Sdk_ReflectionBenchmarkInvoker.cs" />
+    <Compile Include="Util\Util_AtomicCounters.cs" />
+  </ItemGroup>
+  <ItemGroup>
+    <None Include="App.config" />
+    <None Include="packages.config" />
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\..\src\NBench\NBench.csproj">
+      <Project>{4e0b1852-7b6d-49e7-9939-a315d086b094}</Project>
+      <Name>NBench</Name>
+    </ProjectReference>
+  </ItemGroup>
+  <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
+  <!-- To modify your build process, add your task inside one of the targets below and uncomment it. 
+       Other similar extension points exist, see Microsoft.Common.targets.
+  <Target Name="BeforeBuild">
+  </Target>
+  <Target Name="AfterBuild">
+  </Target>
+  -->
+</Project>
diff --git a/benchmarks/NBench.Microbenchmarks/Program.cs b/benchmarks/NBench.Microbenchmarks/Program.cs
@@ -0,0 +1,25 @@
+// Copyright (c) Petabridge <https://petabridge.com/>. All rights reserved.
+// Licensed under the Apache 2.0 license. See LICENSE file in the project root for full license information.
+
+using BenchmarkDotNet;
+using NBench.Microbenchmarks.SDK;
+using NBench.Microbenchmarks.Util;
+
+namespace NBench.Microbenchmarks
+{
+    class Program
+    {
+        static void Main(string[] args)
+        {
+            var competitionSwitch = new BenchmarkCompetitionSwitch(new []
+            {
+                typeof(Util_AtomicCounters), 
+                typeof(Sdk_ActionBenchmarkInvoker),
+                typeof(Sdk_ReflectionBenchmarkInvoker)
+            });
+
+            competitionSwitch.Run(args);;
+        }
+    }
+}
+
diff --git a/benchmarks/NBench.Microbenchmarks/Properties/AssemblyInfo.cs b/benchmarks/NBench.Microbenchmarks/Properties/AssemblyInfo.cs
@@ -0,0 +1,40 @@
+// Copyright (c) Petabridge <https://petabridge.com/>. All rights reserved.
+// Licensed under the Apache 2.0 license. See LICENSE file in the project root for full license information.
+
+using System.Reflection;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+
+// General Information about an assembly is controlled through the following 
+// set of attributes. Change these attribute values to modify the information
+// associated with an assembly.
+[assembly: AssemblyTitle("NBench.Microbenchmarks")]
+[assembly: AssemblyDescription("")]
+[assembly: AssemblyConfiguration("")]
+[assembly: AssemblyCompany("")]
+[assembly: AssemblyProduct("NBench.Microbenchmarks")]
+[assembly: AssemblyCopyright("Copyright ©  2015")]
+[assembly: AssemblyTrademark("")]
+[assembly: AssemblyCulture("")]
+
+// Setting ComVisible to false makes the types in this assembly not visible 
+// to COM components.  If you need to access a type in this assembly from 
+// COM, set the ComVisible attribute to true on that type.
+[assembly: ComVisible(false)]
+
+// The following GUID is for the ID of the typelib if this project is exposed to COM
+[assembly: Guid("fda04bab-34fd-4e5d-91f4-0503090973a6")]
+
+// Version information for an assembly consists of the following four values:
+//
+//      Major Version
+//      Minor Version 
+//      Build Number
+//      Revision
+//
+// You can specify all the values or you can default the Build and Revision Numbers 
+// by using the '*' as shown below:
+// [assembly: AssemblyVersion("1.0.*")]
+[assembly: AssemblyVersion("1.0.0.0")]
+[assembly: AssemblyFileVersion("1.0.0.0")]
+
diff --git a/benchmarks/NBench.Microbenchmarks/SDK/Sdk_ActionBenchmarkInvoker.cs b/benchmarks/NBench.Microbenchmarks/SDK/Sdk_ActionBenchmarkInvoker.cs
@@ -0,0 +1,23 @@
+// Copyright (c) Petabridge <https://petabridge.com/>. All rights reserved.
+// Licensed under the Apache 2.0 license. See LICENSE file in the project root for full license information.
+
+using BenchmarkDotNet.Tasks;
+using NBench.Sdk;
+
+namespace NBench.Microbenchmarks.SDK
+{
+    [BenchmarkTask(platform: BenchmarkPlatform.X64, jitVersion: BenchmarkJitVersion.LegacyJit)]
+    [BenchmarkTask(platform: BenchmarkPlatform.X86, jitVersion: BenchmarkJitVersion.LegacyJit)]
+    [BenchmarkTask(platform: BenchmarkPlatform.X64, jitVersion: BenchmarkJitVersion.RyuJit)]
+    public class Sdk_ActionBenchmarkInvoker
+    {
+        private readonly IBenchmarkInvoker _actionInvoker = new ActionBenchmarkInvoker("foo", context => {}, context => {}, context => {});
+
+        [BenchmarkDotNet.Benchmark("How quickly can we invoke when injecting context in an ActionBenchmarkInvoker")]
+        public void InvokeRunWithContext()
+        {
+            _actionInvoker.InvokeRun(BenchmarkContext.Empty);
+        }
+    }
+}
+
diff --git a/benchmarks/NBench.Microbenchmarks/SDK/Sdk_ReflectionBenchmarkInvoker.cs b/benchmarks/NBench.Microbenchmarks/SDK/Sdk_ReflectionBenchmarkInvoker.cs
@@ -0,0 +1,89 @@
+// Copyright (c) Petabridge <https://petabridge.com/>. All rights reserved.
+// Licensed under the Apache 2.0 license. See LICENSE file in the project root for full license information.
+
+using System.Linq;
+using BenchmarkDotNet.Tasks;
+using NBench.Sdk;
+using static NBench.Sdk.Compiler.ReflectionDiscovery;
+
+namespace NBench.Microbenchmarks.SDK
+{
+    [BenchmarkTask(platform: BenchmarkPlatform.X64, jitVersion: BenchmarkJitVersion.LegacyJit)]
+    [BenchmarkTask(platform: BenchmarkPlatform.X86, jitVersion: BenchmarkJitVersion.LegacyJit)]
+    [BenchmarkTask(platform: BenchmarkPlatform.X64, jitVersion: BenchmarkJitVersion.RyuJit)]
+    public class Sdk_ReflectionBenchmarkInvoker
+    {
+        public class BenchmarkWithContext
+        {
+            [PerfSetup]
+            public void Setup(BenchmarkContext context)
+            {
+
+            }
+
+            [PerformanceBenchmark]
+            [MemoryMeasurement(MemoryMetric.TotalBytesAllocated)]
+            public void Run(BenchmarkContext context)
+            {
+
+            }
+
+            [PerfCleanup]
+            public void Cleanup(BenchmarkContext context)
+            {
+
+            }
+        }
+
+        public class BenchmarkWithoutContext
+        {
+            [PerfSetup]
+            public void Setup()
+            {
+
+            }
+
+            [PerformanceBenchmark]
+            [MemoryMeasurement(MemoryMetric.TotalBytesAllocated)]
+            public void Run()
+            {
+
+            }
+
+            [PerfCleanup]
+            public void Cleanup()
+            {
+
+            }
+        }
+
+        private readonly IBenchmarkInvoker _contextInvoker;
+        private readonly IBenchmarkInvoker _withoutContextInvoker;
+
+        public Sdk_ReflectionBenchmarkInvoker()
+        {
+            var benchmarkType = typeof (BenchmarkWithContext);
+            var benchmarks = CreateBenchmarksForClass(benchmarkType);
+            _contextInvoker = CreateInvokerForBenchmark(benchmarks.Single());
+            _contextInvoker.InvokePerfSetup(BenchmarkContext.Empty);
+
+            benchmarkType = typeof(BenchmarkWithoutContext);
+            benchmarks = CreateBenchmarksForClass(benchmarkType);
+            _withoutContextInvoker = CreateInvokerForBenchmark(benchmarks.Single());
+            _withoutContextInvoker.InvokePerfSetup(BenchmarkContext.Empty);
+        }
+
+        [BenchmarkDotNet.Benchmark("How quickly can we invoke when injecting context into a ReflectionBenchmarkInvoker")]
+        public void InvokeRunWithContext()
+        {
+            _contextInvoker.InvokeRun(BenchmarkContext.Empty);
+        }
+
+        [BenchmarkDotNet.Benchmark("How quickly can we invoke when NOT injecting context into a ReflectionBenchmarkInvoker")]
+        public void InvokeRunWithoutContext()
+        {
+            _withoutContextInvoker.InvokeRun(BenchmarkContext.Empty);
+        }
+    }
+}
+