Skip to content

Commit

Permalink
Add nunit3-console package test under .NET Core 3.1
Browse files Browse the repository at this point in the history
  • Loading branch information
@CharliePoole
CharliePoole committed Apr 13, 2021
1 parent 381f1ae commit 82ca51d
Show file tree
Hide file tree
Showing 10 changed files with 81 additions and 30 deletions.
3 changes: 3 additions & 0 deletions NUnitConsole.sln
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,10 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution
build.cake = build.cake
BUILDING.md = BUILDING.md
CHANGES.txt = CHANGES.txt
ci.cake = ci.cake
CONTRIBUTING.md = CONTRIBUTING.md
src\Directory.Build.props = src\Directory.Build.props
header-check.cake = header-check.cake
LICENSE.txt = LICENSE.txt
NetFXTests.nunit = NetFXTests.nunit
NOTICES.txt = NOTICES.txt
Expand All @@ -24,6 +26,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution
package-checks.cake = package-checks.cake
package-tests.cake = package-tests.cake
README.md = README.md
test-results.cake = test-results.cake
EndProjectSection
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "nuget", "nuget", "{A972031D-2F61-4183-AF75-99EE1A9F6B32}"
Expand Down
1 change: 1 addition & 0 deletions build.cake
View file
Original file line number Diff line number Diff line change
Expand Up @@ -487,6 +487,7 @@ Task("BuildChocolateyPackages")
new ChocolateyNuSpecContent { Source = CURRENT_IMG_DIR + "bin/agents/net40/testcentric.engine.metadata.dll", Target="tools/agents/net40" },
new ChocolateyNuSpecContent { Source = CURRENT_IMG_DIR + "bin/agents/netcoreapp3.1/nunit-agent.dll", Target="tools/agents/netcoreapp3.1" },
new ChocolateyNuSpecContent { Source = CURRENT_IMG_DIR + "bin/agents/netcoreapp3.1/nunit-agent.dll.config", Target="tools/agents/netcoreapp3.1" },
new ChocolateyNuSpecContent { Source = CURRENT_IMG_DIR + "bin/agents/netcoreapp3.1/nunit-agent.runtimeconfig.json", Target="tools/agents/netcoreapp3.1" },
new ChocolateyNuSpecContent { Source = CHOCO_DIR + "nunit-agent.exe.ignore", Target="tools/agents/netcoreapp3.1" },
new ChocolateyNuSpecContent { Source = CURRENT_IMG_DIR + "bin/agents/netcoreapp3.1/nunit-agent-x86.dll", Target="tools/agents/netcoreapp3.1" },
new ChocolateyNuSpecContent { Source = CURRENT_IMG_DIR + "bin/agents/netcoreapp3.1/nunit-agent-x86.dll.config", Target="tools/agents/netcoreapp3.1" },
Expand Down
1 change: 1 addition & 0 deletions nuget/engine/nunit.engine.nuspec
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@

<file src="bin/agents/netcoreapp3.1/nunit-agent.dll" target="contentFiles/any/agents/netcoreapp3.1" />
<file src="bin/agents/netcoreapp3.1/nunit-agent.dll.config" target="contentFiles/any/agents/netcoreapp3.1" />
<file src="bin/agents/netcoreapp3.1/nunit-agent.runtimeconfig.json" target="tools/agents/netcoreapp3.1" />
<file src="bin/agents/netcoreapp3.1/nunit-agent-x86.dll" target="contentFiles/any/agents/netcoreapp3.1" />
<file src="bin/agents/netcoreapp3.1/nunit-agent-x86.dll.config" target="contentFiles/any/agents/netcoreapp3.1" />
<file src="bin/agents/netcoreapp3.1/nunit.engine.core.dll" target="contentFiles/any/agents/netcoreapp3.1" />
Expand Down
20 changes: 11 additions & 9 deletions nuget/runners/nunit.console-runner.nuspec
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,15 +46,17 @@
<file src="bin/agents/net40/testcentric.engine.metadata.dll" target="tools/agents/net40" />
<file src="../../nuget/runners/nunit.agent.addins" target="tools/agents/net40"/>

<file src="bin/agents/netcoreapp3.1/nunit-agent.dll" target="contentFiles/any/agents/netcoreapp3.1" />
<file src="bin/agents/netcoreapp3.1/nunit-agent.dll.config" target="contentFiles/any/agents/netcoreapp3.1" />
<file src="bin/agents/netcoreapp3.1/nunit-agent-x86.dll" target="contentFiles/any/agents/netcoreapp3.1" />
<file src="bin/agents/netcoreapp3.1/nunit-agent-x86.dll.config" target="contentFiles/any/agents/netcoreapp3.1" />
<file src="bin/agents/netcoreapp3.1/nunit.engine.api.dll" target="contentFiles/any/agents/netcoreapp3.1" />
<file src="bin/agents/netcoreapp3.1/nunit.engine.api.xml" target="contentFiles/any/agents/netcoreapp3.1" />
<file src="bin/agents/netcoreapp3.1/nunit.engine.core.dll" target="contentFiles/any/agents/netcoreapp3.1" />
<file src="bin/agents/netcoreapp3.1/testcentric.engine.metadata.dll" target="contentFiles/any/agents/netcoreapp3.1" />
<file src="../../nuget/engine/nunit.agent.addins" target="contentFiles/any/agents/netcoreapp3.1"/>
<file src="bin/agents/netcoreapp3.1/nunit-agent.dll" target="tools/agents/netcoreapp3.1" />
<file src="bin/agents/netcoreapp3.1/nunit-agent.dll.config" target="tools/agents/netcoreapp3.1" />
<file src="bin/agents/netcoreapp3.1/nunit-agent.runtimeconfig.json" target="tools/agents/netcoreapp3.1" />
<file src="bin/agents/netcoreapp3.1/nunit-agent-x86.dll" target="tools/agents/netcoreapp3.1" />
<file src="bin/agents/netcoreapp3.1/nunit-agent-x86.dll.config" target="tools/agents/netcoreapp3.1" />
<file src="bin/agents/netcoreapp3.1/nunit-agent-x86.runtimeconfig.json" target="tools/agents/netcoreapp3.1" />
<file src="bin/agents/netcoreapp3.1/nunit.engine.api.dll" target="tools/agents/netcoreapp3.1" />
<file src="bin/agents/netcoreapp3.1/nunit.engine.api.xml" target="tools/agents/netcoreapp3.1" />
<file src="bin/agents/netcoreapp3.1/nunit.engine.core.dll" target="tools/agents/netcoreapp3.1" />
<file src="bin/agents/netcoreapp3.1/testcentric.engine.metadata.dll" target="tools/agents/netcoreapp3.1" />
<file src="../../nuget/engine/nunit.agent.addins" target="tools/agents/netcoreapp3.1"/>

<file src="bin/net20/nunit3-console.exe" target="tools" />
<file src="bin/net20/nunit3-console.exe.config" target="tools" />
Expand Down
52 changes: 44 additions & 8 deletions package-tests.cake
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,14 @@
// Representation of a single test to be run against a pre-built package.
public struct PackageTest
{
public string Name;
public string Description;
public string Arguments;
public ExpectedResult ExpectedResult;

public PackageTest(string description, string arguments, ExpectedResult expectedResult)
public PackageTest(string name, string description, string arguments, ExpectedResult expectedResult)
{
Name = name;
Description = description;
Arguments = arguments;
ExpectedResult = expectedResult;
Expand Down Expand Up @@ -38,6 +40,7 @@ public abstract class PackageTester

protected abstract string PackageName { get; }
protected abstract string PackageInstallDirectory { get; }
protected abstract string PackageResultDirectory { get; }
protected abstract string PackageBinDir { get; }

protected string PackageUnderTest => _packageDir + PackageName;
Expand All @@ -64,22 +67,20 @@ public abstract class PackageTester
{
var reporter = new ResultReporter(PackageName);

_context.CleanDirectory(PackageResultDirectory);

foreach (var packageTest in PackageTests)
{
var resultFile = _outputDir + "TestResult.xml";
// Delete result file ahead of time so we don't mistakenly
// read a left-over file from another test run. Leave the
// file after the run in case we need it to debug a failure.
if (_context.FileExists(resultFile))
_context.DeleteFile(resultFile);
var resultDir = PackageResultDirectory + packageTest.Name + "/";
var resultFile = resultDir + "TestResult.xml";

DisplayBanner(packageTest.Description);

int rc = _context.StartProcess(
PackageBinDir + "nunit3-console.exe",
new ProcessSettings()
{
Arguments = packageTest.Arguments,
Arguments = $"{packageTest.Arguments} --work={resultDir}",
WorkingDirectory = _outputDir
});

Expand Down Expand Up @@ -116,13 +117,16 @@ public abstract class PackageTester
}
}

// These are tests using the .NET Framework build of the console runner.
// However, they now include running tests under .Net Core 3.1.
public abstract class NetFXPackageTester : PackageTester
{
public NetFXPackageTester(ICakeContext context, string packageVersion)
: base(context, packageVersion)
{
// Add common tests for running under .NET Framework
PackageTests.Add(new PackageTest(
"net35",
"Run mock-assembly.dll under .NET 3.5",
"net35/mock-assembly.dll",
new ExpectedResult("Failed")
Expand All @@ -136,6 +140,7 @@ public abstract class NetFXPackageTester : PackageTester
}));

PackageTests.Add(new PackageTest(
"net40",
"Run mock-assembly.dll under .NET 4.x",
"net40/mock-assembly.dll",
new ExpectedResult("Failed")
Expand All @@ -149,6 +154,7 @@ public abstract class NetFXPackageTester : PackageTester
}));

PackageTests.Add(new PackageTest(
"net35_plus_net40",
"Run both copies of mock-assembly together",
"net35/mock-assembly.dll net40/mock-assembly.dll",
new ExpectedResult("Failed")
Expand All @@ -160,6 +166,23 @@ public abstract class NetFXPackageTester : PackageTester
Inconclusive = 2 * 1,
Skipped = 2 * 7
}));

// TODO: Remove this check when msi package is
// updated to include .net core assemblies.
if (GetType().Name != "MsiPackageTester")
PackageTests.Add(new PackageTest(
"netcoreapp3.1",
"Run mock-assembly.dll under .NET Core 3.1",
"netcoreapp3.1/mock-assembly.dll --trace:Debug",
new ExpectedResult("Failed")
{
Total = 37,
Passed = 23,
Failed = 5,
Warnings = 0,
Inconclusive = 1,
Skipped = 7
}));
}
}

Expand All @@ -170,6 +193,7 @@ public abstract class NetCorePackageTester : PackageTester
{
// Add common tests for running under .NET Core (2.1 or higher)
PackageTests.Add(new PackageTest(
"netcoreapp2.1",
"Run mock-assembly.dll targeting .NET Core 2.1",
"netcoreapp2.1/mock-assembly.dll",
new ExpectedResult("Failed")
Expand All @@ -183,6 +207,7 @@ public abstract class NetCorePackageTester : PackageTester
}));

PackageTests.Add(new PackageTest(
"netcoreapp3.1",
"Run mock-assembly targeting .NET Core 3.1",
"netcoreapp3.1/mock-assembly.dll",
new ExpectedResult("Failed")
Expand All @@ -196,6 +221,7 @@ public abstract class NetCorePackageTester : PackageTester
}));

PackageTests.Add(new PackageTest(
"netcoreapp2.1_plus_netcoreapp3.1",
"Run both copies of mock-assembly together",
"netcoreapp2.1/mock-assembly.dll netcoreapp3.1/mock-assembly.dll",
new ExpectedResult("Failed")
Expand All @@ -218,6 +244,7 @@ public class NuGetNetFXPackageTester : NetFXPackageTester
protected override string PackageName => $"NUnit.ConsoleRunner.{_packageVersion}.nupkg";
protected override string PackageInstallDirectory => _packageDir + "test/nuget-netfx/";
protected override string PackageBinDir => PackageInstallDirectory + "tools/";
protected override string PackageResultDirectory => _packageDir + "test-results/nuget-netfx/";
}

public class NuGetNetCorePackageTester : NetCorePackageTester
Expand All @@ -228,6 +255,7 @@ public class NuGetNetCorePackageTester : NetCorePackageTester
protected override string PackageName => $"NUnit.ConsoleRunner.NetCore.{_packageVersion}.nupkg";
protected override string PackageInstallDirectory => _packageDir + "test/nuget-netcore/";
protected override string PackageBinDir => PackageInstallDirectory + "tools/netcoreapp3.1/any/";
protected override string PackageResultDirectory => _packageDir + "test-results/nuget-netcore/";
}

public class ChocolateyPackageTester : NetFXPackageTester
Expand All @@ -238,6 +266,7 @@ public class ChocolateyPackageTester : NetFXPackageTester
protected override string PackageName => $"nunit-console-runner.{_packageVersion}.nupkg";
protected override string PackageInstallDirectory => _packageDir + "test/choco/";
protected override string PackageBinDir => PackageInstallDirectory + "tools/";
protected override string PackageResultDirectory => _packageDir + "test-results/choco/";
}

public class MsiPackageTester : NetFXPackageTester
Expand All @@ -247,6 +276,7 @@ public class MsiPackageTester : NetFXPackageTester
{
// Add tests specific to the msi package
PackageTests.Add(new PackageTest(
"net35_plus_net40_project",
"Run project with both copies of mock-assembly",
$"../../NetFXTests.nunit --config={_config}",
new ExpectedResult("Failed")
Expand All @@ -263,6 +293,7 @@ public class MsiPackageTester : NetFXPackageTester
protected override string PackageName => $"NUnit.Console-{_packageVersion}.msi";
protected override string PackageInstallDirectory => _packageDir + "test/msi/";
protected override string PackageBinDir => PackageInstallDirectory + "NUnit.org/nunit-console/";
protected override string PackageResultDirectory => _packageDir + "test-results/msi/";

protected override void CreatePackageInstallDirectory()
{
Expand All @@ -284,6 +315,9 @@ public class MsiPackageTester : NetFXPackageTester
_context.CopyFiles(
PackageBinDir + "*.dll",
PackageBinDir + "agents/net40/");
_context.CopyFiles(
PackageBinDir + "*.dll",
PackageBinDir + "agents/netcoreapp3.1");
}
}
}
Expand All @@ -295,6 +329,7 @@ public class ZipPackageTester : NetFXPackageTester
{
// Add tests specific to the zip package
PackageTests.Add(new PackageTest(
"net35_plus_net40_project",
"Run project with both copies of mock-assembly",
$"../../NetFXTests.nunit --config={_config}",
new ExpectedResult("Failed")
Expand All @@ -311,4 +346,5 @@ public class ZipPackageTester : NetFXPackageTester
protected override string PackageName => $"NUnit.Console-{_packageVersion}.zip";
protected override string PackageInstallDirectory => _packageDir + "test/zip/";
protected override string PackageBinDir => PackageInstallDirectory + "bin/net20/";
protected override string PackageResultDirectory => _packageDir + "test-results/zip/";
}
5 changes: 3 additions & 2 deletions src/NUnitEngine/nunit-agent/Program.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ public class NUnitTestAgent
[STAThread]
public static void Main(string[] args)
{
Console.WriteLine("Agent Process Starting");
AgentId = new Guid(args[0]);
AgencyUrl = args[1];

Expand Down Expand Up @@ -63,13 +64,13 @@ public static void Main(string[] args)
InternalTrace.Initialize(Path.Combine(workDirectory, logName), traceLevel);
log = InternalTrace.GetLogger(typeof(NUnitTestAgent));

log.Info("Agent process {0} starting", pid);

if (debugArgPassed)
TryLaunchDebugger();

LocateAgencyProcess(agencyPid);

log.Info("Agent process {0} starting", pid);

#if NETFRAMEWORK
log.Info("Running under version {0}, {1}",
Environment.Version,
Expand Down
4 changes: 1 addition & 3 deletions src/NUnitEngine/nunit-agent/nunit-agent-x86.csproj
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<RootNamespace>nunit.agent</RootNamespace>
<TargetFrameworks>net20;net40;netcoreapp3.1</TargetFrameworks>
<ApplicationManifest>app.manifest</ApplicationManifest>
Expand All @@ -9,9 +10,6 @@
<OutputPath>..\..\..\bin\$(Configuration)\agents\</OutputPath>
<GenerateSupportedRuntime>False</GenerateSupportedRuntime>
</PropertyGroup>
<PropertyGroup Condition="'$(TargetFramework)'=='net20' or '$(TargetFramework)'=='net40'">
<OutputType>Exe</OutputType>
</PropertyGroup>
<ItemGroup Condition="'$(TargetFramework)'=='net20' or '$(TargetFramework)'=='net40'">
<Reference Include="System.Runtime.Remoting" />
</ItemGroup>
Expand Down
4 changes: 1 addition & 3 deletions src/NUnitEngine/nunit-agent/nunit-agent.csproj
View file
Original file line number Diff line number Diff line change
@@ -1,15 +1,13 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<RootNamespace>nunit.agent</RootNamespace>
<TargetFrameworks>net20;net40;netcoreapp3.1</TargetFrameworks>
<ApplicationManifest>app.manifest</ApplicationManifest>
<ApplicationIcon>..\..\..\nunit.ico</ApplicationIcon>
<GenerateSupportedRuntime>false</GenerateSupportedRuntime>
<OutputPath>..\..\..\bin\$(Configuration)\agents\</OutputPath>
</PropertyGroup>
<PropertyGroup Condition="'$(TargetFramework)'=='net20' or '$(TargetFramework)'=='net40'">
<OutputType>Exe</OutputType>
</PropertyGroup>
<ItemGroup Condition="'$(TargetFramework)'=='net20' or '$(TargetFramework)'=='net40'">
<Reference Include="System.Runtime.Remoting" />
</ItemGroup>
Expand Down
18 changes: 14 additions & 4 deletions src/NUnitEngine/nunit.engine/Services/AgentProcess.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,12 @@ public AgentProcess(TestAgency agency, TestPackage package, Guid agentId)
StartInfo.Arguments = AgentArgs.ToString();
StartInfo.LoadUserProfile = loadUserProfile;
}
else if (TargetRuntime.Runtime == RuntimeType.NetCore)
{
StartInfo.FileName = "dotnet";
StartInfo.Arguments = $"{AgentExePath} {AgentArgs}";
StartInfo.LoadUserProfile = loadUserProfile;
}
else
{
StartInfo.FileName = AgentExePath;
Expand All @@ -70,7 +76,7 @@ public AgentProcess(TestAgency agency, TestPackage package, Guid agentId)

// Internal properties exposed for testing

internal RuntimeFramework TargetRuntime { get; }
internal RuntimeFramework TargetRuntime { get; }
internal string AgentExePath { get; }
internal StringBuilder AgentArgs { get; }

Expand All @@ -96,24 +102,28 @@ public static string GetTestAgentExePath(RuntimeFramework targetRuntime, bool re
log.Debug($"Checking for agents at {agentsDir}");

string agentName = requires32Bit
? "nunit-agent-x86.exe"
: "nunit-agent.exe";
? "nunit-agent-x86"
: "nunit-agent";

string runtimeDir;
string agentExtension;
switch (targetRuntime.Runtime)
{
case RuntimeType.Net:
case RuntimeType.Mono:
runtimeDir = targetRuntime.FrameworkVersion.Major >= 4 ? "net40" : "net20";
agentExtension = ".exe";
break;
case RuntimeType.NetCore:
runtimeDir = "netcoreapp3.1";
agentExtension = ".dll";
break;
default:
log.Error($"Unknown runtime type: {targetRuntime.Runtime}");
return null;
}

return Path.Combine(Path.Combine(agentsDir, runtimeDir), agentName);
return Path.Combine(Path.Combine(agentsDir, runtimeDir), agentName + agentExtension);
}
}
}
Expand Down
3 changes: 2 additions & 1 deletion test-results.cake
View file
Original file line number Diff line number Diff line change
Expand Up @@ -130,10 +130,11 @@ public class TestReport
foreach (XmlNode suite in suites)
{
// Narrow down to the specific failures we want
string runState = GetAttribute(suite, "runstate");
string suiteResult = GetAttribute(suite, "result");
string label = GetAttribute(suite, "label");
string site = suite.Attributes["site"]?.Value ?? "Test";
if (suiteResult == "Failed" && site == "Test" && label == "Invalid")
if (runState == "NotRunnable" || suiteResult == "Failed" && site == "Test" && (label == "Invalid" || label=="Error"))
{
string message = suite.SelectSingleNode("reason/message")?.InnerText;
Errors.Add($" {message}");
Expand Down

0 comments on commit 82ca51d

Please sign in to comment.