Skip to content

Commit b4dd81f

Browse files
sarahelsaigsarahelsaig
authored
Merge pull request #673 from Lombiq/issue/OSOE-1308
OSOE-1308: Default .NET test workflows to Microsoft Testing Platform
2 parents 34462d1 + 0fb901d commit b4dd81f

16 files changed

Lines changed: 347 additions & 52 deletions

‎.github/actions/test-dotnet/Invoke-SolutionOrProjectTests.ps1‎

Lines changed: 122 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
11
param (
2+
[ValidateSet('Microsoft.Testing.Platform', 'VSTest')]
3+
[string] $TestPlatform = 'Microsoft.Testing.Platform',
24
[string] $SolutionOrProject,
35
[string] $Verbosity,
46
[string] $Filter,
@@ -13,14 +15,13 @@ param (
1315
# "-InformationAction Continue" to every call. This is not best practice for general PowerShell scripts, but makes sense
1416
# for scripts made for GHA.
1517
$informationPreference = 'Continue'
18+
$useMtp = $TestPlatform -eq 'Microsoft.Testing.Platform'
19+
$SolutionOrProject = (Resolve-Path $SolutionOrProject).Path
20+
Set-GitHubOutput 'test-count' 0
21+
Set-GitHubOutput 'dotnet-test-hang-dump' 0
1622

17-
# First, we globally set test configurations using environment variables. Then acquire the list of all test projects
18-
# (excluding the two test libraries) and then run each until one fails or all concludes. If a test fails, the output is
19-
# sanitized from unnecessary diagnostics messages from chromedriver if the output doesn't already contain groupings,
20-
# then it wraps them in "::group::<project name>". If there are already groupings, then it is not possible to nest them
21-
# (https://github.com/actions/runner/issues/802) so that's omitted. The groupings make the output collapsible region on
22-
# the Actions web UI. Note that we use bash to output the log using bash to avoid pwsh wrapping the output to the
23-
# default buffer width.
23+
# Set test configuration through environment variables, identify test applications, then run each until one fails.
24+
# Preserve the test output, including the UI Testing Toolbox's GitHub Actions groups and annotations.
2425

2526
if ($Env:RUNNER_OS -eq 'Windows')
2627
{
@@ -85,11 +86,45 @@ if ($SolutionOrProject -imatch '\.slnx?$')
8586
$tests = @()
8687
dotnet sln $SolutionOrProject list |
8788
Select-Object -Skip 2 |
88-
Select-String '\.Tests\.' |
89-
Select-String -NotMatch 'Lombiq.Tests.UI.csproj' |
90-
Select-String -NotMatch 'Lombiq.Tests.csproj' |
9189
ForEach-Object {
92-
$absolutePath = Resolve-Path -Path (Join-Path -Path $solutionDirectory -ChildPath $PSItem)
90+
$absolutePath = (Resolve-Path -Path (Join-Path -Path $solutionDirectory -ChildPath $PSItem)).Path
91+
92+
# Evaluate project properties instead of relying on project names or localized test runner output.
93+
$evaluationSwitches = @(
94+
"-p:Configuration=$Configuration"
95+
'-getProperty:IsTestingPlatformApplication,IsTestProject'
96+
'-verbosity:quiet'
97+
)
98+
99+
$properties = dotnet msbuild $absolutePath @evaluationSwitches | Out-String
100+
101+
if ($LASTEXITCODE -ne 0)
102+
{
103+
Write-GitHub "Failed to evaluate test project properties for `"$absolutePath`"."
104+
exit 1
105+
}
106+
107+
$properties = ($properties | ConvertFrom-Json).Properties
108+
109+
if ($useMtp)
110+
{
111+
if ($properties.IsTestingPlatformApplication -eq 'true')
112+
{
113+
$tests += $absolutePath
114+
}
115+
elseif ($properties.IsTestProject -eq 'true')
116+
{
117+
Write-GitHub "The test project `"$absolutePath`" does not support Microsoft.Testing.Platform. Migrate it or use test-platform VSTest."
118+
exit 1
119+
}
120+
121+
return
122+
}
123+
124+
if ($properties.IsTestProject -ne 'true')
125+
{
126+
return
127+
}
93128

94129
# While the test projects are run individually, passing in the solution name and solution dir via the
95130
# conventional MSBuild properties allows build customization.
@@ -132,7 +167,7 @@ if ($SolutionOrProject -imatch '\.slnx?$')
132167
}
133168
elseif ($SolutionOrProject -like '*.csproj')
134169
{
135-
Write-Information "Running tests for the `"$SolutionOrProject`' project."
170+
Write-Information "Running tests for the `"$SolutionOrProject`" project."
136171
$tests = @($SolutionOrProject)
137172
}
138173
else
@@ -215,14 +250,17 @@ function Failed($Job, $ProcessId, $Switches, $Test)
215250

216251
function StartProcessAndWaitForExit($Switches, $Test, $Timeout, $ShowTimeRemainingUntilTimeout)
217252
{
218-
# This is executed in a separate proecess so no variables or settings come through except what's copied over in the
253+
# This is executed in a separate process so no variables or settings come through except what's copied over in the
219254
# "$args" automatic variable. Only Write-Output should be used here, so "Receive-Job" can reliably capture it.
220255
$block = {
221256
Write-Output "StartProcessAndWaitForExitProcessId:$PID"
222257

223258
$argSwitches = $args[0]
224259
$argTest = $args[1]
225-
dotnet test @argSwitches $argTest 2>&1
260+
dotnet test @argSwitches 2>&1
261+
262+
# Use the process exit code, not human-readable runner output, to determine success.
263+
Write-Output "DotnetTestExitCode:$LASTEXITCODE"
226264

227265
if ($LASTEXITCODE -ne 0)
228266
{
@@ -239,20 +277,16 @@ function StartProcessAndWaitForExit($Switches, $Test, $Timeout, $ShowTimeRemaini
239277
{
240278
Receive-Job $job | Tee-Object -Variable line | Out-Host
241279

242-
if ("$line".StartsWith('StartProcessAndWaitForExitProcessId:'))
243-
{
244-
$processId = [int]("$line".Split('StartProcessAndWaitForExitProcessId:')[1].Split()[0])
245-
}
246-
247-
if ("$line".Contains('Test Run Successful.'))
248-
{
249-
$hasTestRunSuccessfully = $true
250-
}
251-
252-
if ("$line".Contains('::error::'))
280+
foreach ($outputLine in $line)
253281
{
254-
$hasTestRunSuccessfully = $false
255-
break
282+
if ("$outputLine" -match '^StartProcessAndWaitForExitProcessId:(\d+)$')
283+
{
284+
$processId = [int]$Matches[1]
285+
}
286+
elseif ("$outputLine" -match '^DotnetTestExitCode:(\d+)$')
287+
{
288+
$hasTestRunSuccessfully = [int]$Matches[1] -eq 0
289+
}
256290
}
257291

258292
if ($Timeout -gt 0)
@@ -278,6 +312,8 @@ function StartProcessAndWaitForExit($Switches, $Test, $Timeout, $ShowTimeRemaini
278312
Failed -Job $job -ProcessId $processId -Switches $Switches -Test $Test
279313
}
280314

315+
Remove-Job $job
316+
281317
return $hasTestRunSuccessfully
282318
}
283319

@@ -292,30 +328,76 @@ foreach ($test in $tests)
292328

293329
$switches = @(
294330
'--configuration', $Configuration
295-
'--nologo',
296-
'--no-build',
297-
'--logger', 'trx;LogFileName=test-results.trx'
298-
# This is for xUnit ITestOutputHelper, see https://xunit.net/docs/capturing-output.
299-
'--logger', 'console;verbosity=detailed'
331+
'--no-build'
300332
'--verbosity', $Verbosity
301333
)
302334

303-
if ($BlameHangTimeout)
335+
if ($useMtp)
304336
{
305-
$switches += ('--blame-hang-timeout', $BlameHangTimeout, '--blame-hang-dump-type', 'full')
306-
}
337+
$switches += @(
338+
'--project', $test
339+
'--output', 'Detailed'
340+
)
307341

308-
if ($Filter)
342+
if ($EnableDiagnosticMode)
343+
{
344+
$switches += ('--diagnostic-output-directory', (Join-Path (Get-Location) 'DiagnosticLogs'))
345+
}
346+
347+
$switches += @(
348+
'--'
349+
'--report-trx'
350+
'--report-gh'
351+
# UI tests already emit per-test groups; GitHub Actions doesn't support nested groups.
352+
'--report-gh-groups', 'off'
353+
'--show-stdout', 'all'
354+
'--show-stderr', 'all'
355+
)
356+
357+
# A solution can contain empty test projects, or a filter can select no tests in some of its projects.
358+
# Explicit project runs still fail when no tests run. Other failures always retain their exit code.
359+
if ($SolutionOrProject -imatch '\.slnx?$')
360+
{
361+
$switches += ('--ignore-exit-code', '8')
362+
}
363+
364+
if ($BlameHangTimeout)
365+
{
366+
$switches += ('--hangdump', '--hangdump-timeout', $BlameHangTimeout, '--hangdump-type', 'Full')
367+
$switches += ('--hangdump-filename', '{asm}_{tfm}_{pid}_hangdump.dmp')
368+
}
369+
370+
if ($EnableDiagnosticMode)
371+
{
372+
$switches += '--diagnostic'
373+
}
374+
}
375+
else
309376
{
310-
$switches += ('--filter', "$Filter")
377+
$switches += @(
378+
$test
379+
'--nologo'
380+
'--logger', 'trx;LogFileName=test-results.trx'
381+
'--logger', 'console;verbosity=detailed'
382+
)
383+
384+
if ($BlameHangTimeout)
385+
{
386+
$switches += ('--blame-hang-timeout', $BlameHangTimeout, '--blame-hang-dump-type', 'full')
387+
}
388+
389+
if ($EnableDiagnosticMode)
390+
{
391+
$switches += ('--diag', 'DiagnosticLogs/dotnet-test.log')
392+
}
311393
}
312394

313-
if ($EnableDiagnosticMode)
395+
if ($Filter)
314396
{
315-
$switches += ('--diag', 'DiagnosticLogs/dotnet-test.log')
397+
$switches += ('--filter', $Filter)
316398
}
317399

318-
Write-Information "Starting testing with ``dotnet test $switches $test``."
400+
Write-Information "Starting testing with ``dotnet test $switches``."
319401

320402
$success = StartProcessAndWaitForExit -Switches $switches -Test $test -Timeout $TestProcessTimeout -ShowTimeRemainingUntilTimeout $ShowTimeRemainingUntilTimeout
321403

‎.github/actions/test-dotnet/Merge-BlameHangDumps.ps1‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,7 @@ param ($Directory, $Configuration)
33

44
$rootDirectory = Resolve-Path $Directory
55
$blameHangDumpsName = 'BlameHangDumps'
6-
$testDirectoryPath = Join-Path $Directory 'test'
7-
$testDirectory = (Test-Path -Path $testDirectoryPath) ? (Resolve-Path $testDirectoryPath) : $rootDirectory
6+
$testDirectory = $rootDirectory
87

98
$dumpCount = (Get-ChildItem -Path $testDirectory -Filter '*_hangdump.dmp' -Recurse | Measure-Object).Count
109
Set-GitHubOutput 'dump-count' $dumpCount
@@ -27,7 +26,8 @@ function ItemFilter($Item, $TestConfiguration)
2726
return $false
2827
}
2928

30-
$allow = (($Item.Name -like 'Sequence_*.xml') -or ($Item.Name -like '*_hangdump.dmp'))
29+
$allow = (($Item.Name -like 'Sequence_*.xml') -or ($Item.Name -like '*_hangdump.dmp') -or
30+
($Item.Name -like '*.sequence.log'))
3131
if (-not $allow -and $TestConfiguration)
3232
{
3333
$allow = ($Item.FullName -like "*$(Join-Path 'bin' $TestConfiguration)*" )

‎.github/actions/test-dotnet/Tests/.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
TestResults/
2+
DiagnosticLogs/

‎.github/actions/test-dotnet/Tests/Directory.Build.props‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
<!-- Stop MSBuild's parent-directory search here so the fixture doesn't inherit the host repository's build settings
2+
(including OSOCE's Orchard Core analyzer configuration) when this repository is checked out as a submodule. -->
3+
<Project />

‎.github/actions/test-dotnet/Tests/Fixture.cs‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
using System;
2+
using Xunit;
3+
4+
namespace TestDotnet;
5+
6+
public class ActionFixture
7+
{
8+
private readonly ITestOutputHelper _output;
9+
10+
public ActionFixture(ITestOutputHelper output) => _output = output;
11+
12+
[Fact]
13+
public void PassingTest() => _output.WriteLine("Passing test output is preserved.");
14+
15+
[Theory]
16+
[InlineData("spaces & punctuation")]
17+
[InlineData("second case")]
18+
public void TheoryTest(string value) => Assert.NotEmpty(value);
19+
20+
[Fact]
21+
public void ControlledFailure()
22+
{
23+
// A success-looking log line must not override the process exit code.
24+
_output.WriteLine("Test Run Successful.");
25+
Assert.NotEqual("true", Environment.GetEnvironmentVariable("LGHA_TEST_FAILURE"));
26+
}
27+
}

‎.github/actions/test-dotnet/Tests/Fixture.csproj‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
<PropertyGroup>
3+
<TargetFramework>net10.0</TargetFramework>
4+
<OutputType>Exe</OutputType>
5+
<UseMicrosoftTestingPlatformRunner>true</UseMicrosoftTestingPlatformRunner>
6+
</PropertyGroup>
7+
<ItemGroup>
8+
<PackageReference Include="xunit.v3" Version="4.0.0" />
9+
<PackageReference Include="Microsoft.Testing.Extensions.GitHubActionsReport" Version="2.4.0" />
10+
<PackageReference Include="Microsoft.Testing.Extensions.HangDump" Version="2.4.0" />
11+
<PackageReference Include="Microsoft.Testing.Extensions.TrxReport" Version="2.4.0" />
12+
</ItemGroup>
13+
</Project>

‎.github/actions/test-dotnet/Tests/Fixture.slnx‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
<Solution>
2+
<Project Path="Fixture.csproj" />
3+
</Solution>

‎.github/actions/test-dotnet/Tests/Invoke-Tests.ps1‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
$errorActionPreference = 'Stop'
2+
$actionPath = (Resolve-Path "$PSScriptRoot/..").Path
3+
$repositoryPath = (Resolve-Path "$PSScriptRoot/../../../..").Path
4+
$artifactPath = Join-Path $PSScriptRoot "artifacts/$([Guid]::NewGuid().ToString('N'))"
5+
New-Item -ItemType Directory -Path $artifactPath -Force | Out-Null
6+
7+
$environmentVariableNames = @('PATH', 'GITHUB_ACTIONS', 'GITHUB_OUTPUT', 'GITHUB_STEP_SUMMARY', 'GITHUB_WORKSPACE', 'LGHA_TEST_FAILURE')
8+
$savedEnvironment = Get-ChildItem Env: | Where-Object { $PSItem.Name -in $environmentVariableNames }
9+
10+
function Assert-True($Condition, $Message)
11+
{
12+
if (-not $Condition) { throw $Message }
13+
}
14+
15+
function Invoke-Scenario($Name, $Target, $Filter, $ExpectedExitCode = 0)
16+
{
17+
$Env:GITHUB_OUTPUT = Join-Path $artifactPath "$Name-output.txt"
18+
$Env:GITHUB_STEP_SUMMARY = Join-Path $artifactPath "$Name-summary.md"
19+
$logPath = Join-Path $artifactPath "$Name.log"
20+
21+
$arguments = @(
22+
'-NoProfile',
23+
'-File', "$actionPath/Invoke-SolutionOrProjectTests.ps1"
24+
'-SolutionOrProject', $Target
25+
'-Filter', $Filter
26+
'-Verbosity', 'quiet'
27+
'-Configuration', 'Debug'
28+
'-TestProcessTimeout', '60000'
29+
'-EnableDiagnosticMode:$true'
30+
)
31+
32+
# Run in a child process so the action's exit and environment changes don't affect the test harness.
33+
& (Join-Path $PSHOME 'pwsh') @arguments *> $logPath
34+
35+
if ($LASTEXITCODE -ne $ExpectedExitCode)
36+
{
37+
Get-Content $logPath | Write-Output
38+
throw "$Name returned $LASTEXITCODE instead of $ExpectedExitCode."
39+
}
40+
}
41+
42+
Push-Location $PSScriptRoot
43+
44+
try
45+
{
46+
$Env:PATH = (Join-Path $repositoryPath 'Scripts') + [IO.Path]::PathSeparator + $Env:PATH
47+
$Env:GITHUB_ACTIONS = 'true'
48+
$Env:GITHUB_WORKSPACE = $repositoryPath
49+
$Env:LGHA_TEST_FAILURE = 'false'
50+
51+
# The fixture name deliberately has no '.Tests.' segment; discovery must use MSBuild properties.
52+
Invoke-Scenario -Name passing -Target Fixture.slnx -Filter 'FullyQualifiedName!~ControlledFailure'
53+
$passingLog = Get-Content (Join-Path $artifactPath 'passing.log') -Raw
54+
Assert-True ($passingLog -like '*Passing test output is preserved.*') 'Passing test output was lost.'
55+
56+
$reportPath = Join-Path $PSScriptRoot 'TestResults/Fixture_net10.0_x64.trx'
57+
[xml]$report = Get-Content $reportPath -Raw
58+
Assert-True ($report.TestRun.ResultSummary.Counters.passed -eq '3') 'Expected three passing tests in the TRX report.'
59+
Assert-True (Test-Path $Env:GITHUB_STEP_SUMMARY) 'The native GitHub Actions summary was not generated.'
60+
Assert-True (Get-ChildItem (Join-Path $PSScriptRoot 'DiagnosticLogs') -Filter '*.diag') 'MTP diagnostic logs were not generated.'
61+
62+
Invoke-Scenario -Name empty-solution -Target Fixture.slnx -Filter 'FullyQualifiedName~DoesNotExist'
63+
Invoke-Scenario -Name empty-project -Target Fixture.csproj -Filter 'FullyQualifiedName~DoesNotExist' -ExpectedExitCode 100
64+
65+
$Env:LGHA_TEST_FAILURE = 'true'
66+
Invoke-Scenario -Name failing -Target Fixture.csproj -Filter 'FullyQualifiedName~ControlledFailure' -ExpectedExitCode 100
67+
[xml]$report = Get-Content $reportPath -Raw
68+
Assert-True ($report.TestRun.ResultSummary.Counters.failed -eq '1') 'Expected one intentional failure in the TRX report.'
69+
$summary = Get-Content $Env:GITHUB_STEP_SUMMARY -Raw
70+
Assert-True ($summary -like '*ControlledFailure*') 'Failure details are missing from the native GitHub Actions summary.'
71+
72+
Write-Output 'MTP action regression checks passed: filtering, theory discovery, output, reports, diagnostics, empty selections, and failure exit codes.'
73+
}
74+
finally
75+
{
76+
Pop-Location
77+
$savedEnvironment | ForEach-Object {
78+
Set-Item ('Env:' + $PSItem.Key) -Value $PSItem.Value
79+
}
80+
}
81+
82+
# GitHub Actions checks LASTEXITCODE after the script returns. The final negative scenario deliberately returns 100.
83+
exit 0

‎.github/actions/test-dotnet/Tests/global.json‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
{
2+
"test": {
3+
"runner": "Microsoft.Testing.Platform"
4+
}
5+
}

0 commit comments

Comments
 (0)