Skip to content

Commit eaff89b

Browse files
Copilotbcollamore
Copilot
and
bcollamore
committed
Add PH2159: PreferCombinatorialTestingAnalyzer with tests and documentation
Co-authored-by: bcollamore <57269455+bcollamore@users.noreply.github.com>
1 parent da9e8b4 commit eaff89b

4 files changed

Lines changed: 283 additions & 0 deletions

File tree

‎Documentation/Diagnostics/PH2159.md‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
# PH2159: Consider using combinatorial testing
2+
3+
| Property | Value |
4+
|--|--|
5+
| Package | [Philips.CodeAnalysis.MsTestAnalyzers](https://www.nuget.org/packages/Philips.CodeAnalysis.MsTestAnalyzers) |
6+
| Diagnostic ID | PH2159 |
7+
| Category | [MsTest](../MsTest.md) |
8+
| Analyzer | [PreferCombinatorialTestingAnalyzer](https://github.com/philips-software/roslyn-analyzers/blob/main/Philips.CodeAnalysis.MsTestAnalyzers/PreferCombinatorialTestingAnalyzer.cs)
9+
| CodeFix | No |
10+
| Severity | Info |
11+
| Enabled By Default | No |
12+
13+
## Introduction
14+
15+
Test methods with multiple DataRow attributes may benefit from using combinatorial testing instead. Combinatorial testing allows you to test multiple combinations of parameter values more efficiently using the `Combinatorial.MSTest` package.
16+
17+
## How to solve
18+
19+
Consider refactoring your test method to use combinatorial testing with the `Combinatorial.MSTest` package. This can make your tests more maintainable when you have many parameter combinations.
20+
21+
First, install the Combinatorial.MSTest package:
22+
```xml
23+
<PackageReference Include="Combinatorial.MSTest" Version="1.0.11" />
24+
```
25+
26+
Then refactor your test to use `CombinatorialValues` attributes.
27+
28+
## Example
29+
30+
Code that triggers a diagnostic:
31+
``` cs
32+
[TestMethod]
33+
[DataRow("private static readonly", false)]
34+
[DataRow("private static", false)]
35+
[DataRow("private const", false)]
36+
[DataRow("private", false)]
37+
[DataRow("public static readonly", true)]
38+
[DataRow("public const", true)]
39+
[DataRow("public static", true)]
40+
[DataRow("public readonly", true)]
41+
[DataRow("public", true)]
42+
[TestCategory(TestDefinitions.UnitTests)]
43+
public void TestMethodWithManyDataRows(string modifiers, bool expected)
44+
{
45+
// Test implementation
46+
}
47+
```
48+
49+
Potential refactored code using combinatorial testing:
50+
``` cs
51+
[TestMethod]
52+
[TestCategory(TestDefinitions.UnitTests)]
53+
public void TestMethodWithCombinatorialValues(
54+
[CombinatorialValues("private", "public", "internal", "protected")] string visibility,
55+
[CombinatorialValues("", "static", "readonly", "const")] string modifier,
56+
[CombinatorialValues(true, false)] bool expected)
57+
{
58+
var modifiers = $"{visibility} {modifier}".Trim();
59+
// Test implementation
60+
}
61+
```
62+
63+
**Note**: The `Combinatorial.MSTest` package may not work with all parameter combinations and has limitations when mixing combinatorial and non-combinatorial parameters. Evaluate whether the refactoring is appropriate for your specific test case.
64+
65+
## Configuration
66+
67+
This analyzer is disabled by default. To enable it, add the following to your .editorconfig:
68+
69+
```ini
70+
dotnet_diagnostic.PH2159.severity = suggestion
71+
```
72+
73+
The analyzer triggers when a test method has 4 or more DataRow attributes.
74+
75+
The general ways of [suppressing](https://learn.microsoft.com/en-us/dotnet/fundamentals/code-analysis/suppress-warnings) diagnostics apply.

‎Philips.CodeAnalysis.Common/DiagnosticId.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -144,5 +144,6 @@ public enum DiagnosticId
144144
AvoidUnusedToString = 2153,
145145
AvoidUnlicensedPackages = 2155,
146146
AvoidPkcsPaddingWithRsaEncryption = 2158,
147+
PreferCombinatorialTestingOverDataRows = 2159,
147148
}
148149
}

‎Philips.CodeAnalysis.MsTestAnalyzers/PreferCombinatorialTestingAnalyzer.cs‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
// © 2025 Koninklijke Philips N.V. See License.md in the project root for license information.
2+
3+
using System.Collections.Immutable;
4+
using Microsoft.CodeAnalysis;
5+
using Microsoft.CodeAnalysis.CSharp.Syntax;
6+
using Microsoft.CodeAnalysis.Diagnostics;
7+
using Philips.CodeAnalysis.Common;
8+
9+
namespace Philips.CodeAnalysis.MsTestAnalyzers
10+
{
11+
[DiagnosticAnalyzer(LanguageNames.CSharp)]
12+
public class PreferCombinatorialTestingAnalyzer : TestMethodDiagnosticAnalyzer
13+
{
14+
private const string Title = @"Consider using combinatorial testing";
15+
public const string MessageFormat = @"Consider using combinatorial testing instead of multiple DataRow attributes. This method has {0} DataRow attributes.";
16+
private const string Description = @"Methods with multiple DataRow attributes may benefit from combinatorial testing using the Combinatorial.MSTest package for better maintainability.";
17+
private const string Category = Categories.MsTest;
18+
19+
private static readonly DiagnosticDescriptor Rule = new(DiagnosticId.PreferCombinatorialTestingOverDataRows.ToId(), Title, MessageFormat, Category, DiagnosticSeverity.Info, isEnabledByDefault: false, description: Description);
20+
21+
public override ImmutableArray<DiagnosticDescriptor> SupportedDiagnostics => ImmutableArray.Create(Rule);
22+
23+
protected override TestMethodImplementation OnInitializeTestMethodAnalyzer(AnalyzerOptions options, Compilation compilation, MsTestAttributeDefinitions definitions)
24+
{
25+
Helper helper = new(options, compilation);
26+
return new PreferCombinatorialTestingImplementation(helper, definitions);
27+
}
28+
29+
public class PreferCombinatorialTestingImplementation : TestMethodImplementation
30+
{
31+
private const int MinimumDataRowsForSuggestion = 4; // Suggest when 4+ DataRows
32+
33+
public PreferCombinatorialTestingImplementation(Helper helper, MsTestAttributeDefinitions definitions) : base(definitions, helper)
34+
{
35+
}
36+
37+
protected override void OnTestMethod(SyntaxNodeAnalysisContext context, MethodDeclarationSyntax methodDeclaration, IMethodSymbol methodSymbol, bool isDataTestMethod)
38+
{
39+
// Count DataRow attributes
40+
SyntaxList<AttributeListSyntax> attributeLists = methodDeclaration.AttributeLists;
41+
var dataRowCount = 0;
42+
43+
foreach (AttributeListSyntax attributeList in attributeLists)
44+
{
45+
foreach (AttributeSyntax attribute in attributeList.Attributes)
46+
{
47+
if (Helper.ForAttributes.IsDataRowAttribute(attribute, context))
48+
{
49+
dataRowCount++;
50+
}
51+
}
52+
}
53+
54+
// Only suggest if there are enough DataRows
55+
if (dataRowCount >= MinimumDataRowsForSuggestion)
56+
{
57+
Location location = methodDeclaration.Identifier.GetLocation();
58+
var diagnostic = Diagnostic.Create(Rule, location, dataRowCount);
59+
context.ReportDiagnostic(diagnostic);
60+
}
61+
}
62+
}
63+
}
64+
}

‎Philips.CodeAnalysis.Test/MsTest/PreferCombinatorialTestingAnalyzerTest.cs‎

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
// © 2025 Koninklijke Philips N.V. See License.md in the project root for license information.
2+
3+
using System.Text.RegularExpressions;
4+
using System.Threading.Tasks;
5+
using Microsoft.CodeAnalysis;
6+
using Microsoft.CodeAnalysis.Diagnostics;
7+
using Microsoft.VisualStudio.TestTools.UnitTesting;
8+
using Philips.CodeAnalysis.Common;
9+
using Philips.CodeAnalysis.MsTestAnalyzers;
10+
using Philips.CodeAnalysis.Test.Helpers;
11+
using Philips.CodeAnalysis.Test.Verifiers;
12+
13+
namespace Philips.CodeAnalysis.Test.MsTest
14+
{
15+
[TestClass]
16+
public class PreferCombinatorialTestingAnalyzerTest : DiagnosticVerifier
17+
{
18+
[TestMethod]
19+
[TestCategory(TestDefinitions.UnitTests)]
20+
public async Task PreferCombinatorialTestingTriggersDiagnosticWithMultipleDataRowsAsync()
21+
{
22+
var testCode = @"
23+
using Microsoft.VisualStudio.TestTools.UnitTesting;
24+
25+
[TestClass]
26+
public class TestClass
27+
{
28+
[TestMethod]
29+
[DataRow(""value1"", 1)]
30+
[DataRow(""value2"", 2)]
31+
[DataRow(""value3"", 3)]
32+
[DataRow(""value4"", 4)]
33+
[TestCategory(TestDefinitions.UnitTests)]
34+
public void TestMethodWithManyDataRows(string text, int number)
35+
{
36+
// Test implementation
37+
}
38+
}
39+
";
40+
41+
DiagnosticResult expected = new()
42+
{
43+
Id = DiagnosticId.PreferCombinatorialTestingOverDataRows.ToId(),
44+
Message = new Regex(@"Consider using combinatorial testing instead of multiple DataRow attributes. This method has 4 DataRow attributes."),
45+
Severity = DiagnosticSeverity.Info,
46+
Locations = new[] { new DiagnosticResultLocation("Test0.cs", 13, 14) }
47+
};
48+
49+
await VerifyDiagnostic(testCode, expected).ConfigureAwait(false);
50+
}
51+
52+
[TestMethod]
53+
[TestCategory(TestDefinitions.UnitTests)]
54+
public async Task PreferCombinatorialTestingNoDiagnosticWithFewDataRowsAsync()
55+
{
56+
var testCode = @"
57+
using Microsoft.VisualStudio.TestTools.UnitTesting;
58+
59+
[TestClass]
60+
public class TestClass
61+
{
62+
[TestMethod]
63+
[DataRow(""value1"", 1)]
64+
[DataRow(""value2"", 2)]
65+
[DataRow(""value3"", 3)]
66+
[TestCategory(TestDefinitions.UnitTests)]
67+
public void TestMethodWithFewDataRows(string text, int number)
68+
{
69+
// Test implementation
70+
}
71+
}
72+
";
73+
74+
await VerifySuccessfulCompilation(testCode).ConfigureAwait(false);
75+
}
76+
77+
[TestMethod]
78+
[TestCategory(TestDefinitions.UnitTests)]
79+
public async Task PreferCombinatorialTestingNoDiagnosticWithNoDataRowsAsync()
80+
{
81+
var testCode = @"
82+
using Microsoft.VisualStudio.TestTools.UnitTesting;
83+
84+
[TestClass]
85+
public class TestClass
86+
{
87+
[TestMethod]
88+
[TestCategory(TestDefinitions.UnitTests)]
89+
public void TestMethodWithoutDataRows()
90+
{
91+
// Test implementation
92+
}
93+
}
94+
";
95+
96+
await VerifySuccessfulCompilation(testCode).ConfigureAwait(false);
97+
}
98+
99+
[TestMethod]
100+
[TestCategory(TestDefinitions.UnitTests)]
101+
public async Task PreferCombinatorialTestingTriggersDiagnosticWithManyDataRowsAsync()
102+
{
103+
var testCode = @"
104+
using Microsoft.VisualStudio.TestTools.UnitTesting;
105+
106+
[TestClass]
107+
public class TestClass
108+
{
109+
[TestMethod]
110+
[DataRow(""private static readonly"", false)]
111+
[DataRow(""private static"", false)]
112+
[DataRow(""private const"", false)]
113+
[DataRow(""private"", false)]
114+
[DataRow(""public static readonly"", true)]
115+
[DataRow(""public const"", true)]
116+
[DataRow(""public static"", true)]
117+
[DataRow(""public readonly"", true)]
118+
[DataRow(""public"", true)]
119+
[TestCategory(TestDefinitions.UnitTests)]
120+
public void TestMethodWithManyDataRows(string modifiers, bool expected)
121+
{
122+
// Test implementation similar to HelperTest example
123+
}
124+
}
125+
";
126+
127+
DiagnosticResult expected = new()
128+
{
129+
Id = DiagnosticId.PreferCombinatorialTestingOverDataRows.ToId(),
130+
Message = new Regex(@"Consider using combinatorial testing instead of multiple DataRow attributes. This method has 9 DataRow attributes."),
131+
Severity = DiagnosticSeverity.Info,
132+
Locations = new[] { new DiagnosticResultLocation("Test0.cs", 18, 14) }
133+
};
134+
135+
await VerifyDiagnostic(testCode, expected).ConfigureAwait(false);
136+
}
137+
138+
protected override DiagnosticAnalyzer GetDiagnosticAnalyzer()
139+
{
140+
return new PreferCombinatorialTestingAnalyzer();
141+
}
142+
}
143+
}

0 commit comments

Comments
 (0)