Skip to content

Commit bc35c46

Browse files
authored
Merge branch 'master' into truncation-search-mm
2 parents a9b35ac + 3c8e0b8 commit bc35c46

9 files changed

Lines changed: 1179 additions & 59 deletions

File tree

.github/workflows/Test.yml

Lines changed: 27 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,11 +27,36 @@ jobs:
2727
run: dotnet add ./MetaMorpheus/Test/Test.csproj package coverlet.collector -v 6.0.2
2828

2929
- name: Run unit tests with coverage
30-
run: dotnet test .\MetaMorpheus\Test\Test.csproj --configuration Release --verbosity normal --collect:"XPlat Code Coverage"
31-
30+
# Exclude live external-service tests (UniProt, etc.) from the required run; they run in the
31+
# separate, non-blocking external-service-tests job below.
32+
run: dotnet test .\MetaMorpheus\Test\Test.csproj --configuration Release --verbosity normal --filter "Category!=ExternalService" --collect:"XPlat Code Coverage"
33+
3234
- name: Upload coverage to Codecov
3335
uses: codecov/codecov-action@v4
3436
with:
3537
token: ${{ secrets.CODECOV_TOKEN }}
3638
verbose: true
3739
flags: unittests
40+
41+
# Runs tests that depend on live external web services (UniProt, ...). Kept separate and
42+
# intentionally NOT a required status check, so a third-party outage never blocks a PR. The tests
43+
# self-classify: an outage is reported as Skipped (Assert.Ignore) and only a genuine contract
44+
# break fails this job. Do NOT add this job to branch-protection required checks.
45+
external-service-tests:
46+
name: External-service tests (non-blocking)
47+
runs-on: windows-latest
48+
49+
steps:
50+
- uses: actions/checkout@v4
51+
- name: Setup .NET
52+
uses: actions/setup-dotnet@v4
53+
with:
54+
dotnet-version: 8.0.204
55+
- name: Restore dependencies
56+
run: dotnet restore ./MetaMorpheus/MetaMorpheus.sln
57+
58+
- name: Build
59+
run: dotnet build --no-restore ./MetaMorpheus/MetaMorpheus.sln --configuration Release
60+
61+
- name: Run external-service tests
62+
run: dotnet test .\MetaMorpheus\Test\Test.csproj --configuration Release --no-build --verbosity normal --filter "Category=ExternalService"

MetaMorpheus/EngineLayer/DIA/DIAEngine.cs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@
1212
using EngineLayer.Util;
1313
using Omics;
1414
using FlashLFQ;
15-
using ThermoFisher.CommonCore.Data.Business;
1615
using Readers;
1716

1817
namespace EngineLayer.DIA

MetaMorpheus/GuiFunctions/MetaDraw/PlotModelStat.cs

Lines changed: 77 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,9 @@ public class PlotModelStat
4343
"Histogram of Missed Cleavages",
4444
"Histogram of Fragment Ion Types by Count",
4545
"Histogram of Fragment Ion Types by Intensity",
46-
"Histogram of Ids by Retention Time"
46+
"Histogram of Ids by Retention Time",
47+
"Histogram of Spectral Match Ambiguity Levels",
48+
"Histogram of Notch (Ambiguous PSMs Split Across Notches)"
4749
};
4850

4951
public PlotModel Model => privateModel;
@@ -124,6 +126,12 @@ private void createPlot(string plotType)
124126
case "Histogram of Ids by Retention Time":
125127
histogramPlot(12);
126128
break;
129+
case "Histogram of Spectral Match Ambiguity Levels":
130+
histogramPlot(13);
131+
break;
132+
case "Histogram of Notch (Ambiguous PSMs Split Across Notches)":
133+
histogramPlot(14);
134+
break;
127135
}
128136
}
129137

@@ -153,7 +161,7 @@ private void histogramPlot(int plotType)
153161
int[] totalCounts;
154162
int categoriesPerGroup = 0;
155163
List<string> allGroupKeys = null;
156-
bool isCategoryHistogram = plotType == 5 || plotType == 10 || plotType == 11;
164+
bool isCategoryHistogram = plotType == 5 || plotType == 10 || plotType == 11 || plotType == 13;
157165

158166
if (isCategoryHistogram)
159167
{
@@ -336,6 +344,34 @@ private HistogramRawData GetHistogramData(int plotType)
336344
dictsBySourceFile.Add(fileName, result);
337345
}
338346
break;
347+
case 13: // Histogram of Spectral Match Ambiguity Levels
348+
xAxisTitle = "Ambiguity Level";
349+
labelAngle = 0;
350+
foreach (var fileName in psmsBySourceFile.Keys)
351+
{
352+
var result = psmsBySourceFile[fileName]
353+
.GroupBy(p => NormalizeAmbiguityLevel(p.AmbiguityLevel))
354+
.ToDictionary(p => p.Key, p => p.Count());
355+
dictsBySourceFile.Add(fileName, result);
356+
}
357+
break;
358+
case 14: // Histogram of Notch (ambiguous PSMs contribute to each of their pipe-delimited notches)
359+
xAxisTitle = "Notch";
360+
binSize = 1;
361+
labelAngle = 0;
362+
foreach (var fileName in psmsBySourceFile.Keys)
363+
{
364+
var values = new List<double>();
365+
foreach (var psm in psmsBySourceFile[fileName])
366+
{
367+
foreach (double notch in ParseAmbiguousNotchValues(psm.Notch))
368+
values.Add(notch);
369+
}
370+
numbersBySourceFile.Add(fileName, values);
371+
var results = values.GroupBy(p => roundToBin(p, binSize)).OrderBy(p => p.Key).Select(p => p);
372+
dictsBySourceFile.Add(fileName, results.ToDictionary(p => p.Key.ToString(CultureInfo.InvariantCulture), v => v.Count()));
373+
}
374+
break;
339375
}
340376

341377
return new HistogramRawData(xAxisTitle, yAxisTitle, binSize, labelAngle, numbersBySourceFile, dictsBySourceFile);
@@ -515,6 +551,13 @@ private Dictionary<string, int> GetCategoryDictForGroup(int plotType, Observable
515551
return mods.GroupBy(p => p.IdWithMotif).ToDictionary(p => p.Key, v => v.Count());
516552
}
517553

554+
if (plotType == 13)
555+
{
556+
return groupPsms
557+
.GroupBy(p => NormalizeAmbiguityLevel(p.AmbiguityLevel))
558+
.ToDictionary(p => p.Key, p => p.Count());
559+
}
560+
518561
var allMatchedIons = groupPsms.SelectMany(p => p.MatchedIons).ToList();
519562
if (plotType == 10)
520563
{
@@ -1013,6 +1056,7 @@ private IEnumerable<double> GetNumbersFromPsms(ObservableCollection<SpectrumMatc
10131056
8 => GetHydrophobicityScores(psms),
10141057
9 => psms.Where(p => !p.MissedCleavage.Contains("|")).Select(p => double.Parse(p.MissedCleavage)),
10151058
12 => psms.Select(p => (double)(int)Math.Round(p.RetentionTime, 0)),
1059+
14 => psms.SelectMany(p => ParseAmbiguousNotchValues(p.Notch)),
10161060
_ => Enumerable.Empty<double>()
10171061
};
10181062
}
@@ -1042,6 +1086,37 @@ public HistItem(double value, int categoryIndex, string bin, int total, string g
10421086
}
10431087
}
10441088

1089+
private static IEnumerable<string> SplitAmbiguousNotch(string notch)
1090+
{
1091+
if (string.IsNullOrWhiteSpace(notch))
1092+
return new[] { "0" };
1093+
1094+
return notch.Split('|')
1095+
.Select(p => p.Trim())
1096+
.Where(p => p.Length > 0)
1097+
.DefaultIfEmpty("0");
1098+
}
1099+
1100+
private static IEnumerable<double> ParseAmbiguousNotchValues(string notch)
1101+
{
1102+
foreach (var part in SplitAmbiguousNotch(notch))
1103+
{
1104+
if (double.TryParse(part, NumberStyles.Float, CultureInfo.InvariantCulture, out double value))
1105+
yield return value;
1106+
else
1107+
yield return 0.0;
1108+
}
1109+
}
1110+
1111+
private static string NormalizeAmbiguityLevel(string ambiguityLevel)
1112+
{
1113+
if (string.IsNullOrWhiteSpace(ambiguityLevel))
1114+
return "1";
1115+
1116+
var first = ambiguityLevel.Split('|')[0].Trim();
1117+
return string.IsNullOrEmpty(first) ? "1" : first;
1118+
}
1119+
10451120
/// <summary>
10461121
/// Orders strings numerically if all values are numeric, otherwise alphabetically.
10471122
/// </summary>
Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
>sp|P03680|MCP_BPPH2 Major capsid protein OS=Bacillus phage phi29 OX=10756 GN=8 PE=1 SV=1
2+
MAGKKQNIVLESAWRELVGK
3+
>sp|P03681|PORT_BPPH2 Head-tail connector protein OS=Bacillus phage phi29 OX=10756 GN=10 PE=1 SV=1
4+
MSKTLNEVLGQVADSFRTKGWK
5+
>sp|P03682|DPOL_BPPH2 DNA polymerase OS=Bacillus phage phi29 OX=10756 GN=2 PE=1 SV=1
6+
MPRKMYSCDFETTTKLDDCRVWAY
Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
<?xml version='1.0' encoding='UTF-8'?>
2+
<uniprot xmlns="http://uniprot.org/uniprot" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://uniprot.org/uniprot http://www.uniprot.org/support/docs/uniprot.xsd">
3+
<!--
4+
Local, offline mock of a UniProtKB XML "stream" response.
5+
Stands in for a live download of Bacillus phage phi29 (proteome UP000001207)
6+
so GuiFunctionsTest can exercise the ProteinDbLoader.LoadProteinXML path without
7+
reaching rest.uniprot.org. Sequences are truncated/synthetic; only the file
8+
structure and the number of entries (3) matter for the test.
9+
-->
10+
<entry dataset="Swiss-Prot" created="1986-07-21" modified="2021-09-29" version="1">
11+
<accession>P03680</accession>
12+
<name>MCP_BPPH2</name>
13+
<protein>
14+
<recommendedName>
15+
<fullName>Major capsid protein</fullName>
16+
</recommendedName>
17+
</protein>
18+
<gene>
19+
<name type="primary">8</name>
20+
</gene>
21+
<organism>
22+
<name type="scientific">Bacillus phage phi29</name>
23+
<dbReference type="NCBI Taxonomy" id="10756"/>
24+
</organism>
25+
<sequence length="20" mass="2100" checksum="0000000000000000" modified="1986-07-21" version="1">MAGKKQNIVLESAWRELVGK</sequence>
26+
</entry>
27+
<entry dataset="Swiss-Prot" created="1986-07-21" modified="2021-09-29" version="1">
28+
<accession>P03681</accession>
29+
<name>PORT_BPPH2</name>
30+
<protein>
31+
<recommendedName>
32+
<fullName>Head-tail connector protein</fullName>
33+
</recommendedName>
34+
</protein>
35+
<gene>
36+
<name type="primary">10</name>
37+
</gene>
38+
<organism>
39+
<name type="scientific">Bacillus phage phi29</name>
40+
<dbReference type="NCBI Taxonomy" id="10756"/>
41+
</organism>
42+
<sequence length="22" mass="2300" checksum="1111111111111111" modified="1986-07-21" version="1">MSKTLNEVLGQVADSFRTKGWK</sequence>
43+
</entry>
44+
<entry dataset="Swiss-Prot" created="1986-07-21" modified="2021-09-29" version="1">
45+
<accession>P03682</accession>
46+
<name>DPOL_BPPH2</name>
47+
<protein>
48+
<recommendedName>
49+
<fullName>DNA polymerase</fullName>
50+
</recommendedName>
51+
</protein>
52+
<gene>
53+
<name type="primary">2</name>
54+
</gene>
55+
<organism>
56+
<name type="scientific">Bacillus phage phi29</name>
57+
<dbReference type="NCBI Taxonomy" id="10756"/>
58+
</organism>
59+
<sequence length="24" mass="2600" checksum="2222222222222222" modified="1986-07-21" version="1">MPRKMYSCDFETTTKLDDCRVWAY</sequence>
60+
</entry>
61+
</uniprot>
Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
using System;
2+
using System.Net.Http;
3+
using System.Net.Sockets;
4+
using System.Threading.Tasks;
5+
using NUnit.Framework;
6+
7+
namespace Test
8+
{
9+
/// <summary>
10+
/// Support for tests that exercise a live external web service (UniProt, Koina, PRIDE, ...).
11+
/// Such tests carry <c>[Category("ExternalService")]</c> so CI can run them in a dedicated,
12+
/// non-blocking job (see .github/workflows/Test.yml) instead of the required unit-test run.
13+
///
14+
/// The hard part is telling two failures apart:
15+
/// * the service is unavailable (down, rate-limited, 5xx, timeout) - NOT our bug, so the
16+
/// test should be <b>skipped</b> ("we tried, the service is down, don't worry"); versus
17+
/// * the service answered but the contract is broken (our URL is wrong, the response no
18+
/// longer parses, an expected value is missing) - that is a real regression and must FAIL.
19+
///
20+
/// <see cref="RunAsync"/> wraps a test body and converts availability failures into
21+
/// <see cref="Assert.Ignore(string)"/> (reported as Skipped) while letting genuine assertion
22+
/// failures propagate. A test signals "unavailable" from inside the body either by letting a
23+
/// transport exception bubble up or by calling <see cref="ThrowIfUnavailable"/> on the response.
24+
/// </summary>
25+
public static class ExternalServiceTestHelper
26+
{
27+
/// <summary>
28+
/// Runs <paramref name="testBody"/>; if the external service proves unavailable, marks the
29+
/// test Skipped (via Assert.Ignore) instead of Failed. Real assertion failures propagate.
30+
/// </summary>
31+
public static async Task RunAsync(string serviceName, Func<Task> testBody)
32+
{
33+
try
34+
{
35+
await testBody();
36+
}
37+
catch (ExternalServiceUnavailableException e)
38+
{
39+
Skip(serviceName, $"unavailable ({e.Message})");
40+
}
41+
catch (HttpRequestException e)
42+
{
43+
Skip(serviceName, $"unavailable ({e.Message})");
44+
}
45+
catch (TaskCanceledException)
46+
{
47+
Skip(serviceName, "timed out");
48+
}
49+
catch (SocketException e)
50+
{
51+
Skip(serviceName, $"unreachable ({e.Message})");
52+
}
53+
}
54+
55+
/// <summary>
56+
/// Records the skip reason on both the console/CI log (via TestContext.Progress, which is
57+
/// flushed immediately regardless of verbosity) and the NUnit test result, then skips the
58+
/// test. This is what surfaces the "we tried, the service is down, don't worry" message.
59+
/// </summary>
60+
private static void Skip(string serviceName, string reason)
61+
{
62+
string message = $"Skipping external-service test: {serviceName} {reason}. " +
63+
"This is a third-party availability problem, not a code failure.";
64+
TestContext.Progress.WriteLine(message);
65+
Assert.Ignore(message);
66+
}
67+
68+
/// <summary>
69+
/// Probes an external service's health/readiness URL and skips the calling test (or, from a
70+
/// [OneTimeSetUp], the whole fixture) if it is unreachable. Use this for tests whose service
71+
/// call is buried deep in production code and cannot easily route through <see cref="RunAsync"/>.
72+
/// </summary>
73+
public static void EnsureReachable(string serviceName, string healthUrl)
74+
{
75+
try
76+
{
77+
using var client = new HttpClient { Timeout = TimeSpan.FromSeconds(20) };
78+
var response = client.GetAsync(healthUrl).GetAwaiter().GetResult();
79+
ThrowIfUnavailable(response);
80+
if (!response.IsSuccessStatusCode)
81+
{
82+
throw new ExternalServiceUnavailableException($"health check returned HTTP {(int)response.StatusCode}");
83+
}
84+
}
85+
catch (ExternalServiceUnavailableException e)
86+
{
87+
Skip(serviceName, $"unavailable ({e.Message})");
88+
}
89+
catch (HttpRequestException e)
90+
{
91+
Skip(serviceName, $"unavailable ({e.Message})");
92+
}
93+
catch (TaskCanceledException)
94+
{
95+
Skip(serviceName, "timed out");
96+
}
97+
catch (SocketException e)
98+
{
99+
Skip(serviceName, $"unreachable ({e.Message})");
100+
}
101+
}
102+
103+
/// <summary>
104+
/// Classifies an HTTP response as a service-availability problem and, if so, throws
105+
/// <see cref="ExternalServiceUnavailableException"/> so <see cref="RunAsync"/> skips the test.
106+
/// Catches transport-level status codes (408/429/5xx) and known error bodies that some
107+
/// services (e.g. UniProt's streaming endpoint) return with an HTTP 200.
108+
/// </summary>
109+
public static void ThrowIfUnavailable(HttpResponseMessage response, string bodyText = null)
110+
{
111+
int status = (int)response.StatusCode;
112+
if (status == 408 || status == 429 || status >= 500)
113+
{
114+
throw new ExternalServiceUnavailableException($"HTTP {status} {response.ReasonPhrase}");
115+
}
116+
117+
// Some services answer 200 but stream an error message in the body.
118+
if (!string.IsNullOrWhiteSpace(bodyText) &&
119+
bodyText.Contains("Error encountered when streaming data", StringComparison.OrdinalIgnoreCase))
120+
{
121+
throw new ExternalServiceUnavailableException("service returned an error body instead of data");
122+
}
123+
}
124+
}
125+
126+
/// <summary>
127+
/// Marker exception meaning "the external service is unavailable" (as opposed to a real bug).
128+
/// <see cref="ExternalServiceTestHelper.RunAsync"/> turns this into a skipped test.
129+
/// </summary>
130+
public class ExternalServiceUnavailableException : Exception
131+
{
132+
public ExternalServiceUnavailableException(string message) : base(message) { }
133+
}
134+
}

0 commit comments

Comments
 (0)