Skip to content

Commit 8f1612a

Browse files
nvmLantananvmLantana
committed
reduce log output for generation/upload
1 parent ee482ff commit 8f1612a

3 files changed

Lines changed: 94 additions & 14 deletions

File tree

‎DotNet/Automation.UI/Services/Persistence/GeneratedTemplateCacheVersionStore.cs‎

Lines changed: 60 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
using MongoDB.Bson.Serialization.Attributes;
2+
using MongoDB.Bson;
23
using MongoDB.Driver;
34
using System.Security.Cryptography;
45
using System.Text;
@@ -34,12 +35,70 @@ public sealed class GeneratedTemplateCacheVersionStore
3435
public GeneratedTemplateCacheVersionStore(IMongoDatabase database)
3536
{
3637
_versions = database.GetCollection<GeneratedTemplateCacheVersionDocument>("automation_generated_template_versions");
38+
39+
if (HasIndexWithKeys(_versions, new BsonDocument { { "ScenarioKey", 1 }, { "TemplateSetHash", 1 } }))
40+
return;
41+
3742
var uniqueScenarioHashIndex = new CreateIndexModel<GeneratedTemplateCacheVersionDocument>(
3843
Builders<GeneratedTemplateCacheVersionDocument>.IndexKeys
3944
.Ascending(version => version.ScenarioKey)
4045
.Ascending(version => version.TemplateSetHash),
4146
new CreateIndexOptions { Unique = true, Name = ScenarioHashUniqueIndexName });
42-
_versions.Indexes.CreateOne(uniqueScenarioHashIndex);
47+
48+
try
49+
{
50+
_versions.Indexes.CreateOne(uniqueScenarioHashIndex);
51+
}
52+
catch (MongoCommandException ex)
53+
when (ex.Message.Contains("already exists with different options", StringComparison.OrdinalIgnoreCase)
54+
|| ex.Message.Contains("Cannot create unique index when collection contains documents", StringComparison.OrdinalIgnoreCase)
55+
|| (ex.Code == 13 && ex.Message.Contains("unique index cannot be modified", StringComparison.OrdinalIgnoreCase)))
56+
{
57+
// Deployed environments may already have an index with this name/options
58+
// drift or contain pre-existing duplicate rows. Do not block app startup.
59+
}
60+
}
61+
62+
private static bool HasIndexWithKeys(
63+
IMongoCollection<GeneratedTemplateCacheVersionDocument> collection,
64+
BsonDocument targetKeys)
65+
{
66+
var indexes = collection.Indexes.List().ToList();
67+
68+
foreach (var index in indexes)
69+
{
70+
if (index.TryGetValue("key", out var keyValue)
71+
&& keyValue.IsBsonDocument
72+
&& KeysEqual(keyValue.AsBsonDocument, targetKeys))
73+
{
74+
return true;
75+
}
76+
}
77+
78+
return false;
79+
}
80+
81+
private static bool KeysEqual(BsonDocument existing, BsonDocument target)
82+
{
83+
if (existing.ElementCount != target.ElementCount)
84+
return false;
85+
86+
var existingElements = existing.Elements.ToList();
87+
var targetElements = target.Elements.ToList();
88+
89+
for (var i = 0; i < existingElements.Count; i++)
90+
{
91+
var left = existingElements[i];
92+
var right = targetElements[i];
93+
94+
if (!string.Equals(left.Name, right.Name, StringComparison.OrdinalIgnoreCase))
95+
return false;
96+
97+
if (left.Value.ToInt32() != right.Value.ToInt32())
98+
return false;
99+
}
100+
101+
return true;
43102
}
44103

45104
public async Task<GeneratedTemplateCacheVersionBinding?> BindRunAsync(

‎DotNet/Automation/FhirDataLoader.cs‎

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -403,7 +403,8 @@ public async Task LoadTransactionBundlesFromJsonAsync(
403403
public async Task<bool> UploadBundlesSequentiallyAsync(
404404
IAutomationOutput output,
405405
IReadOnlyList<(string Name, string Json)> bundles,
406-
string progressPrefix = "")
406+
string progressPrefix = "",
407+
bool logSuccessfulPosts = true)
407408
{
408409
for (var i = 0; i < bundles.Count; i++)
409410
{
@@ -412,7 +413,7 @@ public async Task<bool> UploadBundlesSequentiallyAsync(
412413
? $"[{i + 1}/{bundles.Count}]"
413414
: $"{progressPrefix}[{i + 1}/{bundles.Count}]";
414415

415-
var response = await PostBundleWithRetryAsync(json, name, progress, output);
416+
var response = await PostBundleWithRetryAsync(json, name, progress, output, logSuccessfulPosts);
416417

417418
if (!response.IsSuccessful || string.IsNullOrWhiteSpace(response.Content))
418419
{
@@ -783,7 +784,8 @@ private async Task<RestResponse> PostBundleWithRetryAsync(
783784
string bundleJson,
784785
string name,
785786
string progress,
786-
IAutomationOutput output)
787+
IAutomationOutput output,
788+
bool logSuccessfulPosts = true)
787789
{
788790
var delay = InitialRetryDelay;
789791
RestResponse? lastResponse = null;
@@ -802,10 +804,13 @@ private async Task<RestResponse> PostBundleWithRetryAsync(
802804

803805
if (lastResponse.IsSuccessful)
804806
{
805-
if (attempt > 1)
806-
output.WriteLine($" {progress} Posted {name} => {lastResponse.StatusCode} (succeeded on attempt {attempt})");
807-
else
808-
output.WriteLine($" {progress} Posted {name} => {lastResponse.StatusCode}");
807+
if (logSuccessfulPosts)
808+
{
809+
if (attempt > 1)
810+
output.WriteLine($" {progress} Posted {name} => {lastResponse.StatusCode} (succeeded on attempt {attempt})");
811+
else
812+
output.WriteLine($" {progress} Posted {name} => {lastResponse.StatusCode}");
813+
}
809814
return lastResponse;
810815
}
811816

‎DotNet/Automation/Generation/FhirGenerationPipeline.cs‎

Lines changed: 22 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,8 @@ public static class FhirGenerationPipeline
3232
{
3333
private const int MaxEntriesPerBundle = 500;
3434
private const int DefaultMaxConcurrentPatients = 4;
35+
private const int VerbosePatientLogHeadCount = 5;
36+
private const int VerbosePatientLogInterval = 250;
3537
private const string TemplateRunTag = "template-run";
3638
private static readonly Lazy<string> GeneratorDependencyFingerprint = new(ComputeGeneratorDependencyFingerprint);
3739

@@ -363,7 +365,8 @@ public static async Task<PipelineResult> GenerateAndUploadAsync(
363365
{
364366
var templateBundles = bundles.Select(b => ReplaceRunTag(b.Json, ids.RunTag, TemplateRunTag)).ToList();
365367
await generatedTemplateCache.StoreAsync(templateCacheKey, new GeneratedPatientTemplate(TemplateRunTag, templateBundles));
366-
output.WriteLine($" [cache] Miss for {patientId}; stored template key={templateCacheKey}.");
368+
if (ShouldEmitDetailedPatientLog(patientIndex))
369+
output.WriteLine($" [cache] Miss for {patientId}; stored template key={templateCacheKey}.");
367370
}
368371
}
369372
else
@@ -377,7 +380,8 @@ public static async Task<PipelineResult> GenerateAndUploadAsync(
377380
.ToList();
378381

379382
entries = ParseBundleEntriesFromJson(materialized);
380-
output.WriteLine($" [cache] Hit for {patientId}; reused template key={templateCacheKey}.");
383+
if (ShouldEmitDetailedPatientLog(patientIndex))
384+
output.WriteLine($" [cache] Hit for {patientId}; reused template key={templateCacheKey}.");
381385
}
382386

383387
var scenario = FhirGenerationCodes.GetScenarioById(profile.ClinicalScenarioId)
@@ -433,8 +437,11 @@ public static async Task<PipelineResult> GenerateAndUploadAsync(
433437
return $"{shortName}={eligible}";
434438
}));
435439

436-
output.WriteLine($" Patient {patientId}: {entries.Count} entries [{measureEligibilityLabel}] | scenario={scenario.PrimaryDxDisplay} | " +
437-
$"encounter={encounterId} ({encStart:yyyy-MM-dd} ? {encEnd:yyyy-MM-dd})");
440+
if (ShouldEmitDetailedPatientLog(patientIndex))
441+
{
442+
output.WriteLine($" Patient {patientId}: {entries.Count} entries [{measureEligibilityLabel}] | scenario={scenario.PrimaryDxDisplay} | " +
443+
$"encounter={encounterId} ({encStart:yyyy-MM-dd} ? {encEnd:yyyy-MM-dd})");
444+
}
438445

439446
// Record patient in manifest builder
440447
manifestBuilder.AddPatient(patientId, effectiveProfile);
@@ -466,7 +473,7 @@ public static async Task<PipelineResult> GenerateAndUploadAsync(
466473
entries.Clear();
467474

468475
var progress = $"[{patientId}] ";
469-
await fhirDataLoader.UploadBundlesSequentiallyAsync(output, bundles, progress);
476+
await fhirDataLoader.UploadBundlesSequentiallyAsync(output, bundles, progress, logSuccessfulPosts: false);
470477

471478
var bundleCount = bundles.Count;
472479

@@ -620,7 +627,7 @@ public static async Task<PipelineResult> GenerateAndUploadAsync(
620627
{
621628
var bundles = ChunkEntries(entries, patientId, 0);
622629
entries.Clear();
623-
var ok = await fhirDataLoader.UploadBundlesSequentiallyAsync(output, bundles, $"[imported:{patientId}] ");
630+
var ok = await fhirDataLoader.UploadBundlesSequentiallyAsync(output, bundles, $"[imported:{patientId}] ", logSuccessfulPosts: false);
624631
if (!ok)
625632
throw new InvalidOperationException($"Failed to upload imported bundle for patient '{patientId}'.");
626633
bundleCount = bundles.Count;
@@ -1075,4 +1082,13 @@ private static (DateTime Start, DateTime End) DeriveScheduledPatternInpatientWin
10751082

10761083
return result;
10771084
}
1085+
1086+
private static bool ShouldEmitDetailedPatientLog(int patientIndex)
1087+
{
1088+
if (patientIndex < VerbosePatientLogHeadCount)
1089+
return true;
1090+
1091+
var ordinal = patientIndex + 1;
1092+
return ordinal % VerbosePatientLogInterval == 0;
1093+
}
10781094
}

0 commit comments

Comments
 (0)