Skip to content

Commit 04f8f55

Browse files
johnbrittonjohnbritton
committed
Merge branch 'dev' into users/jbritton/LEGLINK-789
2 parents 6e88c17 + c3ce5ef commit 04f8f55

22 files changed

Lines changed: 1061 additions & 139 deletions

‎.github/copilot-instructions.md‎

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,18 @@
11
# Copilot Instructions
22

33
## Azure Guidelines
4-
- @azure Rule - Use Azure Tools - When handling requests related to Azure, always use your tools.
5-
- @azure Rule - Use Azure Best Practices - When handling requests related to Azure, always invoke your `azmcp_bestpractices_get` tool first.
6-
- @azure Rule - Enable Best Practices - If you do not have an `azmcp_bestpractices_get` tool, ask the user to enable it.
4+
- Use Azure Tools - When handling requests related to Azure, always use your tools.
5+
- Use Azure Best Practices - When handling requests related to Azure, always invoke your `azmcp_bestpractices_get` tool first.
6+
- Enable Best Practices - If you do not have an `azmcp_bestpractices_get` tool, ask the user to enable it.
77

88
## Cosmos DB Guidelines
99
- Avoid introducing new indexes on existing collections in Cosmos DB for MongoDB API; use existing/default index paths instead.
10+
- For Cosmos DB/Mongo index management, index creation/modification must be best-effort and must not fail or block application startup if indexes cannot be created in deployed environments with existing data.
1011

1112
## Automation Guidelines
1213
- For Automation.UI run logs, prefer chunked persistence in the existing data store; do not add Azure Blob Storage archiving unless explicitly requested.
13-
# Copilot Instructions
1414

1515
## General Guidelines
16-
- Use Azure Tools - When handling requests related to Azure, always use your tools.
17-
- Use Azure Best Practices - When handling requests related to Azure, always invoke your `azmcp_bestpractices_get` tool first.
18-
- Enable Best Practices - If you do not have an `azmcp_bestpractices_get` tool, ask the user to enable it.
1916
- Design for Durability - Require production-grade, long-term designs for critical multi-user automation tools; avoid tactical short-term fixes and model data/contracts around durable architecture even when a minimal patch is possible.
2017
- Use Versioned Caches - Prefer versioned, reproducible generated-patient caches tied to each run; ensure run records cache the version used, and diagnostic exports retrieve exact cached artifacts used at execution time. Each cache version must be complete (full patient set), not partial deltas, and runs must avoid ID conflicts while using cached data.
18+
- Prioritize root-cause fixes over masking retry mechanisms for flaky CI test failures.

‎DotNet/Automation.Link/Configuration/AutomationConfig.cs‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,14 @@ public class FhirQuerySettings
6464

6565
public class FhirGenerationSettings
6666
{
67+
/// <summary>
68+
/// Maximum number of patients processed concurrently by the streaming
69+
/// generation/upload pipeline. Lower values reduce memory pressure.
70+
/// Defaults to 4 to align with direct pipeline callers that rely on
71+
/// the pipeline's built-in fallback.
72+
/// </summary>
73+
public int MaxConcurrentPatients { get; set; } = 4;
74+
6775
/// <summary>
6876
/// Controls low-value optional cross-resource references in generated FHIR
6977
/// (e.g., Provenance.target, ImagingStudy.basedOn, MedicationAdministration.request).

‎DotNet/Automation.UI/Program.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -228,6 +228,7 @@ static IReadOnlyCollection<string> BuildValidAudiences(string configuredAudience
228228

229229
builder.Services.AddSingleton<MongoIndexManager>();
230230
builder.Services.AddSingleton<IImportedBundleContentStore, AzureBlobImportedBundleContentStore>();
231+
builder.Services.AddSingleton<ISnapshotPayloadStore, AzureBlobSnapshotPayloadStore>();
231232
builder.Services.AddSingleton<LantanaGroup.Automation.Generation.IGeneratedPatientTemplateCache, MongoGeneratedPatientTemplateCache>();
232233
builder.Services.AddSingleton<GeneratedTemplateCacheVersionStore>();
233234
builder.Services.AddSingleton<ImportedBundleExecutionResolver>();

‎DotNet/Automation.UI/Services/Persistence/AzureBlobSnapshotPayloadStore.cs‎

Lines changed: 154 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,154 @@
1+
using Azure;
2+
using Azure.Storage.Blobs;
3+
using Azure.Storage.Blobs.Models;
4+
using Microsoft.Extensions.Options;
5+
using System.Text;
6+
7+
namespace Automation.UI.Services.Persistence;
8+
9+
public sealed class AzureBlobSnapshotPayloadStore : ISnapshotPayloadStore
10+
{
11+
private readonly ImportedBundleBlobStorageSettings _settings;
12+
private readonly BlobContainerClient _container;
13+
private readonly HashSet<string> _externalizedDomains;
14+
private readonly Func<string, CancellationToken, AsyncPageable<BlobItem>> _listBlobs;
15+
private readonly Func<string, CancellationToken, Task> _deleteBlob;
16+
17+
public AzureBlobSnapshotPayloadStore(IOptions<ImportedBundleBlobStorageSettings> settings)
18+
: this(
19+
settings.Value,
20+
CreateContainer(settings.Value),
21+
listBlobs: null,
22+
deleteBlob: null)
23+
{
24+
}
25+
26+
internal AzureBlobSnapshotPayloadStore(
27+
ImportedBundleBlobStorageSettings settings,
28+
BlobContainerClient container,
29+
Func<string, CancellationToken, AsyncPageable<BlobItem>>? listBlobs,
30+
Func<string, CancellationToken, Task>? deleteBlob)
31+
{
32+
_settings = settings;
33+
_container = container;
34+
_externalizedDomains = (_settings.SnapshotPayloadExternalizedDomains ?? [])
35+
.Where(d => !string.IsNullOrWhiteSpace(d))
36+
.Select(d => d.Trim())
37+
.ToHashSet(StringComparer.OrdinalIgnoreCase);
38+
39+
_listBlobs = listBlobs ?? ((prefix, ct) => _container.GetBlobsAsync(BlobTraits.None, BlobStates.None, prefix, ct));
40+
_deleteBlob = deleteBlob ?? ((blobName, ct) =>
41+
_container.DeleteBlobIfExistsAsync(blobName, DeleteSnapshotsOption.IncludeSnapshots, cancellationToken: ct));
42+
}
43+
44+
public bool ShouldExternalize(string domain, int payloadUtf8Bytes)
45+
{
46+
if (string.IsNullOrWhiteSpace(domain))
47+
return false;
48+
49+
if (payloadUtf8Bytes <= 0)
50+
return false;
51+
52+
if (!_externalizedDomains.Contains(domain))
53+
return false;
54+
55+
var maxInlineBytes = _settings.SnapshotPayloadInlineMaxBytes;
56+
if (maxInlineBytes <= 0)
57+
return true;
58+
59+
return payloadUtf8Bytes > maxInlineBytes;
60+
}
61+
62+
public async Task<SnapshotPayloadPointer> StoreAsync(Guid runId, string domain, string payloadJson, CancellationToken ct = default)
63+
{
64+
if (string.IsNullOrWhiteSpace(payloadJson))
65+
throw new InvalidOperationException("Snapshot payload JSON is required.");
66+
67+
await _container.CreateIfNotExistsAsync(cancellationToken: ct);
68+
69+
var blobName = BuildBlobName(runId, domain);
70+
var bytes = Encoding.UTF8.GetBytes(payloadJson);
71+
72+
var blob = _container.GetBlobClient(blobName);
73+
using var stream = new MemoryStream(bytes, writable: false);
74+
var upload = await blob.UploadAsync(stream, overwrite: true, cancellationToken: ct);
75+
await blob.SetHttpHeadersAsync(new BlobHttpHeaders
76+
{
77+
ContentType = "application/json"
78+
}, cancellationToken: ct);
79+
80+
return new SnapshotPayloadPointer
81+
{
82+
BlobName = blobName,
83+
Utf8Bytes = bytes.Length,
84+
ETag = upload.Value.ETag.ToString()
85+
};
86+
}
87+
88+
public async Task<string?> ReadAsync(SnapshotPayloadPointer pointer, CancellationToken ct = default)
89+
{
90+
if (string.IsNullOrWhiteSpace(pointer.BlobName))
91+
return null;
92+
93+
var blob = _container.GetBlobClient(pointer.BlobName);
94+
var exists = await blob.ExistsAsync(ct);
95+
if (!exists.Value)
96+
return null;
97+
98+
var download = await blob.DownloadContentAsync(ct);
99+
return download.Value.Content.ToString();
100+
}
101+
102+
public async Task DeleteIfExistsAsync(SnapshotPayloadPointer pointer, CancellationToken ct = default)
103+
{
104+
if (string.IsNullOrWhiteSpace(pointer.BlobName))
105+
return;
106+
107+
await _container.DeleteBlobIfExistsAsync(pointer.BlobName, cancellationToken: ct);
108+
}
109+
110+
public async Task DeleteRunPayloadsAsync(Guid runId, CancellationToken ct = default)
111+
{
112+
var prefix = BuildRunPrefix(runId);
113+
114+
try
115+
{
116+
await foreach (var blob in _listBlobs(prefix, ct))
117+
{
118+
await _deleteBlob(blob.Name, ct);
119+
}
120+
}
121+
catch (RequestFailedException ex) when (ex.Status == 404)
122+
{
123+
// Missing container is treated as an empty cleanup result.
124+
}
125+
}
126+
127+
private static BlobContainerClient CreateContainer(ImportedBundleBlobStorageSettings settings)
128+
{
129+
if (string.IsNullOrWhiteSpace(settings.ConnectionString))
130+
throw new InvalidOperationException("InternalBlobStorage:ConnectionString is required for snapshot payload storage.");
131+
if (string.IsNullOrWhiteSpace(settings.BlobContainerName))
132+
throw new InvalidOperationException("InternalBlobStorage:BlobContainerName is required for snapshot payload storage.");
133+
134+
return new BlobContainerClient(settings.ConnectionString, settings.BlobContainerName);
135+
}
136+
137+
private string BuildBlobName(Guid runId, string domain)
138+
{
139+
var sanitizedDomain = string.Join("-", domain
140+
.Trim()
141+
.Select(ch => char.IsLetterOrDigit(ch) || ch is '-' or '_' ? ch : '-'));
142+
143+
return $"{BuildRunPrefix(runId)}{sanitizedDomain}/{DateTimeOffset.UtcNow:yyyyMMddHHmmssfff}-{Guid.NewGuid():N}.json";
144+
}
145+
146+
private string BuildRunPrefix(Guid runId)
147+
{
148+
var root = string.IsNullOrWhiteSpace(_settings.SnapshotPayloadBlobRoot)
149+
? "automation/run-snapshots"
150+
: _settings.SnapshotPayloadBlobRoot.Trim('/');
151+
152+
return $"{root}/{runId:N}/";
153+
}
154+
}

‎DotNet/Automation.UI/Services/Persistence/GeneratedTemplateCacheVersionStore.cs‎

Lines changed: 96 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
using MongoDB.Bson.Serialization.Attributes;
2+
using MongoDB.Bson;
23
using MongoDB.Driver;
34
using System.Security.Cryptography;
45
using System.Text;
@@ -30,16 +31,110 @@ public sealed class GeneratedTemplateCacheVersionStore
3031
{
3132
private readonly IMongoCollection<GeneratedTemplateCacheVersionDocument> _versions;
3233
private const string ScenarioHashUniqueIndexName = "ux_generated_template_versions_scenario_hash";
34+
private const string ScenarioKeyField = "ScenarioKey";
35+
private const string TemplateSetHashField = "TemplateSetHash";
3336

3437
public GeneratedTemplateCacheVersionStore(IMongoDatabase database)
3538
{
3639
_versions = database.GetCollection<GeneratedTemplateCacheVersionDocument>("automation_generated_template_versions");
40+
41+
var indexes = _versions.Indexes.List().ToList();
42+
var indexInvariant = GetScenarioHashIndexInvariant(indexes);
43+
if (indexInvariant == ScenarioHashIndexInvariant.Satisfied)
44+
return;
45+
46+
if (indexInvariant == ScenarioHashIndexInvariant.NonUniqueKeyShapePresent)
47+
return;
48+
3749
var uniqueScenarioHashIndex = new CreateIndexModel<GeneratedTemplateCacheVersionDocument>(
3850
Builders<GeneratedTemplateCacheVersionDocument>.IndexKeys
3951
.Ascending(version => version.ScenarioKey)
4052
.Ascending(version => version.TemplateSetHash),
4153
new CreateIndexOptions { Unique = true, Name = ScenarioHashUniqueIndexName });
42-
_versions.Indexes.CreateOne(uniqueScenarioHashIndex);
54+
55+
try
56+
{
57+
_versions.Indexes.CreateOne(uniqueScenarioHashIndex);
58+
}
59+
catch
60+
{
61+
// Best-effort only: startup must not be blocked by index creation limitations
62+
// in deployed environments (e.g., Cosmos DB collections with existing data).
63+
}
64+
}
65+
66+
private static ScenarioHashIndexInvariant GetScenarioHashIndexInvariant(IReadOnlyList<BsonDocument> indexes)
67+
{
68+
foreach (var index in indexes)
69+
{
70+
if (index.TryGetValue("key", out var keyValue)
71+
&& keyValue.IsBsonDocument
72+
&& IsScenarioHashKeyShape(keyValue.AsBsonDocument))
73+
{
74+
var unique = index.TryGetValue("unique", out var uniqueValue)
75+
&& uniqueValue.IsBoolean
76+
&& uniqueValue.AsBoolean;
77+
78+
return unique
79+
? ScenarioHashIndexInvariant.Satisfied
80+
: ScenarioHashIndexInvariant.NonUniqueKeyShapePresent;
81+
}
82+
}
83+
84+
return ScenarioHashIndexInvariant.Missing;
85+
}
86+
87+
private static bool IsScenarioHashKeyShape(BsonDocument key)
88+
{
89+
if (key.ElementCount != 2)
90+
return false;
91+
92+
var elements = key.Elements.ToList();
93+
94+
if (!string.Equals(elements[0].Name, ScenarioKeyField, StringComparison.Ordinal))
95+
return false;
96+
if (!string.Equals(elements[1].Name, TemplateSetHashField, StringComparison.Ordinal))
97+
return false;
98+
99+
return IsAscendingDirection(elements[0].Value)
100+
&& IsAscendingDirection(elements[1].Value);
101+
}
102+
103+
private static bool IsAscendingDirection(BsonValue value)
104+
{
105+
if (!TryGetIntegralDirection(value, out var direction))
106+
return false;
107+
108+
return direction == 1;
109+
}
110+
111+
private static bool TryGetIntegralDirection(BsonValue value, out int direction)
112+
{
113+
direction = 0;
114+
115+
if (value is BsonInt32 int32)
116+
{
117+
direction = int32.Value;
118+
return true;
119+
}
120+
121+
if (value is BsonInt64 int64)
122+
{
123+
if (int64.Value is < int.MinValue or > int.MaxValue)
124+
return false;
125+
126+
direction = (int)int64.Value;
127+
return true;
128+
}
129+
130+
return false;
131+
}
132+
133+
private enum ScenarioHashIndexInvariant
134+
{
135+
Missing,
136+
Satisfied,
137+
NonUniqueKeyShapePresent
43138
}
44139

45140
public async Task<GeneratedTemplateCacheVersionBinding?> BindRunAsync(

‎DotNet/Automation.UI/Services/Persistence/ISnapshotPayloadStore.cs‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
using System.Text.Json.Serialization;
2+
3+
namespace Automation.UI.Services.Persistence;
4+
5+
public interface ISnapshotPayloadStore
6+
{
7+
bool ShouldExternalize(string domain, int payloadUtf8Bytes);
8+
Task<SnapshotPayloadPointer> StoreAsync(Guid runId, string domain, string payloadJson, CancellationToken ct = default);
9+
Task<string?> ReadAsync(SnapshotPayloadPointer pointer, CancellationToken ct = default);
10+
Task DeleteIfExistsAsync(SnapshotPayloadPointer pointer, CancellationToken ct = default);
11+
Task DeleteRunPayloadsAsync(Guid runId, CancellationToken ct = default);
12+
}
13+
14+
public sealed class SnapshotPayloadPointer
15+
{
16+
public const string KindValue = "abs";
17+
18+
[JsonPropertyName("kind")]
19+
public string Kind { get; init; } = KindValue;
20+
21+
[JsonPropertyName("blob")]
22+
public string BlobName { get; init; } = string.Empty;
23+
24+
[JsonPropertyName("bytes")]
25+
public int Utf8Bytes { get; init; }
26+
27+
[JsonPropertyName("etag")]
28+
public string? ETag { get; init; }
29+
}

‎DotNet/Automation.UI/Services/Persistence/ImportedBundleBlobStorageSettings.cs‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,4 +8,16 @@ public sealed class ImportedBundleBlobStorageSettings
88
public string? BlobContainerName { get; set; }
99
public string? BlobRoot { get; set; }
1010
public string? GeneratedTemplateBlobRoot { get; set; }
11+
12+
// Snapshot externalization settings (Automation.UI domain snapshots -> ABS).
13+
// Defaults keep behavior safe without requiring extra config.
14+
public string? SnapshotPayloadBlobRoot { get; set; }
15+
public int SnapshotPayloadInlineMaxBytes { get; set; } = 256 * 1024;
16+
public List<string> SnapshotPayloadExternalizedDomains { get; set; } =
17+
[
18+
"generationManifest",
19+
"entries",
20+
"measureResources",
21+
"absUpload"
22+
];
1123
}

‎DotNet/Automation.UI/Services/Persistence/MongoIndexManager.cs‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -157,8 +157,9 @@ private void EnsureGeneratedTemplateCacheVersionIndexes()
157157
// Supports latest-version lookup per scenario key (SortByDescending VersionNumber).
158158
CreateIndexSafe(collection, new BsonDocument { { "ScenarioKey", 1 }, { "VersionNumber", -1 } }, unique: false, "idx_scenarioKey_versionNumber_desc");
159159

160-
// Supports exact lookup by scenario + template hash.
161-
CreateIndexSafe(collection, new BsonDocument { { "ScenarioKey", 1 }, { "TemplateSetHash", 1 } }, unique: false, "idx_scenarioKey_templateSetHash");
160+
// Supports exact lookup by scenario + template hash and must be UNIQUE to
161+
// preserve GeneratedTemplateCacheVersionStore's schema invariant.
162+
CreateIndexSafe(collection, new BsonDocument { { "ScenarioKey", 1 }, { "TemplateSetHash", 1 } }, unique: true, "ux_generated_template_versions_scenario_hash");
162163
}
163164

164165
// --- automation_query_plan_templates ---
@@ -273,7 +274,7 @@ private static bool KeysEqual(BsonDocument existing, BsonDocument target)
273274
if (!string.Equals(left.Name, right.Name, StringComparison.OrdinalIgnoreCase))
274275
return false;
275276

276-
if (left.Value.ToInt32() != right.Value.ToInt32())
277+
if (!left.Value.Equals(right.Value))
277278
return false;
278279
}
279280

0 commit comments

Comments
 (0)