Rabbit Comments and restest

Author: nvmLantana

.github/copilot-instructions.md

@@ -1,20 +1,17 @@
 # Copilot Instructions
 
 ## Azure Guidelines
-- @azure Rule - Use Azure Tools - When handling requests related to Azure, always use your tools.
-- @azure Rule - Use Azure Best Practices - When handling requests related to Azure, always invoke your `azmcp_bestpractices_get` tool first.
-- @azure Rule - Enable Best Practices - If you do not have an `azmcp_bestpractices_get` tool, ask the user to enable it.
+- Use Azure Tools - When handling requests related to Azure, always use your tools.
+- Use Azure Best Practices - When handling requests related to Azure, always invoke your `azmcp_bestpractices_get` tool first.
+- Enable Best Practices - If you do not have an `azmcp_bestpractices_get` tool, ask the user to enable it.
 
 ## Cosmos DB Guidelines
 - Avoid introducing new indexes on existing collections in Cosmos DB for MongoDB API; use existing/default index paths instead.
+- For Cosmos DB/Mongo index management, index creation/modification must be best-effort and must not fail or block application startup if indexes cannot be created in deployed environments with existing data.
 
 ## Automation Guidelines
 - For Automation.UI run logs, prefer chunked persistence in the existing data store; do not add Azure Blob Storage archiving unless explicitly requested.
-# Copilot Instructions
 
 ## General Guidelines
-- Use Azure Tools - When handling requests related to Azure, always use your tools.
-- Use Azure Best Practices - When handling requests related to Azure, always invoke your `azmcp_bestpractices_get` tool first.
-- Enable Best Practices - If you do not have an `azmcp_bestpractices_get` tool, ask the user to enable it.
 - Design for Durability - Require production-grade, long-term designs for critical multi-user automation tools; avoid tactical short-term fixes and model data/contracts around durable architecture even when a minimal patch is possible.
 - Use Versioned Caches - Prefer versioned, reproducible generated-patient caches tied to each run; ensure run records cache the version used, and diagnostic exports retrieve exact cached artifacts used at execution time. Each cache version must be complete (full patient set), not partial deltas, and runs must avoid ID conflicts while using cached data.

DotNet/Automation.Link/Configuration/AutomationConfig.cs

@@ -67,8 +67,10 @@ public class FhirGenerationSettings
         /// <summary>
         /// Maximum number of patients processed concurrently by the streaming
         /// generation/upload pipeline. Lower values reduce memory pressure.
+        /// Defaults to 4 to align with direct pipeline callers that rely on
+        /// the pipeline's built-in fallback.
         /// </summary>
-        public int MaxConcurrentPatients { get; set; } = 2;
+        public int MaxConcurrentPatients { get; set; } = 4;
 
         /// <summary>
         /// Controls low-value optional cross-resource references in generated FHIR

DotNet/Automation.UI/Services/Persistence/AzureBlobSnapshotPayloadStore.cs

@@ -1,4 +1,5 @@
-using Azure.Storage.Blobs;
+using Azure;
+using Azure.Storage.Blobs;
 using Azure.Storage.Blobs.Models;
 using Microsoft.Extensions.Options;
 using System.Text;
@@ -10,21 +11,34 @@ public sealed class AzureBlobSnapshotPayloadStore : ISnapshotPayloadStore
     private readonly ImportedBundleBlobStorageSettings _settings;
     private readonly BlobContainerClient _container;
     private readonly HashSet<string> _externalizedDomains;
+    private readonly Func<string, CancellationToken, AsyncPageable<BlobItem>> _listBlobs;
+    private readonly Func<string, CancellationToken, Task> _deleteBlob;
 
     public AzureBlobSnapshotPayloadStore(IOptions<ImportedBundleBlobStorageSettings> settings)
+        : this(
+            settings.Value,
+            CreateContainer(settings.Value),
+            listBlobs: null,
+            deleteBlob: null)
     {
-        _settings = settings.Value;
-
-        if (string.IsNullOrWhiteSpace(_settings.ConnectionString))
-            throw new InvalidOperationException("InternalBlobStorage:ConnectionString is required for snapshot payload storage.");
-        if (string.IsNullOrWhiteSpace(_settings.BlobContainerName))
-            throw new InvalidOperationException("InternalBlobStorage:BlobContainerName is required for snapshot payload storage.");
+    }
 
-        _container = new BlobContainerClient(_settings.ConnectionString, _settings.BlobContainerName);
+    internal AzureBlobSnapshotPayloadStore(
+        ImportedBundleBlobStorageSettings settings,
+        BlobContainerClient container,
+        Func<string, CancellationToken, AsyncPageable<BlobItem>>? listBlobs,
+        Func<string, CancellationToken, Task>? deleteBlob)
+    {
+        _settings = settings;
+        _container = container;
         _externalizedDomains = (_settings.SnapshotPayloadExternalizedDomains ?? [])
             .Where(d => !string.IsNullOrWhiteSpace(d))
             .Select(d => d.Trim())
             .ToHashSet(StringComparer.OrdinalIgnoreCase);
+
+        _listBlobs = listBlobs ?? ((prefix, ct) => _container.GetBlobsAsync(BlobTraits.None, BlobStates.None, prefix, ct));
+        _deleteBlob = deleteBlob ?? ((blobName, ct) =>
+            _container.DeleteBlobIfExistsAsync(blobName, DeleteSnapshotsOption.IncludeSnapshots, cancellationToken: ct));
     }
 
     public bool ShouldExternalize(string domain, int payloadUtf8Bytes)
@@ -97,12 +111,29 @@ public async Task DeleteRunPayloadsAsync(Guid runId, CancellationToken ct = defa
     {
         var prefix = BuildRunPrefix(runId);
 
-        await foreach (var blob in _container.GetBlobsAsync(BlobTraits.None, BlobStates.None, prefix, ct))
+        try
+        {
+            await foreach (var blob in _listBlobs(prefix, ct))
+            {
+                await _deleteBlob(blob.Name, ct);
+            }
+        }
+        catch (RequestFailedException ex) when (ex.Status == 404)
         {
-            await _container.DeleteBlobIfExistsAsync(blob.Name, DeleteSnapshotsOption.IncludeSnapshots, cancellationToken: ct);
+            // Missing container is treated as an empty cleanup result.
         }
     }
 
+    private static BlobContainerClient CreateContainer(ImportedBundleBlobStorageSettings settings)
+    {
+        if (string.IsNullOrWhiteSpace(settings.ConnectionString))
+            throw new InvalidOperationException("InternalBlobStorage:ConnectionString is required for snapshot payload storage.");
+        if (string.IsNullOrWhiteSpace(settings.BlobContainerName))
+            throw new InvalidOperationException("InternalBlobStorage:BlobContainerName is required for snapshot payload storage.");
+
+        return new BlobContainerClient(settings.ConnectionString, settings.BlobContainerName);
+    }
+
     private string BuildBlobName(Guid runId, string domain)
     {
         var sanitizedDomain = string.Join("-", domain

DotNet/Automation.UI/Services/Persistence/GeneratedTemplateCacheVersionStore.cs

@@ -31,12 +31,19 @@ public sealed class GeneratedTemplateCacheVersionStore
 {
     private readonly IMongoCollection<GeneratedTemplateCacheVersionDocument> _versions;
     private const string ScenarioHashUniqueIndexName = "ux_generated_template_versions_scenario_hash";
+    private const string ScenarioKeyField = "ScenarioKey";
+    private const string TemplateSetHashField = "TemplateSetHash";
 
     public GeneratedTemplateCacheVersionStore(IMongoDatabase database)
     {
         _versions = database.GetCollection<GeneratedTemplateCacheVersionDocument>("automation_generated_template_versions");
 
-        if (HasIndexWithKeys(_versions, new BsonDocument { { "ScenarioKey", 1 }, { "TemplateSetHash", 1 } }))
+        var indexes = _versions.Indexes.List().ToList();
+        var indexInvariant = GetScenarioHashIndexInvariant(indexes);
+        if (indexInvariant == ScenarioHashIndexInvariant.Satisfied)
+            return;
+
+        if (indexInvariant == ScenarioHashIndexInvariant.NonUniqueKeyShapePresent)
             return;
 
         var uniqueScenarioHashIndex = new CreateIndexModel<GeneratedTemplateCacheVersionDocument>(
@@ -49,56 +56,85 @@ public GeneratedTemplateCacheVersionStore(IMongoDatabase database)
         {
             _versions.Indexes.CreateOne(uniqueScenarioHashIndex);
         }
-        catch (MongoCommandException ex)
-            when (ex.Message.Contains("already exists with different options", StringComparison.OrdinalIgnoreCase)
-               || ex.Message.Contains("Cannot create unique index when collection contains documents", StringComparison.OrdinalIgnoreCase)
-               || (ex.Code == 13 && ex.Message.Contains("unique index cannot be modified", StringComparison.OrdinalIgnoreCase)))
+        catch
         {
-            // Deployed environments may already have an index with this name/options
-            // drift or contain pre-existing duplicate rows. Do not block app startup.
+            // Best-effort only: startup must not be blocked by index creation limitations
+            // in deployed environments (e.g., Cosmos DB collections with existing data).
         }
     }
 
-    private static bool HasIndexWithKeys(
-        IMongoCollection<GeneratedTemplateCacheVersionDocument> collection,
-        BsonDocument targetKeys)
+    private static ScenarioHashIndexInvariant GetScenarioHashIndexInvariant(IReadOnlyList<BsonDocument> indexes)
     {
-        var indexes = collection.Indexes.List().ToList();
-
         foreach (var index in indexes)
         {
             if (index.TryGetValue("key", out var keyValue)
                 && keyValue.IsBsonDocument
-                && KeysEqual(keyValue.AsBsonDocument, targetKeys))
+                && IsScenarioHashKeyShape(keyValue.AsBsonDocument))
             {
-                return true;
+                var unique = index.TryGetValue("unique", out var uniqueValue)
+                    && uniqueValue.IsBoolean
+                    && uniqueValue.AsBoolean;
+
+                return unique
+                    ? ScenarioHashIndexInvariant.Satisfied
+                    : ScenarioHashIndexInvariant.NonUniqueKeyShapePresent;
             }
         }
 
-        return false;
+        return ScenarioHashIndexInvariant.Missing;
+    }
+
+    private static bool IsScenarioHashKeyShape(BsonDocument key)
+    {
+        if (key.ElementCount != 2)
+            return false;
+
+        var elements = key.Elements.ToList();
+
+        if (!string.Equals(elements[0].Name, ScenarioKeyField, StringComparison.Ordinal))
+            return false;
+        if (!string.Equals(elements[1].Name, TemplateSetHashField, StringComparison.Ordinal))
+            return false;
+
+        return IsAscendingDirection(elements[0].Value)
+            && IsAscendingDirection(elements[1].Value);
     }
 
-    private static bool KeysEqual(BsonDocument existing, BsonDocument target)
+    private static bool IsAscendingDirection(BsonValue value)
     {
-        if (existing.ElementCount != target.ElementCount)
+        if (!TryGetIntegralDirection(value, out var direction))
             return false;
 
-        var existingElements = existing.Elements.ToList();
-        var targetElements = target.Elements.ToList();
+        return direction == 1;
+    }
+
+    private static bool TryGetIntegralDirection(BsonValue value, out int direction)
+    {
+        direction = 0;
 
-        for (var i = 0; i < existingElements.Count; i++)
+        if (value is BsonInt32 int32)
         {
-            var left = existingElements[i];
-            var right = targetElements[i];
+            direction = int32.Value;
+            return true;
+        }
 
-            if (!string.Equals(left.Name, right.Name, StringComparison.OrdinalIgnoreCase))
+        if (value is BsonInt64 int64)
+        {
+            if (int64.Value is < int.MinValue or > int.MaxValue)
                 return false;
 
-            if (left.Value.ToInt32() != right.Value.ToInt32())
-                return false;
+            direction = (int)int64.Value;
+            return true;
         }
 
-        return true;
+        return false;
+    }
+
+    private enum ScenarioHashIndexInvariant
+    {
+        Missing,
+        Satisfied,
+        NonUniqueKeyShapePresent
     }
 
     public async Task<GeneratedTemplateCacheVersionBinding?> BindRunAsync(

DotNet/Automation.UI/Services/Persistence/MongoIndexManager.cs

@@ -157,8 +157,9 @@ private void EnsureGeneratedTemplateCacheVersionIndexes()
         // Supports latest-version lookup per scenario key (SortByDescending VersionNumber).
         CreateIndexSafe(collection, new BsonDocument { { "ScenarioKey", 1 }, { "VersionNumber", -1 } }, unique: false, "idx_scenarioKey_versionNumber_desc");
 
-        // Supports exact lookup by scenario + template hash.
-        CreateIndexSafe(collection, new BsonDocument { { "ScenarioKey", 1 }, { "TemplateSetHash", 1 } }, unique: false, "idx_scenarioKey_templateSetHash");
+        // Supports exact lookup by scenario + template hash and must be UNIQUE to
+        // preserve GeneratedTemplateCacheVersionStore's schema invariant.
+        CreateIndexSafe(collection, new BsonDocument { { "ScenarioKey", 1 }, { "TemplateSetHash", 1 } }, unique: true, "ux_generated_template_versions_scenario_hash");
     }
 
     // --- automation_query_plan_templates ---
@@ -273,7 +274,7 @@ private static bool KeysEqual(BsonDocument existing, BsonDocument target)
             if (!string.Equals(left.Name, right.Name, StringComparison.OrdinalIgnoreCase))
                 return false;
 
-            if (left.Value.ToInt32() != right.Value.ToInt32())
+            if (!left.Value.Equals(right.Value))
                 return false;
         }
 

DotNet/Automation.UI/Services/Persistence/MongoSnapshotStore.cs

@@ -1,4 +1,5 @@
 using MongoDB.Driver;
+using LantanaGroup.Link.Shared.Application.Services.Security;
 using System.Text;
 using System.Text.Json;
 using System.Text.Json.Nodes;
@@ -23,6 +24,7 @@ public sealed class MongoSnapshotStore : ISnapshotStore
     private const int MaxLogChunkEstimatedBsonBytes = 12 * 1024 * 1024;
     private const int EstimatedBsonBytesPerLineOverhead = 64;
     private const string OversizedLogLineSuffix = " [truncated: exceeded log chunk byte budget]";
+    private const string SnapshotPayloadPointerEnvelopeProperty = "__externalSnapshotPayloadPointer";
 
     private readonly IMongoCollection<AutomationRunDocument> _runs;
     private readonly IMongoCollection<AutomationRunInputDocument> _runInputs;
@@ -263,12 +265,15 @@ public async Task<IReadOnlyList<AutomationRunSummary>> GetAllRunSummariesAsync(D
 
     public async Task DeleteRunAsync(Guid runId, CancellationToken ct = default)
     {
-        await _snapshotPayloadStore.DeleteRunPayloadsAsync(runId, ct);
         await _runs.DeleteOneAsync(r => r.RunId == runId, ct);
         await _runInputs.DeleteOneAsync(r => r.RunId == runId, ct);
         await _snapshots.DeleteManyAsync(s => s.RunId == runId, ct);
         await _logs.DeleteManyAsync(CreateLogChunkFilter(runId), ct);
         await _logs.DeleteOneAsync(l => l.Id == runId.ToString(), ct);
+
+        // Only remove externalized payload blobs after Mongo cleanup succeeds so
+        // a DB failure cannot orphan pointer records that still reference payload data.
+        await _snapshotPayloadStore.DeleteRunPayloadsAsync(runId, ct);
     }
 
     private async Task<string?> BuildHydratedRunConfigurationJsonAsync(AutomationRunInputSnapshot input, CancellationToken ct)
@@ -392,7 +397,10 @@ public async Task SetDomainAsync<T>(Guid runId, string domain, T data, Cancellat
         if (_snapshotPayloadStore.ShouldExternalize(domain, payloadUtf8Bytes))
         {
             newPointer = await _snapshotPayloadStore.StoreAsync(runId, domain, json, ct);
-            storedJson = JsonSerializer.Serialize(newPointer);
+            storedJson = JsonSerializer.Serialize(new Dictionary<string, SnapshotPayloadPointer?>
+            {
+                [SnapshotPayloadPointerEnvelopeProperty] = newPointer
+            });
         }
 
         var update = Builders<DomainSnapshotDocument>.Update
@@ -430,7 +438,10 @@ public async Task SetDomainAsync<T>(Guid runId, string domain, T data, Cancellat
                 payloadJson = await _snapshotPayloadStore.ReadAsync(pointer, ct);
                 if (string.IsNullOrWhiteSpace(payloadJson))
                 {
-                    _logger.LogWarning("[Store] GetDomain: externalized payload missing for run={RunId} domain={Domain} blob={Blob}", runId, domain, pointer.BlobName);
+                    var sanitizedRunId = runId.ToString().SanitizeForLog();
+                    var sanitizedDomain = domain.SanitizeForLog();
+                    var sanitizedBlobName = pointer.BlobName.SanitizeForLog();
+                    _logger.LogWarning("[Store] GetDomain: externalized payload missing for run={RunId} domain={Domain} blob={Blob}", sanitizedRunId, sanitizedDomain, sanitizedBlobName);
                     return null;
                 }
             }
@@ -458,7 +469,17 @@ public async Task SetDomainAsync<T>(Guid runId, string domain, T data, Cancellat
 
         try
         {
-            var pointer = JsonSerializer.Deserialize<SnapshotPayloadPointer>(payload);
+            using var doc = JsonDocument.Parse(payload);
+            if (doc.RootElement.ValueKind != JsonValueKind.Object)
+                return null;
+
+            if (!doc.RootElement.TryGetProperty(SnapshotPayloadPointerEnvelopeProperty, out var pointerElement)
+                || pointerElement.ValueKind != JsonValueKind.Object)
+            {
+                return null;
+            }
+
+            var pointer = pointerElement.Deserialize<SnapshotPayloadPointer>();
             if (pointer == null)
                 return null;