LEGLINK-788: Improve Generated Patient Storage and Uploaded Patient Processing.

.github/copilot-instructions.md

@@ -1,3 +1,8 @@
-- @azure Rule - Use Azure Tools - When handling requests related to Azure, always use your tools.
-- @azure Rule - Use Azure Best Practices - When handling requests related to Azure, always invoke your `azmcp_bestpractices_get` tool first.
-- @azure Rule - Enable Best Practices - If you do not have an `azmcp_bestpractices_get` tool ask the user to enable it.
+# Copilot Instructions
+
+## General Guidelines
+- Use Azure Tools - When handling requests related to Azure, always use your tools.
+- Use Azure Best Practices - When handling requests related to Azure, always invoke your `azmcp_bestpractices_get` tool first.
+- Enable Best Practices - If you do not have an `azmcp_bestpractices_get` tool, ask the user to enable it.
+- Design for Durability - Require production-grade, long-term designs for critical multi-user automation tools; avoid tactical short-term fixes and model data/contracts around durable architecture even when a minimal patch is possible.
+- Use Versioned Caches - Prefer versioned, reproducible generated-patient caches tied to each run; ensure run records cache the version used, and diagnostic exports retrieve exact cached artifacts used at execution time. Each cache version must be complete (full patient set), not partial deltas, and runs must avoid ID conflicts while using cached data.

DotNet/Automation.Link/Models/AutomationRunSummary.cs

@@ -19,5 +19,9 @@ public class AutomationRunSummary
     public string? FacilityId { get; set; }
     public string? ReportId { get; set; }
     public string? RunConfigurationJson { get; set; }
+    public Guid? GeneratedTemplateCacheVersionId { get; set; }
+    public int? GeneratedTemplateCacheVersionNumber { get; set; }
+    public string? GeneratedTemplateCacheScenarioKey { get; set; }
+    public string? GeneratedTemplateSetHash { get; set; }
     public IReadOnlyList<string> Logs { get; set; } = [];
 }

DotNet/Automation.UI/Automation.UI.csproj

@@ -7,21 +7,6 @@
 		<RestorePackagesWithLockFile>true</RestorePackagesWithLockFile>
   </PropertyGroup>
 
-  <ItemGroup>
-    <Compile Remove="Services\Caching\**" />
-    <Compile Remove="Services\Notifications\**" />
-    <Compile Remove="Views\Account\**" />
-    <Content Remove="Services\Caching\**" />
-    <Content Remove="Services\Notifications\**" />
-    <Content Remove="Views\Account\**" />
-    <EmbeddedResource Remove="Services\Caching\**" />
-    <EmbeddedResource Remove="Services\Notifications\**" />
-    <EmbeddedResource Remove="Views\Account\**" />
-    <None Remove="Services\Caching\**" />
-    <None Remove="Services\Notifications\**" />
-    <None Remove="Views\Account\**" />
-  </ItemGroup>
-
   <ItemGroup>
     <PackageReference Include="Azure.Storage.Blobs" />
     <PackageReference Include="Microsoft.AspNetCore.Authentication.OpenIdConnect" />
@@ -32,10 +17,4 @@
     <ProjectReference Include="..\LinkSdk\LinkSdk.csproj" />
     <ProjectReference Include="..\Shared\Shared.csproj" />
   </ItemGroup>
-
-  <ItemGroup>
-    <Folder Include="Views\Home\" />
-    <Folder Include="wwwroot\js\" />
-  </ItemGroup>
-
 </Project>

DotNet/Automation.UI/Controllers/NormalizationsController.cs

@@ -1,5 +1,4 @@
 using Automation.UI.Models;
-using Automation.UI.Models;
 using Automation.UI.Services.Persistence;
 using Microsoft.AspNetCore.Mvc;
 
@@ -252,8 +251,4 @@ public async Task<IActionResult> SetDefaultSuite([FromBody] IdRequest request, C
         return Ok();
     }
 
-    public sealed class IdRequest
-    {
-        public Guid Id { get; set; }
-    }
 }

DotNet/Automation.UI/Controllers/OrganizationResourceMapsController.cs

@@ -95,8 +95,4 @@ public async Task<IActionResult> SetDefaultInline([FromBody] IdRequest request,
         return Ok();
     }
 
-    public sealed class IdRequest
-    {
-        public Guid Id { get; set; }
-    }
 }

DotNet/Automation.UI/Controllers/QueryPlansController.cs

@@ -112,11 +112,6 @@ public async Task<IActionResult> SetDefaultInline([FromBody] IdRequest request,
         return Ok();
     }
 
-    public sealed class IdRequest
-    {
-        public Guid Id { get; set; }
-    }
-
     private static QueryEntry ToQueryEntry(QueryPlanQueryEntry src) => new()
     {
         ResourceType = src.ResourceType,

DotNet/Automation.UI/Controllers/RunsController.cs

@@ -26,10 +26,7 @@ public async Task<IActionResult> Index(
         string sortDir = "desc",
         CancellationToken cancellationToken = default)
     {
-        // Normalize: accept "asc"/"desc" only, default to descending. Server-side
-        // store-level whitelisting also clamps unknown sortBy values, so this is
-        // belt-and-suspenders against URL tampering.
-        var descending = !string.Equals(sortDir, "asc", StringComparison.OrdinalIgnoreCase);
+        var descending = IsDescending(sortDir);
 
         var stats = await runManager.GetDashboardStatsAsync(cancellationToken);
         var recentPage = await runManager.GetRunsPageAsync(pageNumber, pageSize, sortBy, descending, cancellationToken);
@@ -38,23 +35,7 @@ public async Task<IActionResult> Index(
             .ThenBy(s => s.Name, StringComparer.OrdinalIgnoreCase)
             .ToList();
 
-        var activeRunMetas = await snapshotStore.GetActiveRunsAsync(cancellationToken);
-        var activeRunSummaries = await Task.WhenAll(activeRunMetas.Select(meta => runManager.GetRunAsync(meta.RunId, cancellationToken)));
-
-        var activeRunsSource = recentPage.PageNumber == 1
-            ? recentPage.Runs
-            : (await runManager.GetRunsPageAsync(1, pageSize, "createdAt", true, cancellationToken)).Runs;
-        var statusActiveRuns = activeRunsSource
-            .Where(r => r.Status is AutomationRunStatus.Queued or AutomationRunStatus.Running);
-
-        var activeRuns = activeRunSummaries
-            .Where(r => r != null && (r.Status is AutomationRunStatus.Queued or AutomationRunStatus.Running))
-            .Select(r => r!)
-            .Concat(statusActiveRuns)
-            .GroupBy(r => r.RunId)
-            .Select(g => g.First())
-            .OrderByDescending(r => r.CreatedAt)
-            .ToList();
+        var activeRuns = await GetActiveRunsAsync(recentPage, pageSize, cancellationToken);
 
         // Populate query plan templates for the shared scenario editor modal embedded in this view.
         ViewBag.QueryPlanTemplates = await queryPlanTemplateStore.GetAllAsync(cancellationToken);
@@ -93,7 +74,7 @@ public async Task<IActionResult> RecentRunsPartial(
         // page navigation. The view model matches the partial's @model so the
         // partial is reused by both this action and the initial server render
         // in Index.cshtml — there's no divergence between the two templates.
-        var descending = !string.Equals(sortDir, "asc", StringComparison.OrdinalIgnoreCase);
+        var descending = IsDescending(sortDir);
         var page = await runManager.GetRunsPageAsync(pageNumber, pageSize, sortBy, descending, cancellationToken);
         return PartialView("_RecentRunsTable", page);
     }
@@ -106,28 +87,12 @@ public async Task<IActionResult> DashboardStats(
         string sortDir = "desc",
         CancellationToken cancellationToken = default)
     {
-        var descending = !string.Equals(sortDir, "asc", StringComparison.OrdinalIgnoreCase);
+        var descending = IsDescending(sortDir);
 
         var stats = await runManager.GetDashboardStatsAsync(cancellationToken);
         var recentPage = await runManager.GetRunsPageAsync(pageNumber, pageSize, sortBy, descending, cancellationToken);
 
-        var activeRunMetas = await snapshotStore.GetActiveRunsAsync(cancellationToken);
-        var activeRunSummaries = await Task.WhenAll(activeRunMetas.Select(meta => runManager.GetRunAsync(meta.RunId, cancellationToken)));
-
-        var activeRunsSource = recentPage.PageNumber == 1
-            ? recentPage.Runs
-            : (await runManager.GetRunsPageAsync(1, pageSize, "createdAt", true, cancellationToken)).Runs;
-        var statusActiveRuns = activeRunsSource
-            .Where(r => r.Status is AutomationRunStatus.Queued or AutomationRunStatus.Running);
-
-        var activeRuns = activeRunSummaries
-            .Where(r => r != null && (r.Status is AutomationRunStatus.Queued or AutomationRunStatus.Running))
-            .Select(r => r!)
-            .Concat(statusActiveRuns)
-            .GroupBy(r => r.RunId)
-            .Select(g => g.First())
-            .OrderByDescending(r => r.CreatedAt)
-            .ToList();
+        var activeRuns = await GetActiveRunsAsync(recentPage, pageSize, cancellationToken);
 
         return Json(new
         {
@@ -527,4 +492,28 @@ public async Task<IActionResult> DataAcquisitionLogDetail(
             return NotFound();
         }
     }
+
+    private static bool IsDescending(string sortDir)
+        => !string.Equals(sortDir, "asc", StringComparison.OrdinalIgnoreCase);
+
+    private async Task<List<AutomationRunSummary>> GetActiveRunsAsync(
+        AutomationRunIndexViewModel recentPage,
+        int pageSize,
+        CancellationToken cancellationToken)
+    {
+        var activeRunMetas = await snapshotStore.GetActiveRunsAsync(cancellationToken);
+        var activeRunSummaries = await Task.WhenAll(activeRunMetas.Select(meta => runManager.GetRunAsync(meta.RunId, cancellationToken)));
+        var activeRunsSource = recentPage.PageNumber == 1
+            ? recentPage.Runs
+            : (await runManager.GetRunsPageAsync(1, pageSize, "createdAt", true, cancellationToken)).Runs;
+
+        return activeRunSummaries
+            .Where(r => r is { Status: AutomationRunStatus.Queued or AutomationRunStatus.Running })
+            .Select(r => r!)
+            .Concat(activeRunsSource.Where(r => r.Status is AutomationRunStatus.Queued or AutomationRunStatus.Running))
+            .GroupBy(r => r.RunId)
+            .Select(g => g.First())
+            .OrderByDescending(r => r.CreatedAt)
+            .ToList();
+    }
 }

DotNet/Automation.UI/Controllers/ScenariosController.cs

@@ -22,6 +22,7 @@ public class ScenariosController(
     IOptions<AutomationConfig> automationConfig,
     IMongoDatabase database,
     IImportedBundleContentStore bundleContentStore,
+    PatientReplacementManager patientReplacementManager,
     ILogger<ScenariosController> logger) : Controller
 {
     private static readonly JsonSerializerOptions FhirJsonOptions = LantanaGroup.Link.Shared.Application.SerDes.LinkFhirSerializerOptions.ForFhirWithoutValidation();
@@ -132,6 +133,9 @@ public async Task<IActionResult> SaveInline([FromBody] TestScenarioDefinition mo
                     return $"Imported bundle '{p.FileName ?? p.PatientId}' content is missing. Please re-upload the file.";
             }
 
+            if (string.IsNullOrWhiteSpace(bundleJson))
+                return $"Imported bundle '{p.FileName ?? p.PatientId}' is missing its uploaded reference.";
+
             Bundle? bundle;
             try
             {
@@ -353,51 +357,31 @@ public async Task<IActionResult> ReplacePatientOnFhirServer([FromBody] ReplacePa
             .Cast<string>()
             .ToList();
 
-        var cfg = automationConfig.Value;
-        var loader = new FhirDataLoader(cfg.FhirServerBase, cfg.FhirServerOAuth, cfg.FhirServerBasicAuth);
         var patientIdForLog = request.PatientId.Replace("\r", string.Empty).Replace("\n", string.Empty);
+        var replayBundles = BuildReplayBundles(entries, request.PatientId.Trim());
+        var operationId = patientReplacementManager.Start(patientIdForLog, resourcesToDelete, replayBundles);
 
         logger.LogInformation(
-            "Replacing FHIR-server data for patient '{PatientId}' using uploaded bundle '{BundleId}'. Deleting {DeleteCount} resource path(s) first.",
+            "Queued FHIR-server replacement {OperationId} for patient '{PatientId}' using uploaded bundle '{BundleId}'. Deleting {DeleteCount} resource path(s) first.",
+            operationId,
             patientIdForLog,
             request.UploadedBundleId,
             resourcesToDelete.Count);
 
-        var purge = await loader.DeleteResourcesWithExpungeAsync(resourcesToDelete, ct);
-        if (purge.Failed > 0)
-        {
-            logger.LogWarning(
-                "FHIR purge before patient replace had failures for patient '{PatientId}': {Failed} failed, {Succeeded} succeeded. First errors: {Errors}",
-                patientIdForLog,
-                purge.Failed,
-                purge.Succeeded,
-                string.Join(" | ", purge.Failures.Take(5)));
-        }
-
-        var replayBundles = BuildReplayBundles(entries, request.PatientId.Trim());
-        var output = new RunAutomationOutput(message => logger.LogInformation("[FHIR Replay] {Message}", message));
-        var replayOk = await loader.UploadBundlesSequentiallyAsync(output, replayBundles, $"[replace:{patientIdForLog}] ");
-        if (!replayOk)
-        {
-            return StatusCode(StatusCodes.Status502BadGateway,
-                "Failed to replay uploaded bundle to FHIR server after purge. Uploaded bundle remains stored for scenario execution.");
-        }
-
-        var messageText = purge.Failed > 0
-            ? $"Replaced FHIR-server data for patient '{patientIdForLog}' with uploaded bundle, but purge had {purge.Failed} failed delete(s)."
-            : $"Successfully replaced FHIR-server data for patient '{patientIdForLog}' with uploaded bundle.";
-
         return Json(new
         {
-            success = true,
-            warningMessage = purge.Failed > 0 ? messageText : null,
-            message = messageText,
-            deletedSucceeded = purge.Succeeded,
-            deletedFailed = purge.Failed,
-            replayBundleCount = replayBundles.Count
+            operationId,
+            statusUrl = Url.Action(nameof(GetPatientReplacementStatus), new { operationId })
         });
     }
 
+    [HttpGet]
+    public IActionResult GetPatientReplacementStatus(Guid operationId)
+    {
+        var status = patientReplacementManager.GetStatus(operationId);
+        return status == null ? NotFound("Patient replacement operation not found.") : Json(status);
+    }
+
     [HttpPost]
     [ValidateAntiForgeryToken]
     public async Task<IActionResult> DiscardUploadedBundle([FromBody] DiscardUploadedBundleRequest request, CancellationToken ct)
@@ -639,11 +623,6 @@ public async Task<IActionResult> CloneInline([FromBody] IdRequest request, Cance
         return Json(new { id = clone.Id });
     }
 
-    public sealed class IdRequest
-    {
-        public Guid Id { get; set; }
-    }
-
     private static string ComputeContentHash(string json)
     {
         var bytes = Encoding.UTF8.GetBytes(json);

DotNet/Automation.UI/Models/IdRequest.cs

@@ -0,0 +1,6 @@
+namespace Automation.UI.Models;
+
+public sealed class IdRequest
+{
+    public Guid Id { get; set; }
+}

DotNet/Automation.UI/Program.cs

@@ -228,6 +228,9 @@ static IReadOnlyCollection<string> BuildValidAudiences(string configuredAudience
 
 builder.Services.AddSingleton<MongoIndexManager>();
 builder.Services.AddSingleton<IImportedBundleContentStore, AzureBlobImportedBundleContentStore>();
+builder.Services.AddSingleton<LantanaGroup.Automation.Generation.IGeneratedPatientTemplateCache, MongoGeneratedPatientTemplateCache>();
+builder.Services.AddSingleton<GeneratedTemplateCacheVersionStore>();
+builder.Services.AddSingleton<ImportedBundleExecutionResolver>();
 builder.Services.AddSingleton<ISnapshotStore, MongoSnapshotStore>();
 builder.Services.AddSingleton<IScenarioStore, MongoScenarioStore>();
 builder.Services.AddSingleton<IQueryPlanTemplateStore, MongoQueryPlanTemplateStore>();
@@ -258,7 +261,7 @@ static IReadOnlyCollection<string> BuildValidAudiences(string configuredAudience
 builder.Services.AddSingleton<Automation.UI.Services.ApiHealth.Seeding.IApiHealthSeedContextAccessor, Automation.UI.Services.ApiHealth.Seeding.ApiHealthSeedContextAccessor>();
 builder.Services.AddSingleton<Automation.UI.Services.ApiHealth.Seeding.IApiHealthSeedOrchestrator, Automation.UI.Services.ApiHealth.Seeding.ApiHealthSeedOrchestrator>();
 builder.Services.AddHostedService<ScenarioRunStartupRecoveryService>();
-builder.Services.AddHostedService<ImportedBundleBlobMigrationService>();
+builder.Services.AddHostedService<PatientBundleExternalizationMigrationService>();
 builder.Services.AddHostedService<Automation.UI.Services.ApiHealth.ApiHealthStartupRecoveryService>();
 builder.Services.AddHttpClient("ApiHealthTest");
 builder.Services.AddHealthChecks();
@@ -307,6 +310,7 @@ static IReadOnlyCollection<string> BuildValidAudiences(string configuredAudience
 builder.Services.AddSingleton<RunSnapshotOrchestrator>();
 builder.Services.AddHostedService(sp => sp.GetRequiredService<RunSnapshotOrchestrator>());
 builder.Services.AddSingleton<IAutomationRunManager, AutomationRunManager>();
+builder.Services.AddSingleton<PatientReplacementManager>();
 builder.Services.AddSingleton<IRunExportService, RunExportService>();
 
 var app = builder.Build();

DotNet/Automation.UI/Services/ApiHealth/ApiHealthExecutionRunManager.cs

@@ -435,7 +435,8 @@ private async Task CompleteAsync(RunState run, bool failed, string? error)
         run.Completed = true;
         run.Failed = failed;
         run.Error = error;
-        run.FinishedAt = DateTimeOffset.UtcNow;
+        var finishedAt = DateTimeOffset.UtcNow;
+        run.FinishedAt = finishedAt;
 
         lock (_sync)
         {
@@ -445,7 +446,7 @@ private async Task CompleteAsync(RunState run, bool failed, string? error)
 
         try
         {
-            await store.CompleteExecutionRunAsync(run.RunId, failed, error, run.FinishedAt.Value);
+            await store.CompleteExecutionRunAsync(run.RunId, failed, error, finishedAt);
         }
         catch (Exception ex)
         {

DotNet/Automation.UI/Services/ApiHealth/TestSuites/IServiceTestSuite.cs

@@ -1,5 +1,4 @@
 using Automation.UI.Models.ApiHealth;
-using Automation.UI.Models.ApiHealth;
 using Automation.UI.Services.ApiHealth.Seeding;
 
 namespace Automation.UI.Services.ApiHealth.TestSuites;

DotNet/Automation.UI/Services/AutomationRunManager.cs

@@ -35,7 +35,10 @@ public AutomationRunManager(
         ISnapshotStore snapshotStore,
         IQueryPlanTemplateStore queryPlanTemplateStore,
         INormalizationStore normalizationStore,
-        IOrganizationResourceMapTemplateStore organizationResourceMapTemplateStore)
+        IOrganizationResourceMapTemplateStore organizationResourceMapTemplateStore,
+        ImportedBundleExecutionResolver importedBundleResolver,
+        LantanaGroup.Automation.Generation.IGeneratedPatientTemplateCache generatedTemplateCache,
+        GeneratedTemplateCacheVersionStore generatedTemplateVersionStore)
     {
         _hub = hub;
         _automationConfig = automationConfig.Value;
@@ -55,6 +58,9 @@ public AutomationRunManager(
             _queryPlanResolver,
             _normalizationSuiteResolver,
             _organizationResourceMapResolver,
+            importedBundleResolver,
+            generatedTemplateCache,
+            generatedTemplateVersionStore,
             configuration,
             _logger);
     }
@@ -65,7 +71,7 @@ public async Task<Guid> StartAsync(StartScenarioRequest request, CancellationTok
         var options = StartScenarioRequestResolver.Resolve(request);
 
         var runNameOverride = string.IsNullOrWhiteSpace(request.ScenarioName) ? null : request.ScenarioName.Trim();
-        var state = new MutableRunState(runId, request.Scenario, options, runNameOverride, request.RunConfigurationJson);
+        var state = new MutableRunState(runId, request.ScenarioId, request.Scenario, options, runNameOverride, request.RunConfigurationJson);
         _runs[runId] = state;
 
         await PersistRunInputAsync(runId, request);
@@ -556,6 +562,10 @@ private static AutomationRunSummary ToSummary(MutableRunState state)
                 Error = state.Error,
                 FacilityId = state.FacilityId,
                 ReportId = state.ReportId,
+                GeneratedTemplateCacheVersionId = state.GeneratedTemplateCacheVersionId,
+                GeneratedTemplateCacheVersionNumber = state.GeneratedTemplateCacheVersionNumber,
+                GeneratedTemplateCacheScenarioKey = state.GeneratedTemplateCacheScenarioKey,
+                GeneratedTemplateSetHash = state.GeneratedTemplateSetHash,
                 Logs = state.Logs.ToList()
             };
         }