LEGLINK-788: Improve Generated Patient Storage and Uploaded Patient Processing. (#1798)

* Update how and where we store generated patient bundles. Update how we process uploaded patient files to avoid timeouts.

* Rabbit Comments

* checkin

Author: nvmLantana

.github/copilot-instructions.md

@@ -10,3 +10,11 @@
 
 ## Automation Guidelines
 - For Automation.UI run logs, prefer chunked persistence in the existing data store; do not add Azure Blob Storage archiving unless explicitly requested.
+# Copilot Instructions
+
+## General Guidelines
+- Use Azure Tools - When handling requests related to Azure, always use your tools.
+- Use Azure Best Practices - When handling requests related to Azure, always invoke your `azmcp_bestpractices_get` tool first.
+- Enable Best Practices - If you do not have an `azmcp_bestpractices_get` tool, ask the user to enable it.
+- Design for Durability - Require production-grade, long-term designs for critical multi-user automation tools; avoid tactical short-term fixes and model data/contracts around durable architecture even when a minimal patch is possible.
+- Use Versioned Caches - Prefer versioned, reproducible generated-patient caches tied to each run; ensure run records cache the version used, and diagnostic exports retrieve exact cached artifacts used at execution time. Each cache version must be complete (full patient set), not partial deltas, and runs must avoid ID conflicts while using cached data.

DotNet/Automation.Link/Models/AutomationRunSummary.cs

@@ -19,5 +19,9 @@ public class AutomationRunSummary
     public string? FacilityId { get; set; }
     public string? ReportId { get; set; }
     public string? RunConfigurationJson { get; set; }
+    public Guid? GeneratedTemplateCacheVersionId { get; set; }
+    public int? GeneratedTemplateCacheVersionNumber { get; set; }
+    public string? GeneratedTemplateCacheScenarioKey { get; set; }
+    public string? GeneratedTemplateSetHash { get; set; }
     public IReadOnlyList<string> Logs { get; set; } = [];
 }

DotNet/Automation.UI/Automation.UI.csproj

@@ -7,21 +7,6 @@
 		<RestorePackagesWithLockFile>true</RestorePackagesWithLockFile>
   </PropertyGroup>
 
-  <ItemGroup>
-    <Compile Remove="Services\Caching\**" />
-    <Compile Remove="Services\Notifications\**" />
-    <Compile Remove="Views\Account\**" />
-    <Content Remove="Services\Caching\**" />
-    <Content Remove="Services\Notifications\**" />
-    <Content Remove="Views\Account\**" />
-    <EmbeddedResource Remove="Services\Caching\**" />
-    <EmbeddedResource Remove="Services\Notifications\**" />
-    <EmbeddedResource Remove="Views\Account\**" />
-    <None Remove="Services\Caching\**" />
-    <None Remove="Services\Notifications\**" />
-    <None Remove="Views\Account\**" />
-  </ItemGroup>
-
   <ItemGroup>
     <PackageReference Include="Azure.Storage.Blobs" />
     <PackageReference Include="Microsoft.AspNetCore.Authentication.OpenIdConnect" />
@@ -32,10 +17,4 @@
     <ProjectReference Include="..\LinkSdk\LinkSdk.csproj" />
     <ProjectReference Include="..\Shared\Shared.csproj" />
   </ItemGroup>
-
-  <ItemGroup>
-    <Folder Include="Views\Home\" />
-    <Folder Include="wwwroot\js\" />
-  </ItemGroup>
-
 </Project>

DotNet/Automation.UI/Controllers/IdRequestValidationExtensions.cs

@@ -0,0 +1,32 @@
+using Automation.UI.Models;
+using Microsoft.AspNetCore.Mvc;
+
+namespace Automation.UI.Controllers;
+
+internal static class IdRequestValidationExtensions
+{
+    internal static bool TryValidateIdRequest(this Controller controller, IdRequest? request, out IActionResult badRequest)
+    {
+        if (request == null)
+        {
+            badRequest = controller.BadRequest("Request body is required.");
+            return false;
+        }
+
+        if (!controller.ModelState.IsValid)
+        {
+            badRequest = controller.BadRequest(controller.ModelState);
+            return false;
+        }
+
+        if (request.Id == Guid.Empty)
+        {
+            controller.ModelState.AddModelError(nameof(IdRequest.Id), "Id must be a non-empty GUID.");
+            badRequest = controller.BadRequest(controller.ModelState);
+            return false;
+        }
+
+        badRequest = default!;
+        return true;
+    }
+}

DotNet/Automation.UI/Controllers/NormalizationsController.cs

@@ -1,5 +1,4 @@
 using Automation.UI.Models;
-using Automation.UI.Models;
 using Automation.UI.Services.Persistence;
 using Microsoft.AspNetCore.Mvc;
 
@@ -56,6 +55,9 @@ public async Task<IActionResult> SaveOperation([FromBody] NormalizationOperation
     [ValidateAntiForgeryToken]
     public async Task<IActionResult> DeleteOperation([FromBody] IdRequest request, CancellationToken ct)
     {
+        if (!this.TryValidateIdRequest(request, out var badRequest))
+            return badRequest;
+
         var op = await store.GetOperationByIdAsync(request.Id, ct);
         if (op == null) return NotFound();
         if (op.IsSystem)
@@ -79,6 +81,9 @@ public async Task<IActionResult> DeleteOperation([FromBody] IdRequest request, C
     [ValidateAntiForgeryToken]
     public async Task<IActionResult> CloneOperation([FromBody] IdRequest request, CancellationToken ct)
     {
+        if (!this.TryValidateIdRequest(request, out var badRequest))
+            return badRequest;
+
         var source = await store.GetOperationByIdAsync(request.Id, ct);
         if (source == null) return NotFound();
 
@@ -138,6 +143,9 @@ public async Task<IActionResult> SaveSequence([FromBody] NormalizationSequenceDe
     [ValidateAntiForgeryToken]
     public async Task<IActionResult> DeleteSequence([FromBody] IdRequest request, CancellationToken ct)
     {
+        if (!this.TryValidateIdRequest(request, out var badRequest))
+            return badRequest;
+
         var seq = await store.GetSequenceByIdAsync(request.Id, ct);
         if (seq == null) return NotFound();
         if (seq.IsSystem)
@@ -156,6 +164,9 @@ public async Task<IActionResult> DeleteSequence([FromBody] IdRequest request, Ca
     [ValidateAntiForgeryToken]
     public async Task<IActionResult> CloneSequence([FromBody] IdRequest request, CancellationToken ct)
     {
+        if (!this.TryValidateIdRequest(request, out var badRequest))
+            return badRequest;
+
         var source = await store.GetSequenceByIdAsync(request.Id, ct);
         if (source == null) return NotFound();
 
@@ -207,6 +218,9 @@ public async Task<IActionResult> SaveSuite([FromBody] NormalizationSuiteDefiniti
     [ValidateAntiForgeryToken]
     public async Task<IActionResult> DeleteSuite([FromBody] IdRequest request, CancellationToken ct)
     {
+        if (!this.TryValidateIdRequest(request, out var badRequest))
+            return badRequest;
+
         var suite = await store.GetSuiteByIdAsync(request.Id, ct);
         if (suite == null) return NotFound();
         if (suite.IsSystem)
@@ -222,6 +236,9 @@ public async Task<IActionResult> DeleteSuite([FromBody] IdRequest request, Cance
     [ValidateAntiForgeryToken]
     public async Task<IActionResult> CloneSuite([FromBody] IdRequest request, CancellationToken ct)
     {
+        if (!this.TryValidateIdRequest(request, out var badRequest))
+            return badRequest;
+
         var source = await store.GetSuiteByIdAsync(request.Id, ct);
         if (source == null) return NotFound();
 
@@ -245,15 +262,14 @@ public async Task<IActionResult> CloneSuite([FromBody] IdRequest request, Cancel
     [ValidateAntiForgeryToken]
     public async Task<IActionResult> SetDefaultSuite([FromBody] IdRequest request, CancellationToken ct)
     {
+        if (!this.TryValidateIdRequest(request, out var badRequest))
+            return badRequest;
+
         var suite = await store.GetSuiteByIdAsync(request.Id, ct);
         if (suite == null) return NotFound();
 
         await store.SetDefaultSuiteAsync(request.Id, ct);
         return Ok();
     }
 
-    public sealed class IdRequest
-    {
-        public Guid Id { get; set; }
-    }
 }

DotNet/Automation.UI/Controllers/OrganizationResourceMapsController.cs

@@ -47,6 +47,9 @@ public async Task<IActionResult> SaveInline([FromBody] OrganizationResourceMapTe
     [ValidateAntiForgeryToken]
     public async Task<IActionResult> DeleteInline([FromBody] IdRequest request, CancellationToken ct)
     {
+        if (!this.TryValidateIdRequest(request, out var badRequest))
+            return badRequest;
+
         var template = await store.GetByIdAsync(request.Id, ct);
         if (template == null) return NotFound();
         if (template.IsSystem)
@@ -62,6 +65,9 @@ public async Task<IActionResult> DeleteInline([FromBody] IdRequest request, Canc
     [ValidateAntiForgeryToken]
     public async Task<IActionResult> CloneInline([FromBody] IdRequest request, CancellationToken ct)
     {
+        if (!this.TryValidateIdRequest(request, out var badRequest))
+            return badRequest;
+
         var source = await store.GetByIdAsync(request.Id, ct);
         if (source == null) return NotFound();
 
@@ -88,15 +94,14 @@ public async Task<IActionResult> CloneInline([FromBody] IdRequest request, Cance
     [ValidateAntiForgeryToken]
     public async Task<IActionResult> SetDefaultInline([FromBody] IdRequest request, CancellationToken ct)
     {
+        if (!this.TryValidateIdRequest(request, out var badRequest))
+            return badRequest;
+
         var template = await store.GetByIdAsync(request.Id, ct);
         if (template == null) return NotFound();
 
         await store.SetDefaultAsync(request.Id, ct);
         return Ok();
     }
 
-    public sealed class IdRequest
-    {
-        public Guid Id { get; set; }
-    }
 }

DotNet/Automation.UI/Controllers/QueryPlansController.cs

@@ -66,6 +66,9 @@ public async Task<IActionResult> SaveInline([FromBody] QueryPlanTemplate model,
     [ValidateAntiForgeryToken]
     public async Task<IActionResult> DeleteInline([FromBody] IdRequest request, CancellationToken ct)
     {
+        if (!this.TryValidateIdRequest(request, out var badRequest))
+            return badRequest;
+
         var template = await store.GetByIdAsync(request.Id, ct);
         if (template == null)
             return NotFound();
@@ -80,6 +83,9 @@ public async Task<IActionResult> DeleteInline([FromBody] IdRequest request, Canc
     [ValidateAntiForgeryToken]
     public async Task<IActionResult> CloneInline([FromBody] IdRequest request, CancellationToken ct)
     {
+        if (!this.TryValidateIdRequest(request, out var badRequest))
+            return badRequest;
+
         var source = await store.GetByIdAsync(request.Id, ct);
         if (source == null) return NotFound();
 
@@ -105,18 +111,16 @@ public async Task<IActionResult> CloneInline([FromBody] IdRequest request, Cance
     [ValidateAntiForgeryToken]
     public async Task<IActionResult> SetDefaultInline([FromBody] IdRequest request, CancellationToken ct)
     {
+        if (!this.TryValidateIdRequest(request, out var badRequest))
+            return badRequest;
+
         var template = await store.GetByIdAsync(request.Id, ct);
         if (template == null) return NotFound();
 
         await store.SetDefaultAsync(request.Id, ct);
         return Ok();
     }
 
-    public sealed class IdRequest
-    {
-        public Guid Id { get; set; }
-    }
-
     private static QueryEntry ToQueryEntry(QueryPlanQueryEntry src) => new()
     {
         ResourceType = src.ResourceType,

DotNet/Automation.UI/Controllers/RunsController.cs

@@ -26,10 +26,7 @@ public async Task<IActionResult> Index(
         string sortDir = "desc",
         CancellationToken cancellationToken = default)
     {
-        // Normalize: accept "asc"/"desc" only, default to descending. Server-side
-        // store-level whitelisting also clamps unknown sortBy values, so this is
-        // belt-and-suspenders against URL tampering.
-        var descending = !string.Equals(sortDir, "asc", StringComparison.OrdinalIgnoreCase);
+        var descending = IsDescending(sortDir);
 
         var stats = await runManager.GetDashboardStatsAsync(cancellationToken);
         var recentPage = await runManager.GetRunsPageAsync(pageNumber, pageSize, sortBy, descending, cancellationToken);
@@ -38,23 +35,7 @@ public async Task<IActionResult> Index(
             .ThenBy(s => s.Name, StringComparer.OrdinalIgnoreCase)
             .ToList();
 
-        var activeRunMetas = await snapshotStore.GetActiveRunsAsync(cancellationToken);
-        var activeRunSummaries = await Task.WhenAll(activeRunMetas.Select(meta => runManager.GetRunAsync(meta.RunId, cancellationToken)));
-
-        var activeRunsSource = recentPage.PageNumber == 1
-            ? recentPage.Runs
-            : (await runManager.GetRunsPageAsync(1, pageSize, "createdAt", true, cancellationToken)).Runs;
-        var statusActiveRuns = activeRunsSource
-            .Where(r => r.Status is AutomationRunStatus.Queued or AutomationRunStatus.Running);
-
-        var activeRuns = activeRunSummaries
-            .Where(r => r != null && (r.Status is AutomationRunStatus.Queued or AutomationRunStatus.Running))
-            .Select(r => r!)
-            .Concat(statusActiveRuns)
-            .GroupBy(r => r.RunId)
-            .Select(g => g.First())
-            .OrderByDescending(r => r.CreatedAt)
-            .ToList();
+        var activeRuns = await GetActiveRunsAsync(recentPage, pageSize, cancellationToken);
 
         // Populate query plan templates for the shared scenario editor modal embedded in this view.
         ViewBag.QueryPlanTemplates = await queryPlanTemplateStore.GetAllAsync(cancellationToken);
@@ -93,7 +74,7 @@ public async Task<IActionResult> RecentRunsPartial(
         // page navigation. The view model matches the partial's @model so the
         // partial is reused by both this action and the initial server render
         // in Index.cshtml — there's no divergence between the two templates.
-        var descending = !string.Equals(sortDir, "asc", StringComparison.OrdinalIgnoreCase);
+        var descending = IsDescending(sortDir);
         var page = await runManager.GetRunsPageAsync(pageNumber, pageSize, sortBy, descending, cancellationToken);
         return PartialView("_RecentRunsTable", page);
     }
@@ -106,28 +87,12 @@ public async Task<IActionResult> DashboardStats(
         string sortDir = "desc",
         CancellationToken cancellationToken = default)
     {
-        var descending = !string.Equals(sortDir, "asc", StringComparison.OrdinalIgnoreCase);
+        var descending = IsDescending(sortDir);
 
         var stats = await runManager.GetDashboardStatsAsync(cancellationToken);
         var recentPage = await runManager.GetRunsPageAsync(pageNumber, pageSize, sortBy, descending, cancellationToken);
 
-        var activeRunMetas = await snapshotStore.GetActiveRunsAsync(cancellationToken);
-        var activeRunSummaries = await Task.WhenAll(activeRunMetas.Select(meta => runManager.GetRunAsync(meta.RunId, cancellationToken)));
-
-        var activeRunsSource = recentPage.PageNumber == 1
-            ? recentPage.Runs
-            : (await runManager.GetRunsPageAsync(1, pageSize, "createdAt", true, cancellationToken)).Runs;
-        var statusActiveRuns = activeRunsSource
-            .Where(r => r.Status is AutomationRunStatus.Queued or AutomationRunStatus.Running);
-
-        var activeRuns = activeRunSummaries
-            .Where(r => r != null && (r.Status is AutomationRunStatus.Queued or AutomationRunStatus.Running))
-            .Select(r => r!)
-            .Concat(statusActiveRuns)
-            .GroupBy(r => r.RunId)
-            .Select(g => g.First())
-            .OrderByDescending(r => r.CreatedAt)
-            .ToList();
+        var activeRuns = await GetActiveRunsAsync(recentPage, pageSize, cancellationToken);
 
         return Json(new
         {
@@ -535,4 +500,28 @@ public async Task<IActionResult> DataAcquisitionLogDetail(
             return NotFound();
         }
     }
+
+    private static bool IsDescending(string sortDir)
+        => !string.Equals(sortDir, "asc", StringComparison.OrdinalIgnoreCase);
+
+    private async Task<List<AutomationRunSummary>> GetActiveRunsAsync(
+        AutomationRunIndexViewModel recentPage,
+        int pageSize,
+        CancellationToken cancellationToken)
+    {
+        var activeRunMetas = await snapshotStore.GetActiveRunsAsync(cancellationToken);
+        var activeRunSummaries = await Task.WhenAll(activeRunMetas.Select(meta => runManager.GetRunAsync(meta.RunId, cancellationToken)));
+        var activeRunsSource = recentPage.PageNumber == 1
+            ? recentPage.Runs
+            : (await runManager.GetRunsPageAsync(1, pageSize, "createdAt", true, cancellationToken)).Runs;
+
+        return activeRunSummaries
+            .Where(r => r is { Status: AutomationRunStatus.Queued or AutomationRunStatus.Running })
+            .Select(r => r!)
+            .Concat(activeRunsSource.Where(r => r.Status is AutomationRunStatus.Queued or AutomationRunStatus.Running))
+            .GroupBy(r => r.RunId)
+            .Select(g => g.First())
+            .OrderByDescending(r => r.CreatedAt)
+            .ToList();
+    }
 }