LEGLINK-770: Supply Redis max-memory via config for Hybrid cache fallback (#1764)

* LEGLINK-770: Supply Redis max-memory via config for Hybrid cache fallback

Azure Managed Redis does not return `maxmemory` from the INFO command, so
HybridResourceCache always assumed Redis was unconstrained and never cut
over to ABS under memory pressure. Per the LEGLINK-603 investigation
(Option C), the max-memory limit is now supplied through configuration.

- Add ResourceCache:Redis:MaxMemoryBytes to ResourceCacheRedisSettings
- SelectCacheType now uses the configured max as the utilization
  denominator (still reads used_memory from INFO); when the value is
  missing or <= 0 it logs a warning and defaults to Redis
- Document the key in app-config.yaml (both DataAcquisition sections) and
  default it to 268435456 (256 MB) in the DataAcquisition, AcquisitionWorker
  and Normalization appsettings; Azure App Config overrides per environment
- docker-compose: cap local Redis at 256mb (allkeys-lru) and pass a
  matching MaxMemoryBytes to the cache-consuming services
- Add HybridResourceCacheTests covering the selection branches

* LEGLINK-770: Document Normalization ResourceCache config keys

Addresses a CodeRabbit review finding on PR #1764: the Normalization
service consumes the Hybrid resource cache but app-config.yaml listed
its section as empty (Normalization: []), leaving all ResourceCache
keys — including the new ResourceCache:Redis:MaxMemoryBytes — undocumented.

Document Normalization's full ResourceCache block (mirroring the
DataAcquisition and DataAcquisitionWorker sections), including the
Azure Redis 1 GB capacity guidance for MaxMemoryBytes.

Author: MikeAtPinnacle

DotNet/DataAcquisition.AcquisitionWorker/appsettings.json

@@ -26,7 +26,8 @@
     "Redis": {
       "ConnectionString": "",
       "Password": "",
-      "MemoryThresholdPercent": 80.0
+      "MemoryThresholdPercent": 80.0,
+      "MaxMemoryBytes": 268435456
     },
     "BlobStorage": {
       "ConnectionString": "",

DotNet/DataAcquisition/appsettings.json

@@ -41,7 +41,8 @@
     "Redis": {
       "ConnectionString": "",
       "Password": "",
-      "MemoryThresholdPercent": 80.0
+      "MemoryThresholdPercent": 80.0,
+      "MaxMemoryBytes": 268435456
     },
     "BlobStorage": {
       "ConnectionString": "",

DotNet/Normalization/appsettings.json

@@ -93,7 +93,8 @@
     "Redis": {
       "ConnectionString": "",
       "Password": "",
-      "MemoryThresholdPercent": 80.0
+      "MemoryThresholdPercent": 80.0,
+      "MaxMemoryBytes": 268435456
     },
     "BlobStorage": {
       "ConnectionString": "",

DotNet/ServiceTests/UnitTests/Shared/ResourceCache/HybridResourceCacheTests.cs

@@ -0,0 +1,167 @@
+using FluentAssertions;
+using Hl7.Fhir.Model;
+using LantanaGroup.Link.Shared.Application.Enums;
+using LantanaGroup.Link.Shared.Application.Interfaces;
+using LantanaGroup.Link.Shared.Application.Models.Configs;
+using LantanaGroup.Link.Shared.Application.Services.ResourceCache;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using Moq;
+using StackExchange.Redis;
+
+namespace UnitTests.Shared.ResourceCache;
+
+/// <summary>
+/// Covers <see cref="HybridResourceCache"/>'s Redis-vs-ABS selection, which since LEGLINK-770
+/// derives the max-memory denominator from configuration (<see cref="ResourceCacheRedisSettings.MaxMemoryBytes"/>)
+/// rather than Redis <c>INFO maxmemory</c> (Azure Managed Redis does not return it).
+/// </summary>
+[Trait("Category", "UnitTests")]
+public class HybridResourceCacheTests
+{
+    private readonly Mock<IResourceCache> _redisCache = new();
+    private readonly Mock<IResourceCache> _absCache = new();
+    private readonly Mock<IConnectionMultiplexer> _multiplexer = new();
+    private readonly Mock<ILogger<HybridResourceCache>> _logger = new();
+
+    private HybridResourceCache CreateSut(ResourceCacheRedisSettings redisSettings)
+    {
+        var settings = new ResourceCacheSettings { Redis = redisSettings };
+        return new HybridResourceCache(
+            _redisCache.Object,
+            _absCache.Object,
+            _multiplexer.Object,
+            Options.Create(settings),
+            _logger.Object);
+    }
+
+    /// <summary>Configures a connected Redis server whose INFO memory section returns the given used_memory.</summary>
+    private void SetupRedisUsedMemory(long usedMemory)
+    {
+        SetupRedisInfo(new[] { new KeyValuePair<string, string>("used_memory", usedMemory.ToString()) });
+    }
+
+    private void SetupRedisInfo(IEnumerable<KeyValuePair<string, string>> memoryPairs)
+    {
+        var grouping = memoryPairs.GroupBy(_ => "memory").First();
+        var server = new Mock<IServer>();
+        server.SetupGet(s => s.IsConnected).Returns(true);
+        server.Setup(s => s.Info(It.IsAny<RedisValue>(), It.IsAny<CommandFlags>()))
+            .Returns(new[] { grouping });
+        _multiplexer.Setup(m => m.GetServers()).Returns(new[] { server.Object });
+    }
+
+    private void SetupNoConnectedServer()
+    {
+        _multiplexer.Setup(m => m.GetServers()).Returns(Array.Empty<IServer>());
+    }
+
+    private void Write(HybridResourceCache sut, string correlationId)
+    {
+        sut.UpdateCorrelationCache(correlationId, new List<DomainResource>(), ResourceType.Patient);
+    }
+
+    private void VerifyWroteTo(Mock<IResourceCache> expected, Mock<IResourceCache> notExpected)
+    {
+        expected.Verify(c => c.UpdateCorrelationCache(It.IsAny<string>(), It.IsAny<List<DomainResource>>(), It.IsAny<ResourceType>()), Times.Once);
+        notExpected.Verify(c => c.UpdateCorrelationCache(It.IsAny<string>(), It.IsAny<List<DomainResource>>(), It.IsAny<ResourceType>()), Times.Never);
+    }
+
+    private void VerifyWarningLogged(Times times)
+    {
+        _logger.Verify(
+            log => log.Log(
+                LogLevel.Warning,
+                It.IsAny<EventId>(),
+                It.IsAny<It.IsAnyType>(),
+                It.IsAny<Exception>(),
+                It.IsAny<Func<It.IsAnyType, Exception?, string>>()),
+            times);
+    }
+
+    [Fact]
+    public void UsedMemory_below_threshold_of_configured_max_uses_Redis()
+    {
+        // 100 MB used of 1000 MB max = 10%, threshold 80% => Redis
+        SetupRedisUsedMemory(100L * 1024 * 1024);
+        var sut = CreateSut(new ResourceCacheRedisSettings { MaxMemoryBytes = 1000L * 1024 * 1024, MemoryThresholdPercent = 80.0 });
+
+        Write(sut, "corr-below");
+
+        VerifyWroteTo(_redisCache, _absCache);
+    }
+
+    [Fact]
+    public void UsedMemory_at_or_above_threshold_of_configured_max_uses_ABS()
+    {
+        // 900 MB used of 1000 MB max = 90%, threshold 80% => ABS
+        SetupRedisUsedMemory(900L * 1024 * 1024);
+        var sut = CreateSut(new ResourceCacheRedisSettings { MaxMemoryBytes = 1000L * 1024 * 1024, MemoryThresholdPercent = 80.0 });
+
+        Write(sut, "corr-above");
+
+        VerifyWroteTo(_absCache, _redisCache);
+    }
+
+    [Theory]
+    [InlineData(null)]
+    [InlineData(0L)]
+    [InlineData(-1L)]
+    public void MaxMemoryBytes_missing_or_invalid_uses_Redis_and_logs_warning(long? maxMemoryBytes)
+    {
+        SetupRedisUsedMemory(900L * 1024 * 1024); // would be ABS if a valid max were configured
+        var sut = CreateSut(new ResourceCacheRedisSettings { MaxMemoryBytes = maxMemoryBytes, MemoryThresholdPercent = 80.0 });
+
+        Write(sut, "corr-nomax");
+
+        VerifyWroteTo(_redisCache, _absCache);
+        VerifyWarningLogged(Times.Once());
+    }
+
+    [Fact]
+    public void UsedMemory_missing_from_info_uses_Redis()
+    {
+        SetupRedisInfo(new[] { new KeyValuePair<string, string>("some_other_stat", "123") });
+        var sut = CreateSut(new ResourceCacheRedisSettings { MaxMemoryBytes = 1000L * 1024 * 1024, MemoryThresholdPercent = 80.0 });
+
+        Write(sut, "corr-nousedmem");
+
+        VerifyWroteTo(_redisCache, _absCache);
+    }
+
+    [Fact]
+    public void No_connected_server_uses_ABS()
+    {
+        SetupNoConnectedServer();
+        var sut = CreateSut(new ResourceCacheRedisSettings { MaxMemoryBytes = 1000L * 1024 * 1024, MemoryThresholdPercent = 80.0 });
+
+        Write(sut, "corr-noserver");
+
+        VerifyWroteTo(_absCache, _redisCache);
+    }
+
+    [Fact]
+    public void Exception_reading_memory_uses_ABS()
+    {
+        _multiplexer.Setup(m => m.GetServers()).Throws(new RedisException("boom"));
+        var sut = CreateSut(new ResourceCacheRedisSettings { MaxMemoryBytes = 1000L * 1024 * 1024, MemoryThresholdPercent = 80.0 });
+
+        Write(sut, "corr-throws");
+
+        VerifyWroteTo(_absCache, _redisCache);
+    }
+
+    [Fact]
+    public void Decision_is_memoized_per_correlationId()
+    {
+        SetupRedisUsedMemory(100L * 1024 * 1024);
+        var sut = CreateSut(new ResourceCacheRedisSettings { MaxMemoryBytes = 1000L * 1024 * 1024, MemoryThresholdPercent = 80.0 });
+
+        Write(sut, "corr-memo");
+        Write(sut, "corr-memo");
+
+        // The memory pressure check (GetServers) should only happen once for the same correlationId.
+        _multiplexer.Verify(m => m.GetServers(), Times.Once);
+        _redisCache.Verify(c => c.UpdateCorrelationCache(It.IsAny<string>(), It.IsAny<List<DomainResource>>(), It.IsAny<ResourceType>()), Times.Exactly(2));
+    }
+}

DotNet/Shared/Application/Models/Configs/ResourceCacheSettings.cs

@@ -25,9 +25,20 @@ public class ResourceCacheRedisSettings
         public string? ConnectionString { get; set; }
         public string? Password { get; set; }
         /// <summary>
-        /// The percentage of maxmemory at which the cache will fall back to ABS.
-        /// Defaults to 80. When Redis maxmemory is 0 (unlimited), Redis is always used.
+        /// The percentage of the configured Redis max-memory (<see cref="MaxMemoryBytes"/>) at
+        /// which Hybrid caching falls back to ABS. Defaults to 80. When
+        /// <see cref="MaxMemoryBytes"/> is unknown (null or &lt;= 0) Redis is always used.
         /// </summary>
         public double MemoryThresholdPercent { get; set; } = 80.0;
+
+        /// <summary>
+        /// The Redis max-memory limit in bytes, supplied via configuration because Azure
+        /// Managed Redis does not return <c>maxmemory</c> via the INFO command. Used as the
+        /// denominator when computing memory utilization for Hybrid cache fallback (the
+        /// numerator, <c>used_memory</c>, is still read from INFO). When null or &lt;= 0 the
+        /// limit is treated as unknown and Redis is always used (a warning is logged).
+        /// NOTE: this must be updated manually if Redis capacity is scaled.
+        /// </summary>
+        public long? MaxMemoryBytes { get; set; }
     }
 }

DotNet/Shared/Application/Services/ResourceCache/HybridResourceCache.cs

@@ -137,16 +137,19 @@ private ResourceCacheType SelectCacheType()
                     return ResourceCacheType.Redis;
                 }
 
-                if (!infoDict.TryGetValue("maxmemory", out var maxMemoryStr) ||
-                    !long.TryParse(maxMemoryStr, out var maxMemory) ||
-                    maxMemory == 0)
+                // Azure Managed Redis does not return `maxmemory` via INFO, so the limit is
+                // supplied through configuration instead. Continue reading `used_memory` above.
+                var maxMemoryBytes = _settings.Redis.MaxMemoryBytes;
+                if (maxMemoryBytes is null or <= 0)
                 {
-                    // No memory limit configured — Redis is unconstrained, always use it.
-                    _logger.LogDebug("Redis maxmemory is not configured or unlimited; defaulting to Redis resource cache.");
+                    _logger.LogWarning(
+                        "ResourceCache:Redis:MaxMemoryBytes is not configured or invalid ({MaxMemoryBytes}); " +
+                        "cannot evaluate Redis memory pressure. Defaulting to Redis resource cache.",
+                        maxMemoryBytes);
                     return ResourceCacheType.Redis;
                 }
 
-                double usagePercent = (double)usedMemory / maxMemory * 100.0;
+                double usagePercent = (double)usedMemory / maxMemoryBytes.Value * 100.0;
 
                 if (usagePercent >= _settings.Redis.MemoryThresholdPercent)
                 {

app-config.yaml

@@ -217,7 +217,10 @@ services:
       description: "Optional Redis password for the resource cache. Defaults to empty."
       required: false
     - key: "ResourceCache:Redis:MemoryThresholdPercent"
-      description: "Optional Redis maxmemory percentage at which Hybrid caching falls back to blob storage. Defaults to 80.0."
+      description: "Optional percentage of ResourceCache:Redis:MaxMemoryBytes at which Hybrid caching falls back to blob storage. Defaults to 80.0."
+      required: false
+    - key: "ResourceCache:Redis:MaxMemoryBytes"
+      description: "The Redis max-memory limit in bytes, used as the denominator for Hybrid cache memory-pressure fallback. Supplied via configuration because Azure Managed Redis does not return maxmemory via INFO. Recommended for Hybrid; when unset or <= 0 Redis is always used (a warning is logged). All LCG Redis instances are currently 1 GB (1073741824); this must be updated if Redis capacity is scaled. Defaults to 268435456 (256 MB) to match local Docker Redis."
       required: false
     - key: "ResourceCache:BlobStorage:ConnectionString"
       description: "The Azure Blob Storage connection string used by the resource cache. Required when ResourceCache:CacheImplementation is Hybrid or ABS."
@@ -248,7 +251,10 @@ services:
       description: "Optional Redis password for the resource cache. Defaults to empty."
       required: false
     - key: "ResourceCache:Redis:MemoryThresholdPercent"
-      description: "Optional Redis maxmemory percentage at which Hybrid caching falls back to blob storage. Defaults to 80.0."
+      description: "Optional percentage of ResourceCache:Redis:MaxMemoryBytes at which Hybrid caching falls back to blob storage. Defaults to 80.0."
+      required: false
+    - key: "ResourceCache:Redis:MaxMemoryBytes"
+      description: "The Redis max-memory limit in bytes, used as the denominator for Hybrid cache memory-pressure fallback. Supplied via configuration because Azure Managed Redis does not return maxmemory via INFO. Recommended for Hybrid; when unset or <= 0 Redis is always used (a warning is logged). All LCG Redis instances are currently 1 GB (1073741824); this must be updated if Redis capacity is scaled. Defaults to 268435456 (256 MB) to match local Docker Redis."
       required: false
     - key: "ResourceCache:BlobStorage:ConnectionString"
       description: "The Azure Blob Storage connection string used by the resource cache. Required when ResourceCache:CacheImplementation is Hybrid or ABS."
@@ -288,7 +294,31 @@ services:
     - key: "resource-cache.blob-storage.blob-root"
       description: "Root path/prefix within the blob container under which cached resources are stored for the ABS resource cache. Required only when the ABS resource-cache backend is used."
       required: false
-  Normalization: []
+  Normalization:
+    - key: "ResourceCache:CacheImplementation"
+      description: "The resource cache implementation to register. Defaults to Hybrid; supported values are Hybrid, Redis, and ABS."
+      required: false
+    - key: "ResourceCache:Redis:ConnectionString"
+      description: "The Redis endpoint used by the resource cache. Required when ResourceCache:CacheImplementation is Hybrid or Redis."
+      required: true
+    - key: "ResourceCache:Redis:Password"
+      description: "Optional Redis password for the resource cache. Defaults to empty."
+      required: false
+    - key: "ResourceCache:Redis:MemoryThresholdPercent"
+      description: "Optional percentage of ResourceCache:Redis:MaxMemoryBytes at which Hybrid caching falls back to blob storage. Defaults to 80.0."
+      required: false
+    - key: "ResourceCache:Redis:MaxMemoryBytes"
+      description: "The Redis max-memory limit in bytes, used as the denominator for Hybrid cache memory-pressure fallback. Supplied via configuration because Azure Managed Redis does not return maxmemory via INFO. Recommended for Hybrid; when unset or <= 0 Redis is always used (a warning is logged). All LCG Redis instances are currently 1 GB (1073741824); this must be updated if Redis capacity is scaled. Defaults to 268435456 (256 MB) to match local Docker Redis."
+      required: false
+    - key: "ResourceCache:BlobStorage:ConnectionString"
+      description: "The Azure Blob Storage connection string used by the resource cache. Required when ResourceCache:CacheImplementation is Hybrid or ABS."
+      required: true
+    - key: "ResourceCache:BlobStorage:BlobContainerName"
+      description: "Optional blob container name used by the resource cache. Defaults to empty."
+      required: false
+    - key: "ResourceCache:BlobStorage:BlobRoot"
+      description: "Optional blob name prefix used by the resource cache. Defaults to empty."
+      required: false
   Notification:
     - key: "SmtpConnection:Host"
       description: "The SMTP server hostname for sending notifications."

docker-compose.yml

@@ -96,7 +96,7 @@ services:
     ports:
       - '6379:6379'
     container_name: redis_cache
-    command: redis-server --save 20 1 --loglevel warning --requirepass ${REDIS_PASS}
+    command: redis-server --save 20 1 --loglevel warning --requirepass ${REDIS_PASS} --maxmemory 256mb --maxmemory-policy allkeys-lru
     volumes: 
       - redis_cache:/data
     networks:
@@ -600,6 +600,7 @@ services:
       ResourceCache__CacheImplementation: Hybrid
       ResourceCache__Redis__ConnectionString: redis_cache:6379
       ResourceCache__Redis__Password: ${REDIS_PASS}
+      ResourceCache__Redis__MaxMemoryBytes: 268435456
       ResourceCache__BlobStorage__ConnectionString: ${AZURITE_CONNECTION_STRING}
       ResourceCache__BlobStorage__BlobContainerName: ${INTERNAL_BLOB_CONTAINER_NAME}
       ResourceCache__BlobStorage__BlobRoot: resource-cache
@@ -643,6 +644,7 @@ services:
        ResourceCache__CacheImplementation: Hybrid
        ResourceCache__Redis__ConnectionString: redis_cache:6379
        ResourceCache__Redis__Password: ${REDIS_PASS}
+       ResourceCache__Redis__MaxMemoryBytes: 268435456
        ResourceCache__BlobStorage__ConnectionString: ${AZURITE_CONNECTION_STRING}
        ResourceCache__BlobStorage__BlobContainerName: ${INTERNAL_BLOB_CONTAINER_NAME}
        ResourceCache__BlobStorage__BlobRoot: resource-cache
@@ -776,6 +778,7 @@ services:
       ConnectionStrings__DatabaseConnection: Server=tcp:mssql,1433;Initial Catalog=link-normalization;Persist Security Info=False;User ID=sa;Password=${LINK_DB_PASS};MultipleActiveResultSets=False;Encrypt=True;TrustServerCertificate=True;Connection Timeout=30;
       ResourceCache__Redis__ConnectionString: redis_cache:6379
       ResourceCache__Redis__Password: ${REDIS_PASS}
+      ResourceCache__Redis__MaxMemoryBytes: 268435456
       ResourceCache__BlobStorage__ConnectionString: ${AZURITE_CONNECTION_STRING}
       ResourceCache__BlobStorage__BlobContainerName: ${INTERNAL_BLOB_CONTAINER_NAME}
       ResourceCache__BlobStorage__BlobRoot: resource-cache