LEGLINK-186: Fail fast on resource-cache blob calls so the retry-topic ladder owns retrying

The Azure SDK's default retry policy (4 exponential tries, effectively
unbounded per-try timeout) hid ~3 minutes of silent retrying inside every
consumer attempt during an ABS outage - blocking the single-threaded
consumer executor (head-of-line blocking for all records) and multiplying
the retry-topic ladder, which stretched from its designed ~3.5 minutes to
~11 before dead-lettering.

The cache blob client now uses a fail-fast policy: FIXED, maxTries=2
(initial call + one quick retry to absorb a dropped connection),
tryTimeout=10s, 1s delay. Real retrying belongs to the Kafka retry-topic
ladder, which is visible (log lines, attempt headers), configurable, and
ends in a dead letter. Both knobs are configuration properties
(resource-cache.blob-storage.max-tries / .try-timeout-seconds).

Measured live with azurite stopped: attempt latency dropped from ~2m48s
to 14.9s, and the recovery leg (azurite restored before the redelivery)
completed the empty-cache success path unchanged.

Deliberately not applied to BlobStorageService (report payload uploads):
that client is on the success path, where the SDK's patient retries help
rather than harm.

Claude-Session: https://claude.ai/code/session_01KN565tFAuJUAEkK2DdRF1e

Author: arianamihailescu

Java/measureeval/src/main/java/com/lantanagroup/link/measureeval/configs/CacheBlobStorageConfig.java

@@ -2,6 +2,8 @@
 
 import com.azure.storage.blob.BlobContainerClient;
 import com.azure.storage.blob.BlobServiceClientBuilder;
+import com.azure.storage.common.policy.RequestRetryOptions;
+import com.azure.storage.common.policy.RetryPolicyType;
 import com.lantanagroup.link.measureeval.services.AbsResourceService;
 import lombok.Getter;
 import lombok.Setter;
@@ -13,6 +15,8 @@
 import org.springframework.context.annotation.Configuration;
 import org.springframework.lang.Nullable;
 
+import java.time.Duration;
+
 @Getter
 @Setter
 @Configuration
@@ -24,18 +28,46 @@ public class CacheBlobStorageConfig {
     private String blobContainerName;
     private String blobRoot;
 
+    /**
+     * Azure SDK in-process tries per blob call (initial call + quick retries). The Kafka
+     * retry-topic ladder owns real retrying; one extra in-process try only absorbs a dropped
+     * connection. The SDK default of 4 exponential tries hid minutes of silent retrying inside
+     * every consumer attempt while blocking the single-threaded consumer executor.
+     */
+    private int maxTries = 2;
+
+    /** Cap on a single network attempt; the SDK default is effectively unbounded. */
+    private int tryTimeoutSeconds = 10;
+
     @Bean
     @Nullable
     public AbsResourceService absResourceService() {
         if (StringUtils.isAnyEmpty(connectionString, blobContainerName)) {
             logger.info("cache-blob-storage not configured, AbsResourceService disabled");
             return null;
         }
-        logger.info("Creating AbsResourceService: container={}, blobRoot={}", blobContainerName, blobRoot);
+        logger.info("Creating AbsResourceService: container={}, blobRoot={}, maxTries={}, tryTimeout={}s",
+                blobContainerName, blobRoot, maxTries, tryTimeoutSeconds);
         BlobContainerClient client = new BlobServiceClientBuilder()
                 .connectionString(connectionString)
+                .retryOptions(buildRetryOptions())
                 .buildClient()
                 .getBlobContainerClient(blobContainerName);
         return new AbsResourceService(client, blobRoot != null ? blobRoot : "");
     }
+
+    /**
+     * Fail-fast policy for the cache blob client: an ABS outage should surface as a quick failure
+     * that the retry-topic ladder handles visibly (log lines, attempt headers, dead-lettering),
+     * not as silent in-SDK retrying. Package-private so the test pins the exact values.
+     */
+    RequestRetryOptions buildRetryOptions() {
+        return new RequestRetryOptions(
+                RetryPolicyType.FIXED,
+                maxTries,
+                Duration.ofSeconds(tryTimeoutSeconds),
+                Duration.ofSeconds(1),
+                Duration.ofSeconds(1),
+                null);
+    }
 }

Java/measureeval/src/test/java/com/lantanagroup/link/measureeval/configs/CacheBlobStorageConfigTest.java

@@ -0,0 +1,39 @@
+package com.lantanagroup.link.measureeval.configs;
+
+import com.azure.storage.common.policy.RequestRetryOptions;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+/**
+ * The resource-cache blob client must fail fast: the Kafka retry-topic ladder owns retrying, and
+ * the Azure SDK's default policy (4 exponential tries, effectively unbounded per-try timeout) hid
+ * ~3 minutes of silent retrying inside every consumer attempt while blocking the single-threaded
+ * consumer executor. These tests pin the tuned policy actually handed to the client builder.
+ */
+class CacheBlobStorageConfigTest {
+
+    @Test
+    void retryOptions_failFast_defaults() {
+        CacheBlobStorageConfig config = new CacheBlobStorageConfig();
+
+        RequestRetryOptions options = config.buildRetryOptions();
+
+        assertEquals(2, options.getMaxTries(),
+                "one quick in-process retry absorbs a dropped connection; anything more belongs to the ladder");
+        assertEquals(10, options.getTryTimeoutDuration().toSeconds(),
+                "each network attempt must be capped, not left to the SDK's effectively unbounded default");
+    }
+
+    @Test
+    void retryOptions_areConfigurable() {
+        CacheBlobStorageConfig config = new CacheBlobStorageConfig();
+        config.setMaxTries(1);
+        config.setTryTimeoutSeconds(5);
+
+        RequestRetryOptions options = config.buildRetryOptions();
+
+        assertEquals(1, options.getMaxTries());
+        assertEquals(5, options.getTryTimeoutDuration().toSeconds());
+    }
+}