Skip to content

Commit 534d186

Browse files
LEGLINK-186: Fail fast on Validation blob calls so the retry-topic ladder owns retrying
Mirrors the MeasureEval resource-cache change: the Validation blob client (bundle download and pre-qual append, both on the consumer path) used the Azure SDK's default retry policy, hiding minutes of silent retrying inside every consumer attempt during an ABS outage while blocking the single- threaded consumer executor. The client now uses a fail-fast policy: FIXED, maxTries=2, tryTimeout=10s, 1s delay, configurable via internal-blob-storage.max-tries / .try-timeout-seconds. Measured live with azurite stopped: attempt latency dropped to 5.1s, and the first retry rung after azurite's restart completed validation and produced ValidationComplete (33s outage-to- recovery total). Claude-Session: https://claude.ai/code/session_01KN565tFAuJUAEkK2DdRF1e
1 parent 73b6246 commit 534d186

3 files changed

Lines changed: 75 additions & 2 deletions

File tree

Java/validation/src/main/java/com/lantanagroup/link/validation/configs/BlobStorageConfig.java

Lines changed: 32 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
package com.lantanagroup.link.validation.configs;
22

3+
import com.azure.storage.common.policy.RequestRetryOptions;
4+
import com.azure.storage.common.policy.RetryPolicyType;
35
import com.lantanagroup.link.validation.services.BlobStorageService;
46
import lombok.Getter;
57
import lombok.Setter;
@@ -8,6 +10,8 @@
810
import org.springframework.context.annotation.Bean;
911
import org.springframework.context.annotation.Configuration;
1012

13+
import java.time.Duration;
14+
1115
@Getter
1216
@Setter
1317
@Configuration
@@ -16,11 +20,38 @@ public class BlobStorageConfig {
1620
private String connectionString;
1721
private String blobContainerName;
1822

23+
/**
24+
* Azure SDK in-process tries per blob call (initial call + quick retries). This client is used
25+
* entirely on the consumer path — the bundle download and the pre-qual append both run inside
26+
* process() — where the Kafka retry-topic ladder owns real retrying. The SDK default of 4
27+
* exponential tries hid minutes of silent retrying inside every consumer attempt while blocking
28+
* the single-threaded consumer executor.
29+
*/
30+
private int maxTries = 2;
31+
32+
/** Cap on a single network attempt; the SDK default is effectively unbounded. */
33+
private int tryTimeoutSeconds = 10;
34+
1935
@Bean
2036
public BlobStorageService blobStorageService() {
2137
if (StringUtils.isAnyEmpty(connectionString, blobContainerName)) {
2238
return null;
2339
}
24-
return new BlobStorageService(connectionString, blobContainerName);
40+
return new BlobStorageService(connectionString, blobContainerName, buildRetryOptions());
41+
}
42+
43+
/**
44+
* Fail-fast policy: an ABS outage should surface as a quick failure the retry-topic ladder
45+
* handles visibly (log lines, attempt headers, dead-lettering), not as silent in-SDK retrying.
46+
* Package-private so the test pins the exact values.
47+
*/
48+
RequestRetryOptions buildRetryOptions() {
49+
return new RequestRetryOptions(
50+
RetryPolicyType.FIXED,
51+
maxTries,
52+
Duration.ofSeconds(tryTimeoutSeconds),
53+
Duration.ofSeconds(1),
54+
Duration.ofSeconds(1),
55+
null);
2556
}
2657
}

Java/validation/src/main/java/com/lantanagroup/link/validation/services/BlobStorageService.java

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,16 +6,18 @@
66
import com.azure.storage.blob.BlobServiceClient;
77
import com.azure.storage.blob.BlobServiceClientBuilder;
88
import com.azure.storage.blob.specialized.AppendBlobClient;
9+
import com.azure.storage.common.policy.RequestRetryOptions;
910

1011
import java.io.ByteArrayInputStream;
1112
import java.nio.charset.StandardCharsets;
1213

1314
public class BlobStorageService {
1415
private final BlobContainerClient containerClient;
1516

16-
public BlobStorageService(String connectionString, String blobContainerName) {
17+
public BlobStorageService(String connectionString, String blobContainerName, RequestRetryOptions retryOptions) {
1718
BlobServiceClient serviceClient = new BlobServiceClientBuilder()
1819
.connectionString(connectionString)
20+
.retryOptions(retryOptions)
1921
.buildClient();
2022
containerClient = serviceClient.getBlobContainerClient(blobContainerName);
2123
}
Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
package com.lantanagroup.link.validation.configs;
2+
3+
import com.azure.storage.common.policy.RequestRetryOptions;
4+
import org.junit.jupiter.api.Test;
5+
6+
import static org.junit.jupiter.api.Assertions.assertEquals;
7+
8+
/**
9+
* The validation blob client is used entirely on the consumer path (the bundle download and the
10+
* pre-qual append both run inside process()), where the Kafka retry-topic ladder owns retrying.
11+
* The Azure SDK's default policy (4 exponential tries, effectively unbounded per-try timeout) hides
12+
* minutes of silent retrying inside every consumer attempt during an ABS outage while blocking the
13+
* single-threaded consumer executor. These tests pin the fail-fast policy handed to the builder.
14+
*/
15+
class BlobStorageConfigTest {
16+
17+
@Test
18+
void retryOptions_failFast_defaults() {
19+
BlobStorageConfig config = new BlobStorageConfig();
20+
21+
RequestRetryOptions options = config.buildRetryOptions();
22+
23+
assertEquals(2, options.getMaxTries(),
24+
"one quick in-process retry absorbs a dropped connection; anything more belongs to the ladder");
25+
assertEquals(10, options.getTryTimeoutDuration().toSeconds(),
26+
"each network attempt must be capped, not left to the SDK's effectively unbounded default");
27+
}
28+
29+
@Test
30+
void retryOptions_areConfigurable() {
31+
BlobStorageConfig config = new BlobStorageConfig();
32+
config.setMaxTries(1);
33+
config.setTryTimeoutSeconds(5);
34+
35+
RequestRetryOptions options = config.buildRetryOptions();
36+
37+
assertEquals(1, options.getMaxTries());
38+
assertEquals(5, options.getTryTimeoutDuration().toSeconds());
39+
}
40+
}

0 commit comments

Comments
 (0)