Added RETRY-UNTIL-TEST-PASS directive

Author: Pavol Beťák

demo/Tests/002-Cars/003-Check-Car-init.csx

@@ -0,0 +1,4 @@
+// Initialize counters for different retry scenarios
+tp.SetVariable("CarAvailabilityCheckCount", 0);
+tp.SetVariable("UnavailableCarCheckCount", 0);
+tp.SetVariable("CarCreationCheckCount", 0);

demo/Tests/002-Cars/003-Check-Car-req.http

@@ -0,0 +1,29 @@
+### Scenario 1: Check Car Availability (passes after 3 attempts)
+## RETRY-STRATEGY: Custom retry
+## RETRY-MAX-ATTEMPTS: 5
+## RETRY-UNTIL-TEST-PASS: Car should be available
+
+# @name CheckCarAvailabilityRequest
+GET {{ApiBaseUrl}}{{ApiCarsSection}}/1
+
+###
+
+### Scenario 2: Check Unavailable Car (never passes - exceeds max attempts)
+## RETRY-STRATEGY: Custom retry
+## RETRY-MAX-ATTEMPTS: 3
+## RETRY-UNTIL-TEST-PASS: Car with special features should exist
+
+# @name CheckUnavailableCarRequest
+GET {{ApiBaseUrl}}{{ApiCarsSection}}/999
+
+###
+
+### Scenario 3: Verify Car Creation (combined RETRY-UNTIL-STATUS + RETRY-UNTIL-TEST-PASS)
+## RETRY-STRATEGY: Custom retry
+## RETRY-MAX-ATTEMPTS: 4
+## RETRY-MAX-DELAY: 00:00:02
+## RETRY-UNTIL-STATUS: [200, 201]
+## RETRY-UNTIL-TEST-PASS: Newly created car should have correct brand
+
+# @name VerifyCarCreationRequest
+GET {{ApiBaseUrl}}{{ApiCarsSection}}/7

demo/Tests/002-Cars/003-Check-Car-test.csx

@@ -0,0 +1,65 @@
+// Test 1: Car becomes available after 3 attempts
+tp.Test("Car should be available", () =>
+{
+    var checkCount = tp.GetVariable<int>("CarAvailabilityCheckCount");
+    checkCount++;
+    tp.SetVariable("CarAvailabilityCheckCount", checkCount);
+
+    var response = tp.Response;
+    var statusCode = response.StatusCode();
+
+    if (checkCount >= 3)
+    {
+        tp.Logger.LogInformation($"Car is now available after {checkCount} attempts!");
+        Equal(200, statusCode);
+    }
+    else
+    {
+        tp.Logger.LogInformation($"Car not available yet (attempt {checkCount})");
+        throw new Exception($"Car not available yet (attempt {checkCount})");
+    }
+});
+
+// Test 2: Car never becomes available (will fail after max attempts)
+await tp.Test("Car with special features should exist", async () =>
+{
+    var checkCount = tp.GetVariable<int>("UnavailableCarCheckCount");
+    checkCount++;
+    tp.SetVariable("UnavailableCarCheckCount", checkCount);
+
+    var response = tp.Response;
+    var statusCode = response.StatusCode();
+
+    tp.Logger.LogInformation($"Checking for unavailable car (attempt {checkCount}), status: {statusCode}");
+
+    // This will always fail - demonstrating max attempts behavior
+    Equal(200, statusCode);
+
+    dynamic carData = await response.GetBodyAsExpandoAsync();
+    Equal("SpecialFeatures", carData.Brand);
+});
+
+// Test 3: Combined scenario - verify newly created car has correct brand
+await tp.Test("Newly created car should have correct brand", async () =>
+{
+    var checkCount = tp.GetVariable<int>("CarCreationCheckCount");
+    checkCount++;
+    tp.SetVariable("CarCreationCheckCount", checkCount);
+
+    var response = tp.Response;
+    var statusCode = response.StatusCode();
+
+    tp.Logger.LogInformation($"Verifying car creation (attempt {checkCount}), status: {statusCode}");
+
+    if (checkCount >= 2 && (statusCode == 200 || statusCode == 201))
+    {
+        dynamic carData = await response.GetBodyAsExpandoAsync();
+        tp.Logger.LogInformation($"Car found with brand: {carData.Brand}");
+        Equal("Audi", carData.Brand);
+    }
+    else
+    {
+        tp.Logger.LogInformation($"Car not ready yet (attempt {checkCount})");
+        throw new Exception($"Car not ready yet (attempt {checkCount})");
+    }
+});

demo/Tests/003-Car-Rentals/001-Rent-Car-test.csx

@@ -36,6 +36,4 @@ await tp.Test($"Rented car should be '{car}'.", async () =>
     Equal(car.Year, retrievedCar.Year);
 });
 
-ClearVariables();
-
 tp.Logger.LogInformation("End of demo collection testing.");

src/TeaPie/ApplicationBuilder.cs

@@ -165,7 +165,7 @@ private static TeaPie CreateUserContext(IServiceProvider provider, ApplicationCo
             provider.GetRequiredService<IVariables>(),
             provider.GetRequiredService<IFunctions>(),
             provider.GetRequiredService<ILogger<TeaPie>>(),
-            provider.GetRequiredService<ITester>(),
+            provider.GetRequiredService<IRegistrator>(),
             provider.GetRequiredService<ICurrentTestCaseExecutionContextAccessor>(),
             provider.GetRequiredService<ITestResultsSummaryReporter>(),
             provider.GetRequiredService<IRetryStrategyRegistry>(),

src/TeaPie/Http/Parsing/DirectivesLineParser.cs

@@ -10,6 +10,7 @@ internal class DirectivesLineParser : ILineParser
         new TestDirectivesLineParser(),
         new RetryStrategyDirectiveLineParser(),
         new RetryUntilStatusCodesLineParser(),
+        new RetryUntilTestPassDirectiveLineParser(),
         new RetryExplicitPropertiesDirectiveLineParser()
     ];
 

src/TeaPie/Http/Parsing/HttpParsingContext.cs

@@ -26,6 +26,7 @@ internal class HttpParsingContext(HttpRequestHeaders defaultHeaders)
     public string RetryStrategyName { get; set; } = string.Empty;
     public IReadOnlyList<HttpStatusCode> RetryUntilStatusCodes { get; set; } = [];
     public RetryStrategyOptions<HttpResponseMessage>? ExplicitRetryStrategy { get; set; }
+    public string RetryUntilTestPassTestName { get; set; } = string.Empty;
 
     public string AuthProviderName { get; set; } = string.Empty;
 

src/TeaPie/Http/Parsing/HttpRequestParser.cs

@@ -131,7 +131,8 @@ private void UpdateResiliencePipeline(RequestExecutionContext requestExecutionCo
             _resiliencePipelineProvider.GetResiliencePipeline(
                 parsingContext.RetryStrategyName,
                 parsingContext.ExplicitRetryStrategy,
-                parsingContext.RetryUntilStatusCodes);
+                parsingContext.RetryUntilStatusCodes,
+                parsingContext.RetryUntilTestPassTestName);
 
     private static void CreateMessageContent(HttpParsingContext context, HttpRequestMessage requestMessage)
     {

src/TeaPie/Http/Retrying/ResiliencePipelineProvider.cs

@@ -3,6 +3,7 @@
 using Polly.Retry;
 using System.Net;
 using System.Text;
+using TeaPie.Testing;
 using ResiliencePipeline = Polly.ResiliencePipeline<System.Net.Http.HttpResponseMessage>;
 using RetryStrategy = Polly.Retry.RetryStrategyOptions<System.Net.Http.HttpResponseMessage>;
 
@@ -11,14 +12,21 @@ namespace TeaPie.Http.Retrying;
 internal interface IResiliencePipelineProvider
 {
     ResiliencePipeline GetResiliencePipeline(
-        string nameOfBaseStrategy, RetryStrategy? explicitlyOverridenStrategy, IReadOnlyList<HttpStatusCode> statusCodes);
+        string nameOfBaseStrategy,
+        RetryStrategy? explicitlyOverridenStrategy,
+        IReadOnlyList<HttpStatusCode> statusCodes,
+        string? retryUntilTestPassTestName = null);
 }
 
-internal class ResiliencePipelineProvider(IRetryStrategyRegistry registry, ILogger<ResiliencePipelineProvider> logger)
+internal class ResiliencePipelineProvider(
+    IRetryStrategyRegistry registry,
+    ILogger<ResiliencePipelineProvider> logger,
+    ITester testExecutor)
     : IResiliencePipelineProvider
 {
     private readonly IRetryStrategyRegistry _retryStrategyRegistry = registry;
     private readonly ILogger<ResiliencePipelineProvider> _logger = logger;
+    private readonly ITester _testExecutor = testExecutor;
 
     public static readonly ResiliencePipeline DefaultResiliencePipeline =
         ResiliencePipeline.Empty;
@@ -27,14 +35,19 @@ internal class ResiliencePipelineProvider(IRetryStrategyRegistry registry, ILogg
         new() { { string.Empty, DefaultResiliencePipeline } };
 
     public ResiliencePipeline<HttpResponseMessage> GetResiliencePipeline(
-        string nameOfBaseRetryStrategy, RetryStrategy? overridRetryStrategy, IReadOnlyList<HttpStatusCode> statusCodes)
+        string nameOfBaseRetryStrategy,
+        RetryStrategy? overridRetryStrategy,
+        IReadOnlyList<HttpStatusCode> statusCodes,
+        string? retryUntilTestPassTestName = null)
     {
         var isBaseRetryStrategyAltered = false;
         CheckAndResolveBaseRetryStrategy(nameOfBaseRetryStrategy, out var finalRetryStrategy, out var nameOfFinalStrategy);
         ApplyExplicitOverridesIfAny(
             overridRetryStrategy, ref finalRetryStrategy, ref nameOfFinalStrategy, ref isBaseRetryStrategyAltered);
         ApplyRetryUntilStatusCodesConditionIfAny(
             statusCodes, ref finalRetryStrategy, ref nameOfFinalStrategy, ref isBaseRetryStrategyAltered);
+        ApplyRetryUntilTestPassConditionIfAny(
+           retryUntilTestPassTestName, ref finalRetryStrategy, ref nameOfFinalStrategy, ref isBaseRetryStrategyAltered);
 
         LogUsageOfRetryStrategy(nameOfFinalStrategy, finalRetryStrategy, isBaseRetryStrategyAltered);
 
@@ -50,7 +63,25 @@ private void ApplyRetryUntilStatusCodesConditionIfAny(
         if (statusCodes.Any())
         {
             nameOfFinalStrategy = GetRetryUntilStatusCodesStrategyName(statusCodes, nameOfFinalStrategy);
-            finalRetryStrategy = GetRetryStrategy(statusCodes, finalRetryStrategy);
+            finalRetryStrategy = GetRetryStrategyWithCondition(
+                response => !statusCodes.Contains(response.StatusCode),
+                finalRetryStrategy);
+            altered = true;
+        }
+    }
+
+    private void ApplyRetryUntilTestPassConditionIfAny(
+    string? testName,
+    ref RetryStrategy finalRetryStrategy,
+    ref string nameOfFinalStrategy,
+    ref bool altered)
+    {
+        if (!string.IsNullOrEmpty(testName))
+        {
+            nameOfFinalStrategy = $"{nameOfFinalStrategy}+{RetryingDirectives.RetryUntilTestPassDirectiveFullName}:{testName}";
+            finalRetryStrategy = GetRetryStrategyWithCondition(
+                response => !_testExecutor.ExecuteTestSync(testName, response),
+                finalRetryStrategy);
             altered = true;
         }
     }
@@ -120,12 +151,14 @@ private static ResiliencePipeline BuildPipeline(RetryStrategy retryStrategy)
             .AddRetry(retryStrategy)
             .Build();
 
-    private static RetryStrategy GetRetryStrategy(IReadOnlyList<HttpStatusCode> statusCodes, RetryStrategy baseRetryStrategy)
+    private static RetryStrategy GetRetryStrategyWithCondition(
+    Func<HttpResponseMessage, bool> shouldRetryPredicate,
+    RetryStrategy baseRetryStrategy)
     {
         var retryStrategyWithCondition = new RetryStrategy()
         {
             ShouldHandle = new PredicateBuilder<HttpResponseMessage>()
-                .HandleResult(response => !statusCodes.Contains(response.StatusCode))
+                .HandleResult(shouldRetryPredicate)
         };
 
         return MergeRetryStrategies(baseRetryStrategy, retryStrategyWithCondition);

src/TeaPie/Http/Retrying/RetryUntilTestPassDirectiveLineParser.cs

@@ -0,0 +1,26 @@
+using System.Text.RegularExpressions;
+using TeaPie.Http.Parsing;
+
+namespace TeaPie.Http.Retrying;
+
+internal partial class RetryUntilTestPassDirectiveLineParser : ILineParser
+{
+    public bool CanParse(string line, HttpParsingContext context)
+        => Regex.IsMatch(line, RetryingDirectives.RetryUntilTestPassDirectivePattern);
+
+    public void Parse(string line, HttpParsingContext context)
+    {
+        var match = Regex.Match(line, RetryingDirectives.RetryUntilTestPassDirectivePattern);
+        if (match.Success)
+        {
+            var testName = match.Groups[RetryingDirectives.RetryUntilTestPassDirectiveParameterName].Value;
+            context.RetryUntilTestPassTestName = testName.Trim();
+        }
+        else
+        {
+            throw new InvalidOperationException(
+                $"Unable to parse '{RetryingDirectives.RetryUntilTestPassDirectiveFullName}' " +
+                "if directive doesn't match the structure.");
+        }
+    }
+}