Added RETRY-UNTIL-TEST-PASS directive

demo/Tests/002-Cars/003-Check-Car-init.csx

@@ -0,0 +1,4 @@
+// Initialize counters for different retry scenarios
+tp.SetVariable("CarAvailabilityCheckCount", 0);
+tp.SetVariable("UnavailableCarCheckCount", 0);
+tp.SetVariable("CarCreationCheckCount", 0);

demo/Tests/002-Cars/003-Check-Car-req.http

@@ -0,0 +1,29 @@
+### Scenario 1: Check Car Availability (passes after 3 attempts)
+## RETRY-STRATEGY: Custom retry
+## RETRY-MAX-ATTEMPTS: 5
+## RETRY-UNTIL-TEST-PASS: Car should be available
+
+# @name CheckCarAvailabilityRequest
+GET {{ApiBaseUrl}}{{ApiCarsSection}}/1
+
+###
+
+### Scenario 2: Check Unavailable Car (never passes - exceeds max attempts)
+## RETRY-STRATEGY: Custom retry
+## RETRY-MAX-ATTEMPTS: 3
+## RETRY-UNTIL-TEST-PASS: Car with special features should exist
+
+# @name CheckUnavailableCarRequest
+GET {{ApiBaseUrl}}{{ApiCarsSection}}/999
+
+###
+
+### Scenario 3: Verify Car Creation (combined RETRY-UNTIL-STATUS + RETRY-UNTIL-TEST-PASS)
+## RETRY-STRATEGY: Custom retry
+## RETRY-MAX-ATTEMPTS: 4
+## RETRY-MAX-DELAY: 00:00:02
+## RETRY-UNTIL-STATUS: [200, 201]
+## RETRY-UNTIL-TEST-PASS: Newly created car should have correct brand
+
+# @name VerifyCarCreationRequest
+GET {{ApiBaseUrl}}{{ApiCarsSection}}/1

demo/Tests/002-Cars/003-Check-Car-test.csx

@@ -0,0 +1,57 @@
+// Test 1: Car becomes available after 3 attempts
+tp.Test("Car should be available", () =>
+{
+    var attempt = tp.GetVariable<int>("CarAvailabilityCheckCount");
+    attempt++;
+    tp.SetVariable("CarAvailabilityCheckCount", attempt);
+
+    var response = tp.Response;
+    var statusCode = response.StatusCode();
+
+    if (attempt >= 3)
+    {
+        Equal(200, statusCode);
+    }
+    else
+    {
+        Fail($"Car not available yet (attempt {attempt})");
+    }
+});
+
+// Test 2: Car never becomes available (will fail after max attempts)
+await tp.Test("Car with special features should exist", async () =>
+{
+    var attempt = tp.GetVariable<int>("UnavailableCarCheckCount");
+    attempt++;
+    tp.SetVariable("UnavailableCarCheckCount", attempt);
+
+    var response = tp.Response;
+    var statusCode = response.StatusCode();
+
+    // This will always fail - demonstrating max attempts behavior
+    Equal(200, statusCode);
+
+    dynamic carData = await response.GetBodyAsExpandoAsync();
+    Equal("SpecialFeatures", carData.Brand);
+});
+
+// Test 3: Combined scenario - verify newly created car has correct brand
+await tp.Test("Newly created car should have correct brand", async () =>
+{
+    var attempt = tp.GetVariable<int>("CarCreationCheckCount");
+    attempt++;
+    tp.SetVariable("CarCreationCheckCount", attempt);
+
+    var response = tp.Response;
+    var statusCode = response.StatusCode();
+
+    if (attempt >= 2 && (statusCode == 200 || statusCode == 201))
+    {
+        dynamic carData = await response.GetBodyAsExpandoAsync();
+        Equal(200, statusCode);
+    }
+    else
+    {
+        Fail($"Car not ready yet (attempt {attempt})");
+    }
+});

demo/Tests/003-Car-Rentals/001-Rent-Car-test.csx

@@ -1,6 +1,4 @@
-// Referencing multiple scripts is also allowed.
-#load "$teapie/ClearVariables.csx"
-#load "$teapie/Definitions/Car.csx"
+#load "$teapie/Definitions/Car.csx"
 #load "$teapie/Definitions/CarRent.csx"
 
 // Sometimes, when writing tests, it may be useful to skip certain tests
@@ -36,6 +34,4 @@ await tp.Test($"Rented car should be '{car}'.", async () =>
     Equal(car.Year, retrievedCar.Year);
 });
 
-ClearVariables();
-
 tp.Logger.LogInformation("End of demo collection testing.");

docs/docs/retrying.md

@@ -26,11 +26,12 @@ Within `.http` files, you can apply a retry strategy using **retry directives**.
 ```http
 # @name GetCarRequest
 # Applying a named retry strategy and overriding specific properties.
-## RETRY-STRATEGY: Default retry   # Uses already registered "Default retry" strategy
-## RETRY-MAX-ATTEMPTS: 5           # Overrides max retry attempts
-## RETRY-BACKOFF-TYPE: Linear      # Changes backoff strategy for this request
-## RETRY-MAX-DELAY: 00:00:03       # Limits the maximum delay between retries
-## RETRY-UNTIL-STATUS: [200, 201]  # Adds condition - retry until one of given status codes is received
+## RETRY-STRATEGY: Default retry      # Uses already registered "Default retry" strategy
+## RETRY-MAX-ATTEMPTS: 5              # Overrides max retry attempts
+## RETRY-BACKOFF-TYPE: Linear         # Changes backoff strategy for this request
+## RETRY-MAX-DELAY: 00:00:03          # Limits the maximum delay between retries
+## RETRY-UNTIL-STATUS: [200, 201]     # Adds condition - retry until one of given status codes is received
+## RETRY-UNTIL-TEST-PASS: Test name   # Retries until specified test (Test name) passes
 GET {{ApiBaseUrl}}{{ApiCarsSection}}/{{RentCarRequest.request.body.$.CarId}}
 ```
 

src/Directory.Build.props

@@ -17,7 +17,7 @@
     </PropertyGroup>
 
     <PropertyGroup>
-        <Version>1.4.0</Version>
+        <Version>1.5.0</Version>
         <Authors>Matej Grochal</Authors>
         <Company>KROS a.s.</Company>
         <Copyright>Copyright © KROS a.s.</Copyright>

src/TeaPie/ApplicationBuilder.cs

@@ -165,7 +165,7 @@ private static TeaPie CreateUserContext(IServiceProvider provider, ApplicationCo
             provider.GetRequiredService<IVariables>(),
             provider.GetRequiredService<IFunctions>(),
             provider.GetRequiredService<ILogger<TeaPie>>(),
-            provider.GetRequiredService<ITester>(),
+            provider.GetRequiredService<IRegistrator>(),
             provider.GetRequiredService<ICurrentTestCaseExecutionContextAccessor>(),
             provider.GetRequiredService<ITestResultsSummaryReporter>(),
             provider.GetRequiredService<IRetryStrategyRegistry>(),

src/TeaPie/Http/Parsing/DirectivesLineParser.cs

@@ -10,6 +10,7 @@ internal class DirectivesLineParser : ILineParser
         new TestDirectivesLineParser(),
         new RetryStrategyDirectiveLineParser(),
         new RetryUntilStatusCodesLineParser(),
+        new RetryUntilTestPassDirectiveLineParser(),
         new RetryExplicitPropertiesDirectiveLineParser()
     ];
 

src/TeaPie/Http/Parsing/HttpParsingContext.cs

@@ -26,6 +26,7 @@ internal class HttpParsingContext(HttpRequestHeaders defaultHeaders)
     public string RetryStrategyName { get; set; } = string.Empty;
     public IReadOnlyList<HttpStatusCode> RetryUntilStatusCodes { get; set; } = [];
     public RetryStrategyOptions<HttpResponseMessage>? ExplicitRetryStrategy { get; set; }
+    public string RetryUntilTestPassTestName { get; set; } = string.Empty;
 
     public string AuthProviderName { get; set; } = string.Empty;
 

src/TeaPie/Http/Parsing/HttpRequestParser.cs

@@ -131,7 +131,8 @@ private void UpdateResiliencePipeline(RequestExecutionContext requestExecutionCo
             _resiliencePipelineProvider.GetResiliencePipeline(
                 parsingContext.RetryStrategyName,
                 parsingContext.ExplicitRetryStrategy,
-                parsingContext.RetryUntilStatusCodes);
+                parsingContext.RetryUntilStatusCodes,
+                parsingContext.RetryUntilTestPassTestName);
 
     private static void CreateMessageContent(HttpParsingContext context, HttpRequestMessage requestMessage)
     {

src/TeaPie/Http/Retrying/ResiliencePipelineProvider.cs

@@ -3,6 +3,7 @@
 using Polly.Retry;
 using System.Net;
 using System.Text;
+using TeaPie.Testing;
 using ResiliencePipeline = Polly.ResiliencePipeline<System.Net.Http.HttpResponseMessage>;
 using RetryStrategy = Polly.Retry.RetryStrategyOptions<System.Net.Http.HttpResponseMessage>;
 
@@ -11,14 +12,21 @@ namespace TeaPie.Http.Retrying;
 internal interface IResiliencePipelineProvider
 {
     ResiliencePipeline GetResiliencePipeline(
-        string nameOfBaseStrategy, RetryStrategy? explicitlyOverridenStrategy, IReadOnlyList<HttpStatusCode> statusCodes);
+        string nameOfBaseStrategy,
+        RetryStrategy? explicitlyOverridenStrategy,
+        IReadOnlyList<HttpStatusCode> statusCodes,
+        string? retryUntilTestPassTestName = null);
 }
 
-internal class ResiliencePipelineProvider(IRetryStrategyRegistry registry, ILogger<ResiliencePipelineProvider> logger)
+internal class ResiliencePipelineProvider(
+    IRetryStrategyRegistry registry,
+    ILogger<ResiliencePipelineProvider> logger,
+    ITester testExecutor)
     : IResiliencePipelineProvider
 {
     private readonly IRetryStrategyRegistry _retryStrategyRegistry = registry;
     private readonly ILogger<ResiliencePipelineProvider> _logger = logger;
+    private readonly ITester _testExecutor = testExecutor;
 
     public static readonly ResiliencePipeline DefaultResiliencePipeline =
         ResiliencePipeline.Empty;
@@ -27,14 +35,19 @@ internal class ResiliencePipelineProvider(IRetryStrategyRegistry registry, ILogg
         new() { { string.Empty, DefaultResiliencePipeline } };
 
     public ResiliencePipeline<HttpResponseMessage> GetResiliencePipeline(
-        string nameOfBaseRetryStrategy, RetryStrategy? overridRetryStrategy, IReadOnlyList<HttpStatusCode> statusCodes)
+        string nameOfBaseRetryStrategy,
+        RetryStrategy? overridRetryStrategy,
+        IReadOnlyList<HttpStatusCode> statusCodes,
+        string? retryUntilTestPassTestName = null)
     {
         var isBaseRetryStrategyAltered = false;
         CheckAndResolveBaseRetryStrategy(nameOfBaseRetryStrategy, out var finalRetryStrategy, out var nameOfFinalStrategy);
         ApplyExplicitOverridesIfAny(
             overridRetryStrategy, ref finalRetryStrategy, ref nameOfFinalStrategy, ref isBaseRetryStrategyAltered);
         ApplyRetryUntilStatusCodesConditionIfAny(
             statusCodes, ref finalRetryStrategy, ref nameOfFinalStrategy, ref isBaseRetryStrategyAltered);
+        ApplyRetryUntilTestPassConditionIfAny(
+           retryUntilTestPassTestName, ref finalRetryStrategy, ref nameOfFinalStrategy, ref isBaseRetryStrategyAltered);
 
         LogUsageOfRetryStrategy(nameOfFinalStrategy, finalRetryStrategy, isBaseRetryStrategyAltered);
 
@@ -50,7 +63,25 @@ private void ApplyRetryUntilStatusCodesConditionIfAny(
         if (statusCodes.Any())
         {
             nameOfFinalStrategy = GetRetryUntilStatusCodesStrategyName(statusCodes, nameOfFinalStrategy);
-            finalRetryStrategy = GetRetryStrategy(statusCodes, finalRetryStrategy);
+            finalRetryStrategy = GetRetryStrategyWithCondition(
+                response => !statusCodes.Contains(response.StatusCode),
+                finalRetryStrategy);
+            altered = true;
+        }
+    }
+
+    private void ApplyRetryUntilTestPassConditionIfAny(
+    string? testName,
+    ref RetryStrategy finalRetryStrategy,
+    ref string nameOfFinalStrategy,
+    ref bool altered)
+    {
+        if (!string.IsNullOrEmpty(testName))
+        {
+            nameOfFinalStrategy = $"{nameOfFinalStrategy}+{RetryingDirectives.RetryUntilTestPassDirectiveFullName}:{testName}";
+            finalRetryStrategy = GetRetryStrategyWithCondition(
+                response => !_testExecutor.ExecuteTestSync(testName, response),
+                finalRetryStrategy);
             altered = true;
         }
     }
@@ -120,12 +151,14 @@ private static ResiliencePipeline BuildPipeline(RetryStrategy retryStrategy)
             .AddRetry(retryStrategy)
             .Build();
 
-    private static RetryStrategy GetRetryStrategy(IReadOnlyList<HttpStatusCode> statusCodes, RetryStrategy baseRetryStrategy)
+    private static RetryStrategy GetRetryStrategyWithCondition(
+    Func<HttpResponseMessage, bool> shouldRetryPredicate,
+    RetryStrategy baseRetryStrategy)
     {
         var retryStrategyWithCondition = new RetryStrategy()
         {
             ShouldHandle = new PredicateBuilder<HttpResponseMessage>()
-                .HandleResult(response => !statusCodes.Contains(response.StatusCode))
+                .HandleResult(shouldRetryPredicate)
         };
 
         return MergeRetryStrategies(baseRetryStrategy, retryStrategyWithCondition);

src/TeaPie/Http/Retrying/RetryUntilTestPassDirectiveLineParser.cs

@@ -0,0 +1,26 @@
+using System.Text.RegularExpressions;
+using TeaPie.Http.Parsing;
+
+namespace TeaPie.Http.Retrying;
+
+internal partial class RetryUntilTestPassDirectiveLineParser : ILineParser
+{
+    public bool CanParse(string line, HttpParsingContext context)
+        => Regex.IsMatch(line, RetryingDirectives.RetryUntilTestPassDirectivePattern);
+
+    public void Parse(string line, HttpParsingContext context)
+    {
+        var match = Regex.Match(line, RetryingDirectives.RetryUntilTestPassDirectivePattern);
+        if (match.Success)
+        {
+            var testName = match.Groups[RetryingDirectives.RetryUntilTestPassDirectiveParameterName].Value;
+            context.RetryUntilTestPassTestName = testName.Trim();
+        }
+        else
+        {
+            throw new InvalidOperationException(
+                $"Unable to parse '{RetryingDirectives.RetryUntilTestPassDirectiveFullName}' " +
+                "if directive doesn't match the structure.");
+        }
+    }
+}

src/TeaPie/Http/Retrying/RetryingDirectives.cs

@@ -50,4 +50,13 @@ internal static class RetryingDirectives
             .WithPrefix(RetryDirectivePrefix)
             .AddTimeOnlyParameter(RetryMaxDelayDirectiveParameterName)
             .Build();
+
+    public const string RetryUntilTestPassDirectiveName = "UNTIL-TEST-PASS";
+    public const string RetryUntilTestPassDirectiveFullName = RetryDirectivePrefix + RetryUntilTestPassDirectiveName;
+    public const string RetryUntilTestPassDirectiveParameterName = "TestName";
+    public static readonly string RetryUntilTestPassDirectivePattern =
+        HttpDirectivePatternBuilder.Create(RetryUntilTestPassDirectiveName)
+            .WithPrefix(RetryDirectivePrefix)
+            .AddStringParameter(RetryUntilTestPassDirectiveParameterName)
+            .Build();
 }

src/TeaPie/Scripts/RunScriptTestsStep.cs

@@ -0,0 +1,27 @@
+using TeaPie.Pipelines;
+using TeaPie.Reporting;
+using TeaPie.TestCases;
+using TeaPie.Testing;
+
+namespace TeaPie.Scripts;
+
+internal class RunScriptTestsStep(
+    ITestCaseExecutionContextAccessor accessor,
+    ITestResultsSummaryReporter resultsSummaryReporter,
+    ITester testExecutor) : IPipelineStep
+{
+    private readonly ITestCaseExecutionContextAccessor _accessor = accessor;
+    private readonly ITestResultsSummaryReporter _resultsSummaryReporter = resultsSummaryReporter;
+    private readonly ITester _testExecutor = testExecutor;
+
+    public async Task Execute(ApplicationContext context, CancellationToken cancellationToken = default)
+    {
+        _resultsSummaryReporter.Initialize();
+
+        var testCaseExecutionContext = _accessor.Context!;
+        foreach (var test in testCaseExecutionContext.GetTests())
+        {
+            await _testExecutor.ExecuteOrSkipTest(test, testCaseExecutionContext.TestCase);
+        }
+    }
+}

src/TeaPie/Scripts/ScriptStepsFactory.cs

@@ -1,5 +1,6 @@
 using Microsoft.Extensions.DependencyInjection;
 using TeaPie.Pipelines;
+using TeaPie.TestCases;
 
 namespace TeaPie.Scripts;
 
@@ -15,6 +16,11 @@ public static IEnumerable<IPipelineStep> CreateStepsForScriptPreProcessAndExecut
         ScriptExecutionContext scriptExecutionContext)
         => CreateSteps(serviceProvider, scriptExecutionContext, GetStepsForScriptPreProcessAndExecution);
 
+    public static IEnumerable<IPipelineStep> CreateStepsForScriptExecution(
+        IServiceProvider serviceProvider,
+        TestCaseExecutionContext testCaseExecutionContext)
+        => CreateSteps(serviceProvider, testCaseExecutionContext, GetStepsForScriptsExecution);
+
     private static IPipelineStep[] CreateSteps(
       IServiceProvider serviceProvider,
       ScriptExecutionContext scriptExecutionContext,
@@ -29,6 +35,20 @@ private static IPipelineStep[] CreateSteps(
         return pipelines(provider);
     }
 
+    private static IPipelineStep[] CreateSteps(
+      IServiceProvider serviceProvider,
+      TestCaseExecutionContext testCaseExecutionContext,
+      Func<IServiceProvider, IPipelineStep[]> pipelines)
+    {
+        using var scope = serviceProvider.CreateScope();
+        var provider = scope.ServiceProvider;
+
+        var accessor = provider.GetRequiredService<ITestCaseExecutionContextAccessor>();
+        accessor.Context = testCaseExecutionContext;
+
+        return pipelines(provider);
+    }
+
     private static IPipelineStep[] GetStepsForScriptPreProcess(IServiceProvider provider)
         => [provider.GetStep<ReadScriptFileStep>(),
             provider.GetStep<PreProcessScriptStep>(),
@@ -42,4 +62,7 @@ private static IPipelineStep[] GetStepsForScriptPreProcessAndExecution(IServiceP
             provider.GetStep<CompileScriptStep>(),
             provider.GetStep<ExecuteScriptStep>(),
             provider.GetStep<DisposeScriptStep>()];
+
+    private static IPipelineStep[] GetStepsForScriptsExecution(IServiceProvider provider)
+    => [provider.GetStep<RunScriptTestsStep>()];
 }