Merge pull request #80 from pavolbetak/master

Fix execution order of scheduled tests

Author: Burgyn

demo/.teapie/init.csx

@@ -125,6 +125,27 @@ tp.RegisterTestDirective(
     }
 );
 
+tp.RegisterTestDirective(
+    "JSON-HAS-ID-PROPERTY",
+    TestDirectivePatternBuilder
+        .Create("JSON-HAS-ID-PROPERTY")
+        .AddStringParameter("VariableName")
+        .Build(),
+    (_) => $"Response should be valid JSON with ID property.",
+    async (response, parameters) =>
+    {
+        try
+        {
+            dynamic body = await tp.Response.GetBodyAsExpandoAsync();
+            tp.SetVariable(parameters["VariableName"], body.Id);
+        }
+        catch (Exception ex)
+        {
+            Fail($"Response should be valid JSON with ID property. {ex.Message}");
+        }
+    }
+);
+
 // CUSTOM CLASS DEFINITIONS
 
 // Custom authentication provider definition

demo/Tests/002-Cars/001-Add-Car-req.http

@@ -11,6 +11,7 @@ Content-Type: application/json
 // Separate multiple requests in a single .http file with a line containing '3 hashtags' separator:
 
 # @name GetNewCarRequest
+## TEST-JSON-HAS-ID-PROPERTY: IdFromDirective
 // Access the body and headers of a named request/response using this syntax.
 // For JSON bodies, use JPath to retrieve properties. For XML, use XPath.
 GET {{ApiBaseUrl}}{{ApiCarsSection}}/{{AddCarRequest.request.body.$.Id}}

demo/Tests/002-Cars/001-Add-Car-test.csx

@@ -25,7 +25,9 @@ await tp.Test("Identifiers of added and retrieved cars should match.", async ()
     dynamic requestJson = await tp.Requests["AddCarRequest"].GetBodyAsExpandoAsync();
     dynamic responseJson = await tp.Responses["GetNewCarRequest"].GetBodyAsExpandoAsync();
 
+    var idFromDirective = tp.GetVariable<long>("IdFromDirective");
     Equal(requestJson.Id, responseJson.Id);
+    Equal(requestJson.Id, idFromDirective);
 
     // Each variable can have none or multiple tags ('cars', 'ids' in this case).
     tp.SetVariable("NewCarId", requestJson.Id, "cars", "ids");

docs/docs/directives.md

@@ -94,6 +94,15 @@ If no retry strategy is explicitly selected, the **default strategy from `Polly.
 | **Purpose** | Sets the maximum allowed delay between retries. |
 | **Parameters** | `hh:mm:ss.fff` – The maximum delay time before retrying a failed request. |
 
+#### `## RETRY-UNTIL-TEST-PASS` Directive
+
+|   |   |
+|----------------------|----------------|
+| **Syntax** | `## RETRY-UNTIL-TEST-PASS: <test-name>` |
+| **Example Usage** | `## RETRY-UNTIL-TEST-PASS: Identifier should be a positive integer` |
+| **Purpose** | Retries the request until the defined test passes. |
+| **Parameters** | `test-name` – The name of test defined in post-response .csx script. (tp.Test(`test-name`, () => )) |
+
 ### Testing Directives
 
 #### `## TEST-EXPECT-STATUS` Directive

src/Directory.Build.props

@@ -17,7 +17,7 @@
     </PropertyGroup>
 
     <PropertyGroup>
-        <Version>1.5.0</Version>
+        <Version>1.5.1</Version>
         <Authors>Matej Grochal</Authors>
         <Company>KROS a.s.</Company>
         <Copyright>Copyright © KROS a.s.</Copyright>

src/TeaPie/Testing/ExecuteScheduledTestsStep.cs

@@ -3,17 +3,17 @@
 
 namespace TeaPie.Testing;
 
-internal class ExecuteScheduledTestsStep(ITestScheduler scheduler, IRegistrator tester) : IPipelineStep
+internal class ExecuteScheduledTestsStep(ITestScheduler scheduler, ITester tester) : IPipelineStep
 {
     private readonly ITestScheduler _scheduler = scheduler;
-    private readonly IRegistrator _tester = tester;
+    private readonly ITester _tester = tester;
 
     public async Task Execute(ApplicationContext context, CancellationToken cancellationToken = default)
     {
         while (_scheduler.HasScheduledTest())
         {
             var test = _scheduler.Dequeue();
-            await _tester.Test(test.Name, test.Function);
+            await _tester.ExecuteOrSkipTest(test, test.TestCase);
 
             context.Logger.LogDebug("Scheduled test with name '{TestName}' was executed.", test.Name);
         }

tests/TeaPie.Tests/ApplicationPipelineShould.cs

@@ -1,5 +1,11 @@
-using NSubstitute;
+using FluentAssertions;
+using NSubstitute;
 using TeaPie.Pipelines;
+using TeaPie.Reporting;
+using TeaPie.Scripts;
+using TeaPie.StructureExploration;
+using TeaPie.TestCases;
+using TeaPie.Testing;
 using TeaPie.Tests.Pipelines;
 
 namespace TeaPie.Tests;
@@ -96,6 +102,55 @@ public async Task EnableAddingStepsDuringPipelineRun()
         await pipeline.Run(context);
     }
 
+    [Fact]
+    public async Task ExecuteScheduledTestsBeforeRegisteredTests()
+    {
+        var pipeline = new ApplicationPipeline();
+        var executionOrder = new List<string>();
+
+        var testCase = new TestCase(new InternalFile("test.http", "test.http", null!));
+        var testCaseContext = new TestCaseExecutionContext(testCase);
+
+        var accessor = Substitute.For<ITestCaseExecutionContextAccessor>();
+        accessor.Context.Returns(testCaseContext);
+
+        var reporter = Substitute.For<ITestResultsSummaryReporter>();
+
+        var tester = Substitute.For<ITester>();
+        tester.ExecuteOrSkipTest(Arg.Any<Test>(), Arg.Any<TestCase?>())
+            .Returns(callInfo =>
+            {
+                var test = callInfo.Arg<Test>();
+                executionOrder.Add(test.Name);
+                return Task.FromResult(test);
+            });
+
+        var scheduler = new TestScheduler();
+        var scheduledTest = CreateTest("scheduled-test", testCase);
+        scheduler.Schedule(scheduledTest);
+
+        var registeredTest = CreateTest("registered-test", testCase);
+        testCaseContext.RegisterTest(registeredTest);
+
+        var executeScheduledTestsStep = new ExecuteScheduledTestsStep(scheduler, tester);
+        var runScriptTestsStep = new RunScriptTestsStep(accessor, reporter, tester);
+
+        pipeline.AddSteps(executeScheduledTestsStep);
+        pipeline.AddSteps(runScriptTestsStep);
+
+        await pipeline.Run(CreateApplicationContext(string.Empty));
+
+        executionOrder.Should().HaveCount(2);
+        executionOrder[0].Should().Be("scheduled-test");
+        executionOrder[1].Should().Be("registered-test");
+    }
+
+    private static Test CreateTest(string name, TestCase testCase)
+    {
+        var result = new TestResult.NotRun { TestName = name, TestCasePath = "test.http" };
+        return new Test(name, false, () => Task.CompletedTask, result, testCase);
+    }
+
     private static ApplicationContext CreateApplicationContext(string path)
         => new ApplicationContextBuilder()
             .WithPath(path)