|
5 | 5 | using LantanaGroup.Link.DataAcquisition.Domain.Application.Models.Api.Configuration; |
6 | 6 | using LantanaGroup.Link.DataAcquisition.Domain.Application.Models.Api.QueryLog; |
7 | 7 | using LantanaGroup.Link.DataAcquisition.Domain.Application.Models.Api.Requests; |
| 8 | +using LantanaGroup.Link.DataAcquisition.Domain.Application.Models.Exceptions; |
8 | 9 | using LantanaGroup.Link.DataAcquisition.Domain.Application.Models.Kafka; |
9 | 10 | using LantanaGroup.Link.DataAcquisition.Domain.Application.Queries; |
10 | 11 | using LantanaGroup.Link.DataAcquisition.Domain.Application.Services; |
|
21 | 22 | using Medallion.Threading; |
22 | 23 | using Microsoft.Extensions.Logging; |
23 | 24 | using Moq; |
24 | | -using Serilog; |
25 | | -using System.Linq.Expressions; |
26 | 25 | using RequestStatus = LantanaGroup.Link.DataAcquisition.Domain.Infrastructure.Models.Enums.RequestStatus; |
27 | 26 | using ResourceType = Hl7.Fhir.Model.ResourceType; |
28 | 27 | using Task = System.Threading.Tasks.Task; |
@@ -719,4 +718,223 @@ public async Task ExecuteLogRequest_ShouldAccumulateResourceIds_FromMultipleFhir |
719 | 718 | cancellationToken), |
720 | 719 | Times.Once); |
721 | 720 | } |
| 721 | + |
| 722 | + [Fact] |
| 723 | + public async Task ExecuteLogRequest_Handles429WithSecondsDelay_ReschedulesLogWithDelay() |
| 724 | + { |
| 725 | + // Arrange |
| 726 | + var request = new AcquisitionRequest(1, "facility-1"); |
| 727 | + var cancellationToken = CancellationToken.None; |
| 728 | + |
| 729 | + var log = new DataAcquisitionLogModel |
| 730 | + { |
| 731 | + Id = 1, |
| 732 | + FacilityId = "facility-1", |
| 733 | + PatientId = "Patient/123", |
| 734 | + Status = RequestStatus.Ready, |
| 735 | + CorrelationId = "corr-1", |
| 736 | + FhirQuery = new List<FhirQueryModel> |
| 737 | + { |
| 738 | + new FhirQueryModel |
| 739 | + { |
| 740 | + QueryType = FhirQueryType.Read, |
| 741 | + ResourceTypes = new List<ResourceType> { ResourceType.Patient } |
| 742 | + } |
| 743 | + }, |
| 744 | + ScheduledReport = new ScheduledReport() |
| 745 | + }; |
| 746 | + |
| 747 | + _mockLogQueries |
| 748 | + .Setup(q => q.GetAsync(1, cancellationToken)) |
| 749 | + .ReturnsAsync(log); |
| 750 | + |
| 751 | + _mockFhirQueryQueries |
| 752 | + .Setup(q => q.GetByFacilityIdAsync("facility-1", cancellationToken)) |
| 753 | + .ReturnsAsync(new FhirQueryConfigurationModel { FacilityId = "facility-1" }); |
| 754 | + |
| 755 | + // Simulate 429 with Retry-After: 30 seconds |
| 756 | + _mockFhirApiService |
| 757 | + .Setup(x => x.ExecuteRead( |
| 758 | + It.IsAny<DataAcquisitionLogModel>(), |
| 759 | + It.IsAny<FhirQueryModel>(), |
| 760 | + ResourceType.Patient, |
| 761 | + It.IsAny<FhirQueryConfigurationModel>(), |
| 762 | + cancellationToken)) |
| 763 | + .ThrowsAsync(new TooManyRequestsException("Rate limited", TimeSpan.FromSeconds(30))); |
| 764 | + |
| 765 | + // Act |
| 766 | + await _service.ExecuteLogRequest(request, cancellationToken); |
| 767 | + |
| 768 | + // Assert: Log updated with delay (ExecutionDate ~30s from now), Pending status, retry incremented |
| 769 | + _mockLogManager.Verify(m => m.UpdateAsync( |
| 770 | + It.Is<UpdateDataAcquisitionLogModel>(u => |
| 771 | + u.Status == RequestStatus.Pending && |
| 772 | + u.RetryAttempts == 0 && |
| 773 | + u.ExecutionDate >= DateTime.UtcNow.AddSeconds(20) && // Widened range to account for execution time |
| 774 | + u.ExecutionDate <= DateTime.UtcNow.AddSeconds(40) && |
| 775 | + u.Notes.Any(n => n.Contains("Throttled (429): Retrying after") && n.Contains("30")) // Check for specific delay in note |
| 776 | + ), |
| 777 | + cancellationToken), |
| 778 | + Times.Exactly(1)); // Exactly once for the reschedule (the Processing update is separate) |
| 779 | + } |
| 780 | + |
| 781 | + [Fact] |
| 782 | + public async Task ExecuteLogRequest_Handles429WithDateDelay_ReschedulesLogWithCalculatedDelay() |
| 783 | + { |
| 784 | + // Arrange |
| 785 | + var request = new AcquisitionRequest(1, "facility-1"); |
| 786 | + var cancellationToken = CancellationToken.None; |
| 787 | + |
| 788 | + var log = new DataAcquisitionLogModel |
| 789 | + { |
| 790 | + Id = 1, |
| 791 | + FacilityId = "facility-1", |
| 792 | + PatientId = "Patient/123", |
| 793 | + Status = RequestStatus.Ready, |
| 794 | + CorrelationId = "corr-1", |
| 795 | + FhirQuery = new List<FhirQueryModel> |
| 796 | + { |
| 797 | + new FhirQueryModel |
| 798 | + { |
| 799 | + QueryType = FhirQueryType.Read, |
| 800 | + ResourceTypes = new List<ResourceType> { ResourceType.Patient } |
| 801 | + } |
| 802 | + }, |
| 803 | + ScheduledReport = new ScheduledReport() |
| 804 | + }; |
| 805 | + |
| 806 | + _mockLogQueries |
| 807 | + .Setup(q => q.GetAsync(1, cancellationToken)) |
| 808 | + .ReturnsAsync(log); |
| 809 | + |
| 810 | + _mockFhirQueryQueries |
| 811 | + .Setup(q => q.GetByFacilityIdAsync("facility-1", cancellationToken)) |
| 812 | + .ReturnsAsync(new FhirQueryConfigurationModel { FacilityId = "facility-1" }); |
| 813 | + |
| 814 | + // Simulate 429 with Retry-After as a future date (e.g., 2 minutes from now) |
| 815 | + var futureDate = DateTimeOffset.UtcNow.AddMinutes(2); |
| 816 | + var expectedDelay = TimeSpan.FromMinutes(2); |
| 817 | + _mockFhirApiService |
| 818 | + .Setup(x => x.ExecuteRead( |
| 819 | + It.IsAny<DataAcquisitionLogModel>(), |
| 820 | + It.IsAny<FhirQueryModel>(), |
| 821 | + ResourceType.Patient, |
| 822 | + It.IsAny<FhirQueryConfigurationModel>(), |
| 823 | + cancellationToken)) |
| 824 | + .ThrowsAsync(new TooManyRequestsException("Rate limited", expectedDelay)); |
| 825 | + |
| 826 | + // Act |
| 827 | + await _service.ExecuteLogRequest(request, cancellationToken); |
| 828 | + |
| 829 | + // Assert: Log rescheduled ~2min from now |
| 830 | + _mockLogManager.Verify(m => m.UpdateAsync( |
| 831 | + It.Is<UpdateDataAcquisitionLogModel>(u => |
| 832 | + u.Status == RequestStatus.Pending && |
| 833 | + u.RetryAttempts == 0 && |
| 834 | + u.ExecutionDate >= DateTime.UtcNow.AddMinutes(1.9) && // Approximate |
| 835 | + u.ExecutionDate <= DateTime.UtcNow.AddMinutes(2.1) && |
| 836 | + u.Notes.Any(n => n.Contains("Throttled (429): Retrying after")) |
| 837 | + ), |
| 838 | + cancellationToken), |
| 839 | + Times.AtLeastOnce); |
| 840 | + } |
| 841 | + |
| 842 | + [Fact] |
| 843 | + public async Task ExecuteLogRequest_Handles429WithInvalidNegativeHeader_UsesParsedDefaultDelay() |
| 844 | + { |
| 845 | + // Arrange |
| 846 | + var request = new AcquisitionRequest(1, "facility-1"); |
| 847 | + var cancellationToken = CancellationToken.None; |
| 848 | + |
| 849 | + var log = new DataAcquisitionLogModel |
| 850 | + { |
| 851 | + Id = 1, |
| 852 | + FacilityId = "facility-1", |
| 853 | + PatientId = "Patient/123", |
| 854 | + Status = RequestStatus.Ready, |
| 855 | + CorrelationId = "corr-1", |
| 856 | + FhirQuery = new List<FhirQueryModel> |
| 857 | + { |
| 858 | + new FhirQueryModel |
| 859 | + { |
| 860 | + QueryType = FhirQueryType.Read, |
| 861 | + ResourceTypes = new List<ResourceType> { ResourceType.Patient } |
| 862 | + } |
| 863 | + }, |
| 864 | + ScheduledReport = new ScheduledReport() |
| 865 | + }; |
| 866 | + |
| 867 | + _mockLogQueries |
| 868 | + .Setup(q => q.GetAsync(1, cancellationToken)) |
| 869 | + .ReturnsAsync(log); |
| 870 | + |
| 871 | + _mockFhirQueryQueries |
| 872 | + .Setup(q => q.GetByFacilityIdAsync("facility-1", cancellationToken)) |
| 873 | + .ReturnsAsync(new FhirQueryConfigurationModel { FacilityId = "facility-1" }); |
| 874 | + |
| 875 | + // Simulate 429 with negative/invalid Retry-After (parser will default to 60s) |
| 876 | + _mockFhirApiService |
| 877 | + .Setup(x => x.ExecuteRead( |
| 878 | + It.IsAny<DataAcquisitionLogModel>(), |
| 879 | + It.IsAny<FhirQueryModel>(), |
| 880 | + ResourceType.Patient, |
| 881 | + It.IsAny<FhirQueryConfigurationModel>(), |
| 882 | + cancellationToken)) |
| 883 | + .ThrowsAsync(new TooManyRequestsException("Rate limited", TimeSpan.FromSeconds(60))); // Mimic parsed default |
| 884 | + |
| 885 | + // Act |
| 886 | + await _service.ExecuteLogRequest(request, cancellationToken); |
| 887 | + |
| 888 | + // Assert: Log rescheduled ~60s from now, Pending, retry=1, note reflects default delay |
| 889 | + _mockLogManager.Verify(m => m.UpdateAsync( |
| 890 | + It.Is<UpdateDataAcquisitionLogModel>(u => |
| 891 | + u.Status == RequestStatus.Pending && |
| 892 | + u.RetryAttempts == 0 && |
| 893 | + u.ExecutionDate >= DateTime.UtcNow.AddSeconds(55) && // Approx for 60s, allowing execution variance |
| 894 | + u.ExecutionDate <= DateTime.UtcNow.AddSeconds(65) && |
| 895 | + u.Notes.Any(n => n.Contains("Throttled (429): Retrying after") && n.Contains("60")) |
| 896 | + ), |
| 897 | + cancellationToken), |
| 898 | + Times.Exactly(1)); // Once for reschedule (Processing update separate) |
| 899 | + } |
| 900 | + |
| 901 | + [Fact] |
| 902 | + public async Task GetNextEligibleBatchForFacility_OrdersByPriorityDescending_ThenExecutionDateAscending_IncludesAllFailedRegardlessOfRetries() |
| 903 | + { |
| 904 | + // Arrange |
| 905 | + var facilityId = "facility-1"; |
| 906 | + long? lastId = null; |
| 907 | + int batchSize = 4; |
| 908 | + var cancellationToken = CancellationToken.None; |
| 909 | + |
| 910 | + // Simulate logs with varying priorities, dates, and retry attempts (including exceeded max) |
| 911 | + var logs = new List<DataAcquisitionLogModel> |
| 912 | + { |
| 913 | + new() { Id = 1, Priority = AcquisitionPriority.Normal, ExecutionDate = DateTime.UtcNow.AddMinutes(-5), Status = RequestStatus.Pending }, |
| 914 | + new() { Id = 2, Priority = AcquisitionPriority.High, ExecutionDate = DateTime.UtcNow.AddMinutes(-10), Status = RequestStatus.Pending }, |
| 915 | + new() { Id = 3, Priority = AcquisitionPriority.High, ExecutionDate = DateTime.UtcNow.AddMinutes(-1), Status = RequestStatus.Pending }, |
| 916 | + new() { Id = 4, Priority = AcquisitionPriority.Normal, ExecutionDate = DateTime.UtcNow.AddMinutes(-2), Status = RequestStatus.Failed, RetryAttempts = 2 }, // Retryable (below max) |
| 917 | + new() { Id = 5, Priority = AcquisitionPriority.Critical, ExecutionDate = DateTime.UtcNow.AddMinutes(-3), Status = RequestStatus.Failed, RetryAttempts = 6 } // Exceeded max retries, but still included |
| 918 | + }; |
| 919 | + |
| 920 | + _mockLogQueries |
| 921 | + .Setup(q => q.GetNextEligibleBatchForFacility(facilityId, lastId, batchSize, cancellationToken)) |
| 922 | + .ReturnsAsync(logs |
| 923 | + .Where(l => l.Status == RequestStatus.Pending || l.Status == RequestStatus.Failed) |
| 924 | + .OrderBy(l => l.Priority) // Ascending: Critical (0), High (1), Normal (2) |
| 925 | + .ThenBy(l => l.ExecutionDate) |
| 926 | + .ThenBy(l => l.Id) |
| 927 | + .Take(batchSize) |
| 928 | + .ToList()); |
| 929 | + |
| 930 | + // Act |
| 931 | + var result = await _mockLogQueries.Object.GetNextEligibleBatchForFacility(facilityId, lastId, batchSize, cancellationToken); |
| 932 | + |
| 933 | + // Assert: All Pending and Failed included, ordered correctly (Critical/High first, then by date; includes exceeded retries) |
| 934 | + Assert.Equal(4, result.Count); // Batch size (original 5 matching, take 4) |
| 935 | + Assert.Equal(5, result[0].Id); // Critical first (even if Failed and exceeded retries) |
| 936 | + Assert.Equal(2, result[1].Id); // High, oldest ExecutionDate |
| 937 | + Assert.Equal(3, result[2].Id); // High, newer ExecutionDate |
| 938 | + Assert.Equal(1, result[3].Id); // Normal Pending (next after highs) |
| 939 | + } |
722 | 940 | } |
0 commit comments