implementing error status (#1115)

* implementing error status
fixing #1100

* reverting testing error

* adding copilot suggestions

Author: gaelcolas

src/powershell/assets/ReportTemplate.html

@@ -89,8 +89,8 @@ function Add-ZtTestResultDetail {
 		[string[]] $Tag,
 
 		# Optional. Custom status to return instead of the default status.
-		[Parameter(Mandatory = $false)]
-		[ValidateSet('Investigate')]
+		[Parameter()]
+		[ValidateSet('Investigate','Error')]
 		[string] $CustomStatus,
 
 		[ValidateSet('Graph', 'Azure', 'AipService', 'ExchangeOnline', 'SecurityCompliance', 'SharePointOnline')]
@@ -133,6 +133,12 @@ function Add-ZtTestResultDetail {
 		}
 	}
 
+	if ($CustomStatus -eq 'Error') {
+		if ([string]::IsNullOrEmpty($Result)) {
+			$Result = 'An error occurred while running this test.'
+		}
+	}
+
 	if (-not $Description) {
 		# Check if a markdown file exists for the cmdlet and parse the content
 		$markdownPath = Join-Path -Path $script:ModuleRoot -ChildPath "tests/Test-Assessment.$actualTestId.md"
@@ -183,10 +189,19 @@ function Add-ZtTestResultDetail {
 		$UserImpact = $testMeta.UserImpact
 	}
 
+	$getZtTestStatusParams = @{
+		Status         = $Status
+		SkippedBecause = $SkippedBecause
+	}
+
+	if ($CustomStatus) {
+		$getZtTestStatusParams.CustomStatus = $CustomStatus
+	}
+
 	$testInfo = @{
 		TestId                 = $actualTestId
 		TestTitle              = $docsTitle
-		TestStatus             = Get-ZtTestStatus -Status $Status -SkippedBecause $SkippedBecause -CustomStatus $CustomStatus
+		TestStatus             = Get-ZtTestStatus @getZtTestStatusParams
 		TestCategory           = $category
 		TestTags               = $Tag
 		TestAppliesTo          = $AppliesTo

src/powershell/private/core/Add-ZtTestResultDetail.ps1

@@ -2,8 +2,9 @@
     Returns the status of the test as a string.
 #>
 function Get-ZtTestStatus {
+    [OutputType([string])]
     [CmdletBinding()]
-    param(
+    param (
         # The status of the test.
         [Parameter(Mandatory = $true)]
         [bool] $Status,
@@ -12,14 +13,14 @@ function Get-ZtTestStatus {
         [string] $SkippedBecause,
 
         # Optional. Custom status to return instead of the default status.
+        [ValidateSet('Investigate','Error')]
         [string] $CustomStatus
     )
 
     if ($CustomStatus) {
         return $CustomStatus
     }
-
-    if ($Status) {
+    elseif ($Status) {
         return "Passed"
     }
     else {
@@ -30,9 +31,9 @@ function Get-ZtTestStatus {
             if ($SkippedBecause -eq "UnderConstruction") {
                 return "Planned"
             }
-            return "Skipped"
+            else {
+                return "Skipped"
+            }
         }
     }
-
-    return $Status
 }

src/powershell/private/core/Get-ZtTestStatus.ps1

@@ -42,6 +42,7 @@ function Test-ZtContext {
 			$message += " Add the missing 'Application' permissions in the Microsoft Entra portal and grant consent. You will also need to Disconnect-Graph to refresh the permissions."
 			$message += " Click here to open the 'API Permissions' blade for this app: $urlTemplate"
 		}
+
 		$validContext = $false
 	}
 

src/powershell/private/core/Test-ZtContext.ps1

@@ -131,7 +131,8 @@
 					return
 				}
 
-				Start-Sleep -Seconds 5
+				# Wait a bit before checking again
+				Start-Sleep -Seconds 1
 			}
 		}
 		#endregion Wait for Dependencies

src/powershell/private/export/Invoke-ZtTenantDataExport.ps1

@@ -38,7 +38,7 @@ function Write-ZtExportLog {
 			[void][System.IO.Directory]::CreateDirectory($LogsPath)
 
 			$name = $Result.Name
-			$status = if ($Result.Success) { 'Done' } else { 'Failed' }
+			$status = if ($Result.Success) { 'Done' } elseif ($Result.Error) { 'Error' } else { 'Failed' }
 			$duration = if ($null -ne $Result.Duration) { $Result.Duration.ToString('hh\:mm\:ss\.fff') } else { 'N/A' }
 			$startTime = if ($Result.Start) { $Result.Start.ToString('yyyy-MM-dd HH:mm:ss.fff') } else { 'N/A' }
 			$endTime = if ($Result.End) { $Result.End.ToString('yyyy-MM-dd HH:mm:ss.fff') } else { 'N/A' }

src/powershell/private/export/Write-ZtExportLog.ps1

@@ -56,7 +56,7 @@ function Write-ZtExportProgress {
 		$LogsPath,
 
 		[Parameter(Mandatory = $true)]
-		[ValidateSet('Started', 'Waiting', 'InProgress', 'Completed', 'Failed', 'Status')]
+		[ValidateSet('Started', 'Waiting', 'InProgress', 'Completed', 'Failed', 'Status', 'Error')]
 		[string]
 		$Action,
 
@@ -81,22 +81,26 @@ function Write-ZtExportProgress {
 			if ($null -ne $Duration) {
 				$line += "  $($Duration.ToString('hh\:mm\:ss\.fff'))"
 			}
+
 			if ($Action -eq 'Waiting' -and $StatusMessage) {
 				$line += "  $StatusMessage"
 			}
+
 			if ($Action -eq 'Status' -and $StatusMessage) {
 				$line += "  $StatusMessage"
 			}
-			if ($Action -eq 'Failed' -and $ErrorMessage) {
+
+			if ($Action -in @('Failed', 'Error') -and $ErrorMessage) {
 				$errorText = "$ErrorMessage"
 				$errorText = $errorText -replace '[\r\n\t]+', ' '
 				if ($errorText.Length -gt 1000) {
 					$errorText = $errorText.Substring(0, 1000) + '...'
 				}
+
 				$line += "  $errorText"
 			}
-			$line += [System.Environment]::NewLine
 
+			$line += [System.Environment]::NewLine
 			$progressFilePath = Join-Path $LogsPath '1-export_progress.log'
 			$fullPath = [System.IO.Path]::GetFullPath($progressFilePath)
 			$normalizedPath = if ($IsWindows) { $fullPath.ToLowerInvariant() } else { $fullPath }
@@ -131,6 +135,7 @@ function Write-ZtExportProgress {
 				if ($lockAcquired -and $null -ne $mutex) {
 					$null = $mutex.ReleaseMutex()
 				}
+
 				if ($null -ne $mutex) {
 					$mutex.Dispose()
 				}

src/powershell/private/export/Write-ZtExportProgress.ps1

@@ -174,6 +174,7 @@
 						Set-ZtTimedOutResult -Result $result -Test $Test -Timeout $TestTimeout
 					}
 					else {
+						Write-PSFMessage -Level Warning -Message "PipelineStoppedException in test '{0}' was not caused by timeout" -StringValues $Test.TestID -Target $Test -ErrorRecord $_
 						throw
 					}
 				}
@@ -184,10 +185,18 @@
 			}
 		}
 		catch {
-			Write-PSFMessage -Level Warning -Message "Error executing test '{0}'" -StringValues $Test.TestID -Target $Test -ErrorRecord $_
+			Write-PSFMessage -Level Warning -Message "Error executing test '{0}': {1}" -StringValues $Test.TestID, $_.Exception.Message -Target $Test -ErrorRecord $_
 			$result.Success = $false
 			$result.Error = $_
-			Update-ZtProgressState -WorkerId $Test.TestID -WorkerName $testDisplayName -WorkerStatus 'Failed' -WorkerDetail "$_"
+			$message = @(
+				'❌  Test {0} failed due to an unexpected error.' -f $Test.TestID
+				' - **Error Message**: {0}.' -f $_.Exception.Message
+				'```'
+				'{0}' -f ($_ | Get-Error | Out-String)
+				'```'
+			) -join "`r`n"
+			Add-ZtTestResultDetail -TestId $Test.TestID -Title $Test.Title -Status $false -Result $message -CustomStatus 'Error'
+			Update-ZtProgressState -WorkerId $Test.TestID -WorkerName $testDisplayName -WorkerStatus 'Error' -WorkerDetail "Error: $($_.Exception.Message)"
 		}
 		finally {
 			$result.End = Get-Date
@@ -196,8 +205,10 @@
 			# Reset marker in an assured way, to prevent confusion about the current test being executed
 			$script:__ztCurrentTest = $null
 		}
+
 		Write-PSFMessage -Message "Processing test '{0}' - Concluded" -StringValues $Test.TestID -Target $Test -Tag end
 	}
+
 	end {
 		$result.Messages = Get-PSFMessage -Runspace ([runspace]::DefaultRunspace.InstanceId) | Where-Object { $_ -notin $previousMessages }
 		Write-ZtTestStatistics -Result $result
@@ -220,8 +231,11 @@
 			elseif ($result.Success) {
 				Write-ZtTestProgress -TestID $result.TestID -LogsPath $LogsPath -Action Completed -Duration $result.Duration
 			}
+			elseif ($result.Error) {
+				Write-ZtTestProgress -TestID $result.TestID -LogsPath $LogsPath -Action Error -Duration $result.Duration -ErrorMessage "Error: $($result.Error.Exception.Message)"
+			}
 			else {
-				$progressError = if ($result.Error) { "$($result.Error)" } else { $null }
+				$progressError = if ($result.Error) { "Error: $($result.Error.Exception.Message)" } else { $null }
 				Write-ZtTestProgress -TestID $result.TestID -LogsPath $LogsPath -Action Failed -Duration $result.Duration -ErrorMessage $progressError
 			}
 		}

src/powershell/private/tests/Invoke-ZtTest.ps1

@@ -95,6 +95,15 @@
 		$testsToRun = $testsToRun.Where{ $_.Pillar -in $stablePillars }
 	}
 
+	# Filter based on Compatible licenses
+	$skippedTestsForLicense = $testsToRun.Where{$_.CompatibleLicense.Count -gt 0 -and (-not (Test-ZtLicense -CompatibleLicense $_.CompatibleLicense)) }
+	$skippedTestsForLicense.ForEach{
+		Write-PSFMessage -Message ('Test {0} is skipped because no compatible license was found' -f $_.TestId) -Level Verbose
+		Add-ZtTestResultDetail -SkippedBecause NoCompatibleLicenseFound -TestId $_.TestId
+	}
+
+	$testsToRun = $testsToRun.Where{ $_.TestId -notin $skippedTestsForLicense.TestId }
+
 	# Filter based on service connection. If no service is specified in the test metadata, it will be run.
 	$skippedTestsForService = $testsToRun.Where{ $_.Service.count -gt 0 -and $_.Service.Count -notin $_.Service.Where{ $_ -in $ConnectedService}.count }
 	$skippedTestsForService.ForEach{
@@ -105,15 +114,6 @@
 
 	$testsToRun = $testsToRun.Where{ $_.TestId -notin $skippedTestsForService.TestId }
 
-	# Filter based on Compatible licenses
-	$skippedTestsForLicense = $testsToRun.Where{$_.CompatibleLicense.Count -gt 0 -and (-not (Test-ZtLicense -CompatibleLicense $_.CompatibleLicense)) }
-	$skippedTestsForLicense.ForEach{
-		Write-PSFMessage -Message ('Test {0} is skipped because no compatible license was found' -f $_.TestId) -Level Verbose
-		Add-ZtTestResultDetail -SkippedBecause NoCompatibleLicenseFound -TestId $_.TestId
-	}
-
-	$testsToRun = $testsToRun.Where{ $_.TestId -notin $skippedTestsForLicense.TestId }
-
 	# Separate Sync Tests (Compliance/ExchangeOnline/SharePointOnline) from Parallel Tests (because of DLL order to manage in runspaces & remoting into WPS)
 	[int[]]$syncTestIds   = $testsToRun.Where{ $_.Pillar -eq 'Data'}.TestId
 	$syncTests     = $testsToRun.Where{ $_.TestId -in $syncTestIds }

src/powershell/private/tests/Invoke-ZtTests.ps1

@@ -21,7 +21,7 @@ function Set-ZtTimedOutResult {
 	$Result.Error = New-ZtTimeoutErrorRecord -Test $Test -Timeout $Timeout
 
 	try {
-		Add-ZtTestResultDetail -TestId $Test.TestID -Status $false -Result "The test did not complete within the configured timeout of $($Timeout.ToString('hh\:mm\:ss')). Partial results, if any, were discarded."
+		Add-ZtTestResultDetail -TestId $Test.TestID -Status $false -CustomStatus 'Error' -Result "The test did not complete within the configured timeout of $($Timeout.ToString('hh\:mm\:ss')). Partial results, if any, were discarded."
 	}
 	catch {
 		Write-PSFMessage -Level Warning -Message "Failed to overwrite timed-out test result detail for test '{0}': {1}" -StringValues $Test.TestID, $_ -Target $Test -Tag timeout