Skip to content

Commit 5bf4a5a

Browse files
alexandairalexandair
authored
Add logging for export tenant data and database creation processes (#1069)
1 parent 65c88d2 commit 5bf4a5a

9 files changed

Lines changed: 468 additions & 36 deletions

‎src/powershell/private/export/Export-Database.ps1‎

Lines changed: 65 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,10 @@ function Export-Database {
3030
# The Zero Trust pillar to assess. Defaults to All.
3131
[ValidateSet('All', 'Identity', 'Devices', 'Network', 'Data')]
3232
[string]
33-
$Pillar = 'All'
33+
$Pillar = 'All',
34+
35+
[string]
36+
$LogsPath
3437
)
3538

3639
#region Utility Function
@@ -116,6 +119,41 @@ as
116119
}
117120
#endregion Utility Function
118121

122+
#region Logged Import Helper
123+
function Import-EntraTableLogged {
124+
[CmdletBinding()]
125+
param (
126+
[Parameter(Mandatory = $true)]
127+
[DuckDB.NET.Data.DuckDBConnection]
128+
$Database,
129+
130+
[Parameter(Mandatory = $true)]
131+
[string]
132+
$ExportPath,
133+
134+
[Parameter(Mandatory = $true)]
135+
[string]
136+
$TableName,
137+
138+
[string]
139+
$LogsPath
140+
)
141+
142+
Write-ZtDatabaseLog -TableName $TableName -LogsPath $LogsPath -Action Started
143+
$tableStart = Get-Date
144+
try {
145+
Import-EntraTable -Database $Database -ExportPath $ExportPath -TableName $TableName
146+
$tableDuration = (Get-Date) - $tableStart
147+
Write-ZtDatabaseLog -TableName $TableName -LogsPath $LogsPath -Action Completed -Duration $tableDuration
148+
}
149+
catch {
150+
$tableDuration = (Get-Date) - $tableStart
151+
Write-ZtDatabaseLog -TableName $TableName -LogsPath $LogsPath -Action Failed -Duration $tableDuration -ErrorMessage $_
152+
throw
153+
}
154+
}
155+
#endregion Logged Import Helper
156+
119157
$activity = "Creating database"
120158
Write-ZtProgress -Activity $activity -Status "Starting"
121159
Update-ZtProgressState -WorkerId 'database' -WorkerName 'Creating Database' -WorkerStatus 'Running' -WorkerDetail 'Initializing...'
@@ -142,40 +180,40 @@ as
142180
}
143181

144182
if ($Pillar -in ('All', 'Identity')) {
145-
Import-EntraTable -Database $database -ExportPath $ExportPath -TableName 'User'
146-
Import-EntraTable -Database $database -ExportPath $ExportPath -TableName 'Application'
147-
Import-EntraTable -Database $database -ExportPath $ExportPath -TableName 'ServicePrincipal'
148-
Import-EntraTable -Database $database -ExportPath $ExportPath -TableName 'ServicePrincipalSignIn'
149-
Import-EntraTable -Database $database -ExportPath $ExportPath -TableName 'SignIn'
150-
Import-EntraTable -Database $database -ExportPath $ExportPath -TableName 'RoleDefinition'
151-
Import-EntraTable -Database $database -ExportPath $ExportPath -TableName 'RoleAssignment'
152-
Import-EntraTable -Database $database -ExportPath $ExportPath -TableName 'RoleAssignmentGroup'
153-
Import-EntraTable -Database $database -ExportPath $ExportPath -TableName 'RoleAssignmentScheduleInstance'
154-
Import-EntraTable -Database $database -ExportPath $ExportPath -TableName 'RoleAssignmentScheduleInstanceGroup'
155-
Import-EntraTable -Database $database -ExportPath $ExportPath -TableName 'RoleEligibilityScheduleInstance'
156-
Import-EntraTable -Database $database -ExportPath $ExportPath -TableName 'RoleEligibilityScheduleInstanceGroup'
157-
Import-EntraTable -Database $database -ExportPath $ExportPath -TableName 'RoleManagementPolicyAssignment'
158-
Import-EntraTable -Database $database -ExportPath $ExportPath -TableName 'UserRegistrationDetails'
183+
Import-EntraTableLogged -Database $database -ExportPath $ExportPath -TableName 'User' -LogsPath $LogsPath
184+
Import-EntraTableLogged -Database $database -ExportPath $ExportPath -TableName 'Application' -LogsPath $LogsPath
185+
Import-EntraTableLogged -Database $database -ExportPath $ExportPath -TableName 'ServicePrincipal' -LogsPath $LogsPath
186+
Import-EntraTableLogged -Database $database -ExportPath $ExportPath -TableName 'ServicePrincipalSignIn' -LogsPath $LogsPath
187+
Import-EntraTableLogged -Database $database -ExportPath $ExportPath -TableName 'SignIn' -LogsPath $LogsPath
188+
Import-EntraTableLogged -Database $database -ExportPath $ExportPath -TableName 'RoleDefinition' -LogsPath $LogsPath
189+
Import-EntraTableLogged -Database $database -ExportPath $ExportPath -TableName 'RoleAssignment' -LogsPath $LogsPath
190+
Import-EntraTableLogged -Database $database -ExportPath $ExportPath -TableName 'RoleAssignmentGroup' -LogsPath $LogsPath
191+
Import-EntraTableLogged -Database $database -ExportPath $ExportPath -TableName 'RoleAssignmentScheduleInstance' -LogsPath $LogsPath
192+
Import-EntraTableLogged -Database $database -ExportPath $ExportPath -TableName 'RoleAssignmentScheduleInstanceGroup' -LogsPath $LogsPath
193+
Import-EntraTableLogged -Database $database -ExportPath $ExportPath -TableName 'RoleEligibilityScheduleInstance' -LogsPath $LogsPath
194+
Import-EntraTableLogged -Database $database -ExportPath $ExportPath -TableName 'RoleEligibilityScheduleInstanceGroup' -LogsPath $LogsPath
195+
Import-EntraTableLogged -Database $database -ExportPath $ExportPath -TableName 'RoleManagementPolicyAssignment' -LogsPath $LogsPath
196+
Import-EntraTableLogged -Database $database -ExportPath $ExportPath -TableName 'UserRegistrationDetails' -LogsPath $LogsPath
159197

160198
New-ViewRole -Database $database
161199
}
162200

163201
if ($Pillar -in ('All', 'Devices')) {
164-
Import-EntraTable -Database $database -ExportPath $ExportPath -TableName 'Device'
165-
Import-EntraTable -Database $database -ExportPath $ExportPath -TableName 'ConfigurationPolicy'
202+
Import-EntraTableLogged -Database $database -ExportPath $ExportPath -TableName 'Device' -LogsPath $LogsPath
203+
Import-EntraTableLogged -Database $database -ExportPath $ExportPath -TableName 'ConfigurationPolicy' -LogsPath $LogsPath
166204
}
167205

168206
if ($Pillar -in ('All', 'Network')) {
169-
Import-EntraTable -Database $database -ExportPath $ExportPath -TableName 'User'
170-
Import-EntraTable -Database $database -ExportPath $ExportPath -TableName 'Application'
171-
Import-EntraTable -Database $database -ExportPath $ExportPath -TableName 'ServicePrincipal'
172-
Import-EntraTable -Database $database -ExportPath $ExportPath -TableName 'RoleDefinition'
173-
Import-EntraTable -Database $database -ExportPath $ExportPath -TableName 'RoleAssignment'
174-
Import-EntraTable -Database $database -ExportPath $ExportPath -TableName 'RoleAssignmentGroup'
175-
Import-EntraTable -Database $database -ExportPath $ExportPath -TableName 'RoleAssignmentScheduleInstance'
176-
Import-EntraTable -Database $database -ExportPath $ExportPath -TableName 'RoleAssignmentScheduleInstanceGroup'
177-
Import-EntraTable -Database $database -ExportPath $ExportPath -TableName 'RoleEligibilityScheduleInstance'
178-
Import-EntraTable -Database $database -ExportPath $ExportPath -TableName 'RoleEligibilityScheduleInstanceGroup'
207+
Import-EntraTableLogged -Database $database -ExportPath $ExportPath -TableName 'User' -LogsPath $LogsPath
208+
Import-EntraTableLogged -Database $database -ExportPath $ExportPath -TableName 'Application' -LogsPath $LogsPath
209+
Import-EntraTableLogged -Database $database -ExportPath $ExportPath -TableName 'ServicePrincipal' -LogsPath $LogsPath
210+
Import-EntraTableLogged -Database $database -ExportPath $ExportPath -TableName 'RoleDefinition' -LogsPath $LogsPath
211+
Import-EntraTableLogged -Database $database -ExportPath $ExportPath -TableName 'RoleAssignment' -LogsPath $LogsPath
212+
Import-EntraTableLogged -Database $database -ExportPath $ExportPath -TableName 'RoleAssignmentGroup' -LogsPath $LogsPath
213+
Import-EntraTableLogged -Database $database -ExportPath $ExportPath -TableName 'RoleAssignmentScheduleInstance' -LogsPath $LogsPath
214+
Import-EntraTableLogged -Database $database -ExportPath $ExportPath -TableName 'RoleAssignmentScheduleInstanceGroup' -LogsPath $LogsPath
215+
Import-EntraTableLogged -Database $database -ExportPath $ExportPath -TableName 'RoleEligibilityScheduleInstance' -LogsPath $LogsPath
216+
Import-EntraTableLogged -Database $database -ExportPath $ExportPath -TableName 'RoleEligibilityScheduleInstanceGroup' -LogsPath $LogsPath
179217

180218
New-ViewRole -Database $database
181219
}

‎src/powershell/private/export/Export-ZtTenantData.ps1‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,10 @@ function Export-ZtTenantData {
5151
$Pillar = 'All',
5252

5353
[int]
54-
$ThrottleLimit = (Get-PSFConfigValue -FullName 'ZeroTrustAssessment.ThrottleLimit.Export' -Fallback 5)
54+
$ThrottleLimit = (Get-PSFConfigValue -FullName 'ZeroTrustAssessment.ThrottleLimit.Export' -Fallback 5),
55+
56+
[string]
57+
$LogsPath
5558
)
5659

5760
#region Helper Functions
@@ -146,8 +149,8 @@ https://github.com/microsoft/zerotrustassessment/issues
146149
# Show $applicableExports
147150
Write-PSFMessage "Applicable exports: $($applicableExports | ForEach-Object { $_.Name } | Sort-Object | Out-String)"
148151

149-
$workflow = Start-ZtTenantDataExport -ExportConfig $applicableExports -ThrottleLimit $ThrottleLimit -ExportPath $ExportPath
150-
Wait-ZtTenantDataExport -Workflow $workflow
152+
$workflow = Start-ZtTenantDataExport -ExportConfig $applicableExports -ThrottleLimit $ThrottleLimit -ExportPath $ExportPath -LogsPath $LogsPath
153+
Wait-ZtTenantDataExport -Workflow $workflow -LogsPath $LogsPath
151154
}
152155
finally {
153156
if ($workflow) {

‎src/powershell/private/export/Invoke-ZtTenantDataExport.ps1‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,10 @@
3434
$ExportPath,
3535

3636
[Psftimespan]
37-
$DependencyTimeout
37+
$DependencyTimeout,
38+
39+
[string]
40+
$LogsPath
3841
)
3942
begin {
4043
$previousMessages = Get-PSFMessage -Runspace ([runspace]::DefaultRunspace.InstanceId)
@@ -71,6 +74,7 @@
7174
}
7275
process {
7376
Write-PSFMessage -Message "Processing export '{0}'" -StringValues $Export.Name -Target $Export -Tag start
77+
Write-ZtExportProgress -ExportName $Export.Name -LogsPath $LogsPath -Action Started
7478

7579
# Update progress dashboard: map this runspace to this export and mark as starting
7680
$runspaceId = [runspace]::DefaultRunspace.InstanceId.ToString()
@@ -86,6 +90,7 @@
8690
$workflow.Data[$Export.Name].Status = 'Waiting'
8791
$workflow.Data[$Export.Name].Updated = Get-Date
8892
Update-ZtProgressState -WorkerId $Export.Name -WorkerName $Export.Name -WorkerStatus 'Waiting' -WorkerDetail "Waiting for dependency: $($Export.DependsOn)"
93+
Write-ZtExportProgress -ExportName $Export.Name -LogsPath $LogsPath -Action Waiting -StatusMessage "dependency:$($Export.DependsOn)"
8994

9095
while (-not $exportState -and $workflow.Data[$Export.DependsOn].Status -ne 'Done') {
9196
Write-PSFMessage -Message "Export '{0}' depends on '{1}', which is not yet completed" -StringValues $Export.Name, $Export.DependsOn -Once "ZeroTrust-$($Export.Name)-$($identity)" -Target $Export
@@ -135,6 +140,7 @@
135140
$result.Start = Get-Date
136141
$workflow.Data[$Export.Name].Status = 'InProgress'
137142
Update-ZtProgressState -WorkerId $Export.Name -WorkerName $Export.Name -WorkerStatus 'Running' -WorkerDetail 'Initializing...'
143+
Write-ZtExportProgress -ExportName $Export.Name -LogsPath $LogsPath -Action InProgress
138144
switch ($Export.Type) {
139145
PrivilegedGroup {
140146
$exportParam = $Export | ConvertTo-PSFHashtable -ReferenceCommand Export-ZtGraphEntityPrivilegedGroup
@@ -161,6 +167,7 @@
161167
$result.Success = $false
162168
$result.Error = $_
163169
}
170+
Write-ZtExportProgress -ExportName $Export.Name -LogsPath $LogsPath -Action Failed -ErrorMessage $_
164171
}
165172
finally {
166173
$result.End = Get-Date
@@ -169,12 +176,14 @@
169176
$workflow.Data[$Export.Name].Status = 'Done'
170177
$workflow.Data[$Export.Name].Updated = Get-Date
171178
Update-ZtProgressState -WorkerId $Export.Name -WorkerName $Export.Name -WorkerStatus 'Done' -WorkerDetail ''
179+
Write-ZtExportProgress -ExportName $Export.Name -LogsPath $LogsPath -Action Completed -Duration $result.Duration
172180
}
173181
}
174182
Write-PSFMessage -Message "Processing test '{0}' - Concluded" -StringValues $Export.Name -Target $Export -Tag end
175183
}
176184
end {
177185
$result.Messages = Get-PSFMessage -Runspace ([runspace]::DefaultRunspace.InstanceId) | Where-Object { $_ -notin $previousMessages }
186+
Write-ZtExportLog -Result $result -LogsPath $LogsPath
178187
$result
179188
}
180189
}

‎src/powershell/private/export/Start-ZtTenantDataExport.ps1‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,14 +39,18 @@
3939
$ExportPath,
4040

4141
[int]
42-
$ThrottleLimit = 5
42+
$ThrottleLimit = 5,
43+
44+
[string]
45+
$LogsPath
4346
)
4447
begin {
4548
#region Calculate Resources to Import
4649
$variables = @{
4750
exportPath = $ExportPath
4851
dependencyTimeout = Get-PSFConfigValue -FullName 'ZeroTrustAssessment.Export.DependencyWaitLimit'
4952
moduleRoot = $script:ModuleRoot
53+
logsPath = $LogsPath
5054
}
5155

5256
# Explicitly including all modules required, as we later import the psm1, not the psd1 file
@@ -90,7 +94,7 @@
9094
$null = $workflow | Add-PSFRunspaceWorker -Name Exporter @param -Begin {
9195
$script:ModuleRoot = $moduleRoot
9296
} -ScriptBlock {
93-
Invoke-ZtTenantDataExport -Export $_ -DependencyTimeout $dependencyTimeout -ExportPath $exportPath
97+
Invoke-ZtTenantDataExport -Export $_ -DependencyTimeout $dependencyTimeout -ExportPath $exportPath -LogsPath $logsPath
9498
}
9599
$workflow | Write-PSFRunspaceQueue -Name Input -BulkValues @($ExportConfig) -Close
96100
foreach ($export in $ExportConfig) {

‎src/powershell/private/export/Wait-ZtTenantDataExport.ps1‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,14 +17,18 @@
1717
[CmdletBinding()]
1818
param (
1919
[PSFramework.Runspace.RSWorkflow]
20-
$Workflow
20+
$Workflow,
21+
22+
[string]
23+
$LogsPath
2124
)
2225
begin {
2326
$failedExports = @{}
2427
$totalCount = $Workflow.Queues["Input"].TotalItemCount
2528
$progressID = Get-Random -Minimum 1 -Maximum 999
2629
$taskProgID = @{ }
2730
$lastMessageScan = [datetime]::MinValue
31+
$lastStatusSnapshot = [datetime]::MinValue
2832

2933
# Initialize progress dashboard summary for the export stage
3034
Update-ZtProgressState -TotalItems $totalCount -CompletedItems 0 -FailedItems 0 -InProgressItems 0
@@ -89,6 +93,13 @@
8993
# Non-critical: don't let message scanning break the wait loop
9094
}
9195

96+
# Write periodic status snapshot to export progress log (~every 10 seconds)
97+
if ($LogsPath -and ([datetime]::Now - $lastStatusSnapshot).TotalSeconds -ge 10) {
98+
$statusMsg = "Pending:$countPending Waiting:$countWaiting InProgress:$countInProgress Done:$countDone Failed:$countFailed"
99+
Write-ZtExportProgress -ExportName '_overall_' -LogsPath $LogsPath -Action Status -StatusMessage $statusMsg
100+
$lastStatusSnapshot = [datetime]::Now
101+
}
102+
92103
foreach ($task in $Workflow.Data.Values) {
93104
if ($task.Status -in 'Waiting', 'InProgress') {
94105
if (-not $taskProgID[$task.Name]) {

‎src/powershell/private/export/Write-ZtDatabaseLog.ps1‎

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
function Write-ZtDatabaseLog {
2+
<#
3+
.SYNOPSIS
4+
Appends a progress entry to the database import progress log.
5+
6+
.DESCRIPTION
7+
Appends a single line to _database_progress.log in the logs folder, recording when
8+
each table import starts, completes, or fails. This append-only log provides an
9+
at-a-glance timeline of all database table imports and makes it easy to identify
10+
which table import caused an issue.
11+
12+
Uses a per-file named mutex plus [System.IO.File]::AppendAllText for
13+
consistency with the export and test progress log patterns.
14+
15+
.PARAMETER TableName
16+
The name of the database table being imported.
17+
18+
.PARAMETER LogsPath
19+
Path to the logs folder. If empty or null, the function is a no-op.
20+
21+
.PARAMETER Action
22+
The progress action: Started, Completed, or Failed.
23+
24+
.PARAMETER Duration
25+
The table import duration (for Completed/Failed actions).
26+
27+
.PARAMETER ErrorMessage
28+
The error message (for Failed actions).
29+
30+
.EXAMPLE
31+
PS C:\> Write-ZtDatabaseLog -TableName 'User' -LogsPath $logsPath -Action Started
32+
33+
Appends a STARTED line for the User table import to the database progress log.
34+
35+
.EXAMPLE
36+
PS C:\> Write-ZtDatabaseLog -TableName 'User' -LogsPath $logsPath -Action Completed -Duration $elapsed
37+
38+
Appends a COMPLETED line for the User table import to the database progress log.
39+
#>
40+
[CmdletBinding()]
41+
param (
42+
[Parameter(Mandatory = $true)]
43+
[string]
44+
$TableName,
45+
46+
[string]
47+
$LogsPath,
48+
49+
[Parameter(Mandatory = $true)]
50+
[ValidateSet('Started', 'Completed', 'Failed')]
51+
[string]
52+
$Action,
53+
54+
[timespan]
55+
$Duration,
56+
57+
$ErrorMessage
58+
)
59+
process {
60+
if (-not $LogsPath) { return }
61+
62+
try {
63+
[void][System.IO.Directory]::CreateDirectory($LogsPath)
64+
65+
$timestamp = (Get-Date).ToString('yyyy-MM-dd HH:mm:ss.fff')
66+
$actionPadded = $Action.ToUpper().PadRight(10)
67+
68+
$line = "$timestamp $actionPadded $TableName"
69+
if ($null -ne $Duration) {
70+
$line += " $($Duration.ToString('hh\:mm\:ss\.fff'))"
71+
}
72+
if ($Action -eq 'Failed' -and $ErrorMessage) {
73+
$errorText = "$ErrorMessage"
74+
$errorText = $errorText -replace '[\r\n\t]+', ' '
75+
if ($errorText.Length -gt 1000) {
76+
$errorText = $errorText.Substring(0, 1000) + '...'
77+
}
78+
$line += " $errorText"
79+
}
80+
$line += [System.Environment]::NewLine
81+
82+
$progressFilePath = Join-Path $LogsPath '_database_progress.log'
83+
$fullPath = [System.IO.Path]::GetFullPath($progressFilePath)
84+
$normalizedPath = if ($IsWindows) { $fullPath.ToLowerInvariant() } else { $fullPath }
85+
86+
# Cache the mutex name per resolved path to avoid repeated SHA256 hashing
87+
if (-not $script:ZtDatabaseProgressMutexCache) {
88+
$script:ZtDatabaseProgressMutexCache = @{}
89+
}
90+
if ($script:ZtDatabaseProgressMutexCache.ContainsKey($normalizedPath)) {
91+
$mutexName = $script:ZtDatabaseProgressMutexCache[$normalizedPath]
92+
}
93+
else {
94+
$pathBytes = [System.Text.Encoding]::UTF8.GetBytes($normalizedPath)
95+
$pathHashBytes = [System.Security.Cryptography.SHA256]::HashData($pathBytes)
96+
$pathHash = [System.BitConverter]::ToString($pathHashBytes).Replace('-', '')
97+
$mutexName = "Local\ZtDatabaseProgress_$pathHash"
98+
$script:ZtDatabaseProgressMutexCache[$normalizedPath] = $mutexName
99+
}
100+
101+
$mutex = $null
102+
$lockAcquired = $false
103+
try {
104+
$mutex = [System.Threading.Mutex]::new($false, $mutexName)
105+
$lockAcquired = $mutex.WaitOne([TimeSpan]::FromSeconds(5))
106+
if (-not $lockAcquired) {
107+
throw "Timed out waiting for database progress log mutex '$mutexName'."
108+
}
109+
110+
[System.IO.File]::AppendAllText($fullPath, $line)
111+
}
112+
finally {
113+
if ($lockAcquired -and $null -ne $mutex) {
114+
$null = $mutex.ReleaseMutex()
115+
}
116+
if ($null -ne $mutex) {
117+
$mutex.Dispose()
118+
}
119+
}
120+
}
121+
catch {
122+
Write-PSFMessage -Level Warning -Message "Failed to write database progress log for '{0}': {1}" -StringValues $TableName, $_.Exception.Message -Tag log
123+
}
124+
}
125+
}

0 commit comments

Comments
 (0)