fix: optimize ownership context imports (#358)

Author: jonathanweinberg

docs/admin-ownership-import.md

@@ -281,6 +281,8 @@ If you already saved mapping profiles for the same files, pass them with `-Mappi
 
 During a multi-source join, ShareSurfer now prints progress in phases: selected CSVs, source rows being processed, merged ownership rows, OBS context merge, AD lookup attempts, output files, and the final matched/source-only/ambiguous counts. Long AD-backed imports should continue to print a heartbeat instead of sitting silently after the header interview.
 
+Large project, OBS, path, or group context files can legitimately create many `ownership_context.csv` and `ownership_relationships.csv` rows. Current optimized builds aggregate OBS-only context and apply it by OBS bucket instead of merging every context row into every matching identity as the file is read. If a large context import is crawling for hours, stop the older run and move to the optimized release before trying again.
+
 If an older ShareSurfer run has been silent for a long time, it is safe to stop it with `Ctrl+C` and rerun with the saved `ownership-import.definition.json` or `ownership-enrichment-rerun.ps1`. The import reads CSVs and AD, then writes local output files; it does not modify AD, shares, or permissions.
 
 ### Project, OBS, Path, And Group Context Files

src/ShareSurfer/Public/Join-ShareSurferOwnershipSources.ps1

@@ -245,12 +245,6 @@ function Join-ShareSurferOwnershipSources {
                     $obsContextRows[$obsKey] = $incoming
                 }
 
-                if ($mergedKeysByObsKey.ContainsKey($obsKey)) {
-                    foreach ($existingKey in @($mergedKeysByObsKey[$obsKey].Keys)) {
-                        $mergedRows[$existingKey] = Merge-ShareSurferOwnershipEnrichmentRow -Existing $mergedRows[$existingKey] -Incoming $incoming
-                    }
-                }
-
                 $elapsedSeconds = [int][Math]::Floor($progressClock.Elapsed.TotalSeconds)
                 $rowIntervalDue = ($ProgressRowInterval -gt 0 -and (($sourceProcessed % $ProgressRowInterval) -eq 0))
                 $timeIntervalDue = ($ProgressIntervalSeconds -gt 0 -and (($elapsedSeconds - $lastProgressSecond) -ge $ProgressIntervalSeconds))
@@ -312,22 +306,31 @@ function Join-ShareSurferOwnershipSources {
     Write-ShareSurferOwnershipImportStatus -Message ('CSV merge complete: {0} merged ownership row(s), {1} OBS-only context row(s).' -f $mergedRows.Count, $obsContextRows.Count) -Quiet:$Quiet
 
     $obsContextProcessed = 0
+    $obsContextStrongRowMergeCount = 0
+    $obsContextOrphanRowCount = 0
     foreach ($obsKey in @($obsContextRows.Keys)) {
         $obsContextProcessed++
         $hasMatchingStrongRow = ($mergedKeysByObsKey.ContainsKey($obsKey) -and @($mergedKeysByObsKey[$obsKey].Keys).Count -gt 0)
 
-        if (-not $hasMatchingStrongRow) {
+        if ($hasMatchingStrongRow) {
+            foreach ($existingKey in @($mergedKeysByObsKey[$obsKey].Keys)) {
+                $mergedRows[$existingKey] = Merge-ShareSurferOwnershipEnrichmentRow -Existing $mergedRows[$existingKey] -Incoming $obsContextRows[$obsKey]
+                $obsContextStrongRowMergeCount++
+            }
+        }
+        else {
             $mergedRows[$obsKey] = $obsContextRows[$obsKey]
             Add-ShareSurferOwnershipObsMergeIndex -Index $mergedKeysByObsKey -ObsKey $obsKey -MergeKey $obsKey
+            $obsContextOrphanRowCount++
         }
 
         $elapsedSeconds = [int][Math]::Floor($progressClock.Elapsed.TotalSeconds)
         $rowIntervalDue = ($ProgressRowInterval -gt 0 -and (($obsContextProcessed % $ProgressRowInterval) -eq 0))
         $timeIntervalDue = ($ProgressIntervalSeconds -gt 0 -and (($elapsedSeconds - $lastProgressSecond) -ge $ProgressIntervalSeconds))
         if ($rowIntervalDue -or $timeIntervalDue -or $obsContextProcessed -eq $obsContextRows.Count) {
             $lastProgressSecond = $elapsedSeconds
-            Write-ShareSurferOwnershipImportProgress -Activity 'ShareSurfer ownership OBS context merge' -Status ('{0}/{1} OBS context row(s)' -f $obsContextProcessed, $obsContextRows.Count) -CurrentOperation ('Merged rows: {0}' -f $mergedRows.Count) -Processed $obsContextProcessed -Total $obsContextRows.Count -Quiet:$Quiet
-            Write-ShareSurferOwnershipImportStatus -Message ('OBS context merge: processed {0}/{1}; merged rows {2}; elapsed {3}.' -f $obsContextProcessed, $obsContextRows.Count, $mergedRows.Count, (Get-ShareSurferOwnershipElapsedText -Stopwatch $progressClock)) -Quiet:$Quiet
+            Write-ShareSurferOwnershipImportProgress -Activity 'ShareSurfer ownership OBS context merge' -Status ('{0}/{1} OBS context row(s)' -f $obsContextProcessed, $obsContextRows.Count) -CurrentOperation ('Applied to strong rows: {0}; orphan OBS rows: {1}; merged rows: {2}' -f $obsContextStrongRowMergeCount, $obsContextOrphanRowCount, $mergedRows.Count) -Processed $obsContextProcessed -Total $obsContextRows.Count -Quiet:$Quiet
+            Write-ShareSurferOwnershipImportStatus -Message ('OBS context merge: processed {0}/{1}; applied to {2} strong row(s); orphan OBS rows {3}; merged rows {4}; elapsed {5}.' -f $obsContextProcessed, $obsContextRows.Count, $obsContextStrongRowMergeCount, $obsContextOrphanRowCount, $mergedRows.Count, (Get-ShareSurferOwnershipElapsedText -Stopwatch $progressClock)) -Quiet:$Quiet
         }
     }
     Write-ShareSurferOwnershipImportProgress -Activity 'ShareSurfer ownership OBS context merge' -Completed -Quiet:$Quiet
@@ -425,6 +428,8 @@ function Join-ShareSurferOwnershipSources {
         PotentialServiceAccountCount = @($enrichedRows | Where-Object { [string]$_.PotentialServiceAccount -eq 'True' }).Count
         ForbiddenOu = (@($selectedForbiddenOus) -join '; ')
         ObsAttribute = $ObsAttribute
+        ObsContextStrongRowMergeCount = $obsContextStrongRowMergeCount
+        ObsContextOrphanRowCount = $obsContextOrphanRowCount
         AdLookupAttemptCount = $lookupAttemptCount
         DirectoryLookupCacheHitCount = $lookupCacheHitCount
         ElapsedSeconds = [Math]::Round($progressClock.Elapsed.TotalSeconds, 3)

tests/Invoke-ShareSurferTests.ps1

@@ -2721,6 +2721,55 @@ $tests = @(
             Assert-True (Test-Path -LiteralPath $rerunSummary.RelationshipOutputPath) 'Definition rerun should reproduce relationship output.'
         }
     },
+    @{
+        Name = 'Join-ShareSurferOwnershipSources defers OBS context fan-out until bucket merge'
+        Body = {
+            Import-Module $moduleManifest -Force
+            $identityPath = Join-Path ([System.IO.Path]::GetTempPath()) ('ShareSurferOwnershipFanoutIdentity-' + [guid]::NewGuid().ToString('N') + '.csv')
+            $contextPath = Join-Path ([System.IO.Path]::GetTempPath()) ('ShareSurferOwnershipFanoutContext-' + [guid]::NewGuid().ToString('N') + '.csv')
+            $outputPath = Join-Path ([System.IO.Path]::GetTempPath()) ('ShareSurferOwnershipFanoutOutput-' + [guid]::NewGuid().ToString('N') + '.csv')
+            $ownershipContextPath = Join-Path ([System.IO.Path]::GetTempPath()) ('ShareSurferOwnershipFanoutContextOut-' + [guid]::NewGuid().ToString('N') + '.csv')
+            $relationshipPath = Join-Path ([System.IO.Path]::GetTempPath()) ('ShareSurferOwnershipFanoutRelationships-' + [guid]::NewGuid().ToString('N') + '.csv')
+            $manifestPath = Join-Path ([System.IO.Path]::GetTempPath()) ('ShareSurferOwnershipFanoutManifest-' + [guid]::NewGuid().ToString('N') + '.csv')
+
+            $identityRows = for ($index = 1; $index -le 5; $index++) {
+                [pscustomobject]@{
+                    employee_id = 'E{0:0000}' -f $index
+                    display_name = 'User {0}' -f $index
+                    obs = 'CORP.FIN.AP'
+                    business_unit = 'Finance'
+                }
+            }
+            $contextRows = for ($index = 1; $index -le 10; $index++) {
+                [pscustomobject]@{
+                    obs = 'CORP.FIN.AP'
+                    project_code = 'AP-{0:0000}' -f $index
+                    project = 'Accounts Payable {0}' -f $index
+                    business_unit = 'Finance'
+                    data_owner = 'Finance Operations'
+                }
+            }
+            $identityRows | Export-Csv -LiteralPath $identityPath -NoTypeInformation -Encoding UTF8
+            $contextRows | Export-Csv -LiteralPath $contextPath -NoTypeInformation -Encoding UTF8
+
+            $summary = Join-ShareSurferOwnershipSources `
+                -Path @($identityPath, $contextPath) `
+                -OutputPath $outputPath `
+                -ContextOutputPath $ownershipContextPath `
+                -RelationshipOutputPath $relationshipPath `
+                -ManifestOutputPath $manifestPath `
+                -IncludeContextGraph `
+                -AdLookupMode DirectoryOnly `
+                -Quiet `
+                -Force
+            $rows = @(Import-Csv -LiteralPath $outputPath)
+
+            Assert-Equal $summary.RowCount 5 'Strong identity rows should remain the output ownership rows.'
+            Assert-Equal $summary.ObsContextStrongRowMergeCount 5 'OBS context should be applied once per matching strong row, not once per context row per strong row.'
+            Assert-Equal $summary.ObsContextOrphanRowCount 0 'Matching OBS context should not create orphan ownership rows.'
+            Assert-True (@($rows | Where-Object { $_.ProjectCode -eq 'AP-0001' }).Count -eq 5) 'Strong rows should still receive aggregated OBS context.'
+        }
+    },
     @{
         Name = 'Join-ShareSurferOwnershipSources reruns from ownership import definition'
         Body = {