diff --git a/README.md b/README.md index 85b0c675..46233018 100644 --- a/README.md +++ b/README.md @@ -59,11 +59,13 @@ Listed as [security monitoring tool](https://docs.microsoft.com/en-us/azure/arch ## Release history -__Changes__ (2022-Jun-14 / Major) +__Changes__ (2022-Jun-22 / Major) -* Fix issue #110 / handle `DisallowedProvider` errorCode (Blueprints, PolicyInsights) -* Fix issue #111 / replace .AddRange with foreach/.Add -* Use AzAPICall PowerShell module version 1.1.16 +* New feature 'Orphaned Resources' - Azure Resource Graph based reporting on orphaned resources (TenantSummary, ScopeInsights, CSV export). [Azure Orphan Resources - GitHub](https://github.com/dolevshor/azure-orphan-resources) ARG queries and workbooks by Dolev Shor +* New feature 'Resource fluctuation' - Compare against Resources from previous run and output aggregated summary of the Resource fluctuation (TenantSummary, ScopeInsights, CSV export) +* Fix `/providers/Microsoft.Authorization/roleAssignmentScheduleInstances` AzAPICall errorhandling (error 400, 500) +* Optimize procedure to update the AzAPICall module +* Use AzAPICall PowerShell module version 1.1.17 Passed tests: Powershell Core 7.2.4 on Windows Passed tests: Powershell Core 7.2.4 Azure DevOps hosted agent ubuntu-20.04 @@ -174,7 +176,7 @@ Short presentation on AzGovViz [[download](slides/AzGovViz_intro.pdf)] * Hierarchy Settings | Require authorization for Management Group creation * __Subscriptions, Resources & Defender__ * Subscription insights - * QuotaId, State, Tags, Microsoft Defender for Cloud Secure Score, Cost, Management Group path, Role assignment limit + * QuotaId, State, Tags, Microsoft Defender for Cloud Secure Score, Cost, Management Group path, Role assignment limit, enabled Preview features * Tag Name usage * Insights on usage of Tag Names on Subscriptions, ResourceGroups and Resources * Resources @@ -185,6 +187,7 @@ Short presentation on AzGovViz [[download](slides/AzGovViz_intro.pdf)] * Explicit Resource Provider state per Subscription * Resource Locks * Aggregated insights for Lock and respective Lock-type usage on Subscriptions, ResourceGroups and Resources + * Orphaned Resources (ARG) * Microsoft Defender for Cloud * Summary of Microsoft Defender for Cloud coverage by plan (count of Subscription per plan/tier) * Summary of Microsoft Defender for Cloud plans coverage by Subscription (plan/tier) @@ -192,6 +195,8 @@ Short presentation on AzGovViz [[download](slides/AzGovViz_intro.pdf)] * UserAssigned Managed Identities assigned to Resources / vice versa * Summary of all UserAssigned Managed Identities assigned to Resources * Summary of Resources that have an UserAssigned Managed Identity assigned + * PSRule for Azure + * Well-Architected Framework aligned best practice analysis for resources, including guidance for remediation * __Diagnostics__ * Management Groups Diagnostic settings report * Management Group, Diagnostic setting name, target type (LA, SA, EH), target Id, Log Category status diff --git a/history.md b/history.md index 4b2c59da..0a3268c4 100644 --- a/history.md +++ b/history.md @@ -4,6 +4,14 @@ ### AzGovViz version 6 +__Changes__ (2022-Jun-22 / Major) + +* New feature 'Orphaned Resources' - Azure Resource Graph based reporting on orphaned resources (TenantSummary, ScopeInsights, CSV export). [Azure Orphan Resources - GitHub](https://github.com/dolevshor/azure-orphan-resources) ARG queries and workbooks by Dolev Shor +* New feature 'Resource fluctuation' - Compare against Resources from previous run and output aggregated summary of the Resource fluctuation (TenantSummary, ScopeInsights, CSV export) +* Fix `/providers/Microsoft.Authorization/roleAssignmentScheduleInstances` AzAPICall errorhandling (error 400, 500) +* Optimize procedure to update the AzAPICall module +* Use AzAPICall PowerShell module version 1.1.17 + __Changes__ (2022-Jun-14 / Major) * Fix issue #110 / handle `DisallowedProvider` errorCode (Blueprints, PolicyInsights) diff --git a/pwsh/AzGovVizParallel.ps1 b/pwsh/AzGovVizParallel.ps1 index ef20310f..5cc539e3 100644 --- a/pwsh/AzGovVizParallel.ps1 +++ b/pwsh/AzGovVizParallel.ps1 @@ -283,10 +283,10 @@ Param $Product = 'AzGovViz', [string] - $AzAPICallVersion = '1.1.16', + $AzAPICallVersion = '1.1.17', [string] - $ProductVersion = 'v6_major_20220614_1', + $ProductVersion = 'v6_major_20220622_1', [string] $GithubRepository = 'aka.ms/AzGovViz', @@ -2926,7 +2926,7 @@ function getMDfCSecureScoreMG { Write-Host ' Microsoft Defender for Cloud SecureScore for Management Groups will not be available' -ForegroundColor Yellow } else { - foreach ($entry in $getMgAscSecureScore.data) { + foreach ($entry in $getMgAscSecureScore) { $script:htMgASCSecureScore.($entry.mgId) = @{} if ($entry.secureScore -eq 404) { $script:htMgASCSecureScore.($entry.mgId).SecureScore = 'n/a' @@ -2938,6 +2938,110 @@ function getMDfCSecureScoreMG { } } } +function getOrphanedResources { + $start = Get-Date + Write-Host 'Getting orphaned resources (ARG)' + + $queries = [System.Collections.ArrayList]@() + $intent = 'clean up' + $null = $queries.Add([PSCustomObject]@{ + queryName = 'microsoft.resources/subscriptions/resourceGroups' + query = "ResourceContainers | where type =~ 'microsoft.resources/subscriptions/resourceGroups' | extend rgAndSub = strcat(resourceGroup, '--', subscriptionId) | join kind=leftouter (Resources | extend rgAndSub = strcat(resourceGroup, '--', subscriptionId) | summarize count() by rgAndSub) on rgAndSub | where isnull(count_) | project type, subscriptionId, Resource=id, Intent='$intent'" + }) + + $intent = 'misconfiguration' + $null = $queries.Add([PSCustomObject]@{ + queryName = 'microsoft.network/networkSecurityGroups' + query = "Resources | where type =~ 'microsoft.network/networkSecurityGroups' and isnull(properties.networkInterfaces) and isnull(properties.subnets) | project type, subscriptionId, Resource=id, Intent='$intent'" + }) + + $intent = 'misconfiguration' + $null = $queries.Add([PSCustomObject]@{ + queryName = 'microsoft.network/routeTables' + query = "resources | where type =~ 'microsoft.network/routeTables' | where isnull(properties.subnets) | project type, subscriptionId, Resource=id, Intent='$intent'" + }) + + $intent = 'misconfiguration' + $null = $queries.Add([PSCustomObject]@{ + queryName = 'microsoft.network/networkInterfaces' + query = "Resources | where type =~ 'microsoft.network/networkInterfaces' | where isnull(properties.privateEndpoint) | where isnull(properties.privateLinkService) | where properties.hostedWorkloads == '[]' | where properties !has 'virtualmachine' | project type, subscriptionId, Resource=id, Intent='$intent'" + }) + + $intent = 'cost savings' + $null = $queries.Add([PSCustomObject]@{ + queryName = 'microsoft.compute/disks' + query = "Resources | where type =~ 'microsoft.compute/disks' | extend diskState = tostring(properties.diskState) | where managedBy == '' | where not(name endswith '-ASRReplica' or name startswith 'ms-asr-') | project type, subscriptionId, Resource=id, Intent='$intent'" + }) + + $intent = 'cost savings' + $null = $queries.Add([PSCustomObject]@{ + queryName = 'microsoft.network/publicIpAddresses' + query = "Resources | where type =~ 'microsoft.network/publicIpAddresses' | where properties.ipConfiguration == '' | project type, subscriptionId, Resource=id, Intent='$intent'" + }) + + $intent = 'misconfiguration' + $null = $queries.Add([PSCustomObject]@{ + queryName = 'microsoft.compute/availabilitySets' + query = "Resources | where type =~ 'microsoft.compute/availabilitySets' | where properties.virtualMachines == '[]' | project type, subscriptionId, Resource=id, Intent='$intent'" + }) + + $intent = 'misconfiguration' + $null = $queries.Add([PSCustomObject]@{ + queryName = 'microsoft.network/loadBalancers' + query = "Resources | where type =~ 'microsoft.network/loadBalancers' | where properties.backendAddressPools == '[]' | project type, subscriptionId, Resource=id, Intent='$intent'" + }) + + $intent = 'clean up' + $null = $queries.Add([PSCustomObject]@{ + queryName = 'microsoft.web/serverfarms' + query = "Resources | where type =~ 'microsoft.web/serverfarms' | where properties.numberOfSites == 0 | project type, subscriptionId, Resource=id, Intent='$intent'" + }) + + $queries | foreach-object -Parallel { + $queryDetail = $_ + $arrayOrphanedResources = $using:arrayOrphanedResources + $subsToProcessInCustomDataCollection = $using:subsToProcessInCustomDataCollection + $azAPICallConf = $using:azAPICallConf + + #Batching: https://docs.microsoft.com/en-us/azure/governance/resource-graph/troubleshoot/general#toomanysubscription + $counterBatch = [PSCustomObject] @{ Value = 0 } + $batchSize = 1000 + $subscriptionsBatch = $subsToProcessInCustomDataCollection | Group-Object -Property { [math]::Floor($counterBatch.Value++ / $batchSize) } + + $uri = "$($azAPICallConf['azAPIEndpointUrls'].ARM)/providers/Microsoft.ResourceGraph/resources?api-version=2021-03-01" + $method = "POST" + foreach ($batch in $subscriptionsBatch) { + " Getting orphaned $($queryDetail.queryName) for $($batch.Group.subscriptionId.Count) Subscriptions" + $subscriptions = '"{0}"' -f ($batch.Group.subscriptionId -join '","') + $body = @" +{ +"query": "$($queryDetail.query)", +"subscriptions": [$($subscriptions)] +} +"@ + + $res = (AzAPICall -AzAPICallConfiguration $azAPICallConf -uri $uri -method $method -body $body -listenOn 'Content' -currentTask "Getting orphaned $($queryDetail.queryName)") + #Write-Host '$res.count:' $res.count + if ($res.count -gt 0) { + foreach ($resource in $res) { + $null = $script:arrayOrphanedResources.Add($resource) + } + } + } + } -ThrottleLimit ($queries.Count) + + if ($arrayOrphanedResources.Count -gt 0) { + Write-Host " Found $($arrayOrphanedResources.Count) orphaned Resources" + Write-Host " Exporting OrphanedResources CSV '$($outputPath)$($DirectorySeparatorChar)$($fileName)_ResourcesOrphaned.csv'" + $arrayOrphanedResources | Export-Csv -Path "$($outputPath)$($DirectorySeparatorChar)$($fileName)_ResourcesOrphaned.csv" -Delimiter "$csvDelimiter" -NoTypeInformation + } + else { + Write-Host " No orphaned Resources found" + } + + $end = Get-Date + Write-Host "Getting orphaned resources (ARG) processing duration: $((NEW-TIMESPAN -Start $start -End $end).TotalMinutes) minutes ($((NEW-TIMESPAN -Start $start -End $end).TotalSeconds) seconds)" +} function getResourceDiagnosticsCapability { Write-Host 'Checking Resource Types Diagnostics capability (1st party only)' $startResourceDiagnosticsCheck = Get-Date @@ -3990,6 +4094,204 @@ function processDataCollection { if (-not $azAPICallConf['htParameters'].HierarchyMapOnly -and -not $azAPICallConf['htParameters'].ManagementGroupsOnly) { if (-not $NoCsvExport) { + + #fluctuation + Write-Host "Process Resource fluctuation" + $start = get-date + if (Test-Path -Filter "*$($ManagementGroupId)_ResourcesAll.csv" -LiteralPath "$($outputPath)") { + $startImportPrevious = get-date + $doResourceFluctuation = $true + + try { + $previous = Get-ChildItem -Path $outputPath -Filter "*$($ManagementGroupId)_ResourcesAll.csv" | Sort-Object -Descending -Property LastWriteTime | Select-Object -First 1 -ErrorAction Stop + $importPrevious = Import-Csv -LiteralPath "$($outputPath)$($DirectorySeparatorChar)$($previous.Name)" -Encoding utf8 -Delimiter ";" | Select-Object -ExpandProperty id + Write-Host " Import previous ($($previous.Name)) duration: $((NEW-TIMESPAN -Start $startImportPrevious -End (get-date)).TotalSeconds)" + } + catch { + Write-Host " FAILED: Import-Csv '$($outputPath)$($DirectorySeparatorChar)$($previous.Name)'" + $doResourceFluctuation = $false + + } + + if ($doResourceFluctuation) { + #$importPrevious.Count + + #https://gist.github.com/fatherjack/4c91cc6832b8b02d1b7319716a5fba52 + function Compare-StringSet { + <# + .SYNOPSIS + Compare two sets of strings and see the matched and unmatched elements from each input + + .DESCRIPTION + Compares sets of + + .PARAMETER Ref + The reference set of values to be compared + + .PARAMETER Diff + The difference set of values to be compared + + .PARAMETER CaseSensitive + Enables a case-sensitive comparison + + .EXAMPLE + $ref, $dif = @( + , @('a', 'b', 'c') + , @('b', 'c', 'd') + ) + $Sets = Compare-StringSet $ref $dif + $Sets.RefOnly + + $Sets.DiffOnly + + $Sets.Both + + This example sets up two arrays with some similar values and then passes them both to the Compare-StringSet function. the results of this are stored in the variable $Sets. + $Sets is an object that has three properties - RefOnly, DiffOnly, and Both. These are sets of the incoming values where they intersect or not. + + .EXAMPLE + $ref, $dif = @( + , @('tree', 'house', 'football') + , @('dog', 'cat', 'tree', 'house', 'Football') + ) + $Sets = Compare-StringSet $ref $dif -CaseSensitive + $Sets.RefOnly + $Sets.DiffOnly + $Sets.Both + + This example sets up two arrays with some similar values and then passes them both to the Compare-StringSet function using the -CaseSensitive switch. The results of this are stored in the variable $Sets. + $Sets is an object that has three properties - RefOnly, DiffOnly, and Both. + + Because of the -CaseSensitive switch usage 'football' is shown as in RefOnly and 'Football' is shown as in DiffOnly. + + .NOTES + From https://gist.github.com/IISResetMe/57ce7b76e1001974a4f7170e10775875 + #> + + param( + [string[]]$Ref, + [string[]]$Diff, + + [switch]$CaseSensitive + ) + + $Comparer = if ($CaseSensitive) { + [System.StringComparer]::InvariantCulture + } + else { + [System.StringComparer]::InvariantCultureIgnoreCase + } + + $Results = [ordered]@{ + RefOnly = @() + Both = @() + DiffOnly = @() + } + + $temp = [System.Collections.Generic.HashSet[string]]::new($Ref, $Comparer) + $temp.IntersectWith($Diff) + $Results['Both'] = $temp + + #$temp = [System.Collections.Generic.HashSet[string]]::new($Ref, [System.StringComparer]::CurrentCultureIgnoreCase) + $temp = [System.Collections.Generic.HashSet[string]]::new($Ref, $Comparer) + $temp.ExceptWith($Diff) + $Results['RefOnly'] = $temp + + #$temp = [System.Collections.Generic.HashSet[string]]::new($Diff, [System.StringComparer]::CurrentCultureIgnoreCase) + $temp = [System.Collections.Generic.HashSet[string]]::new($Diff, $Comparer) + $temp.ExceptWith($Ref) + $Results['DiffOnly'] = $temp + + return [pscustomobject]$Results + } + + Write-Host " Comparing previous ($($importPrevious.Count)) with latest ($($resourcesIdsAll.Count))" + $start = get-date + $x = Compare-StringSet $importPrevious $resourcesIdsAll.id + Write-Host " unique values in previous (deleted):" $x.RefOnly.Count + Write-Host " values that are contained in previous and latest: $($x.Both.Count)" + Write-Host " unique values in latest (added):" $x.DiffOnly.Count + $end = get-date + Write-Host " Compare previous with latest duration: $((NEW-TIMESPAN -Start $start -End $end).TotalMinutes) mins ($((NEW-TIMESPAN -Start $start -End $end).TotalSeconds) sec)" + + $script:arrayResourceFluctuationFinal = [System.Collections.ArrayList]@() + + #ADDED + $arrayAdded = [System.Collections.ArrayList]@() + foreach ($resource in $x.DiffOnly) { + $resourceSplitted = $resource.split('/') + #$resourceSplitted + + $null = $arrayAdded.Add([PSCustomObject]@{ + subscriptionId = $resourceSplitted[2] + resourceType0 = $resourceSplitted[6] + resourceType1 = $resourceSplitted[7] + resourceType2 = $resourceSplitted[9] + resourceType3 = $resourceSplitted[11] + }) + if ($resourceSplitted.Count -gt 13) { + Write-Host " Unforeseen Resource type!" + Write-Host " Please report this Resource type at $($GithubRepository): '$resource'" + } + } + + if ($arrayAdded.Count -gt 0) { + $arrayGroupedByResourceType = $arrayAdded | group-object -Property resourceType0, resourceType1, resourceType2, resourceType3 + foreach ($resourceType in $arrayGroupedByResourceType) { + $arrayGroupedBySubscription = $arrayGroupedByResourceType.where({ $_.Name -eq $resourceType.Name }).Group | Group-Object -Property subscriptionId | Select-Object -ExcludeProperty Group + $null = $arrayResourceFluctuationFinal.Add([PSCustomObject]@{ + Event = 'Added' + ResourceType = $resourceType.Name + 'Resource count' = $resourceType.Count + 'Subscription count' = ($arrayGroupedBySubscription | Measure-Object).Count + }) + } + } + + #REMOVED + $arrayRemoved = [System.Collections.ArrayList]@() + foreach ($resource in $x.RefOnly) { + $resourceSplitted = $resource.split('/') + #$resourceSplitted + + $null = $arrayRemoved.Add([PSCustomObject]@{ + subscriptionId = $resourceSplitted[2] + resourceType0 = $resourceSplitted[6] + resourceType1 = $resourceSplitted[7] + resourceType2 = $resourceSplitted[9] + resourceType3 = $resourceSplitted[11] + }) + if ($resourceSplitted.Count -gt 13) { + Write-Host " Unforeseen Resource type!" + Write-Host " Please report this Resource type at $($GithubRepository): '$resource'" + } + } + + if ($arrayRemoved.Count -gt 0) { + $arrayGroupedByResourceType = $arrayRemoved | group-object -Property resourceType0, resourceType1, resourceType2, resourceType3 + foreach ($resourceType in $arrayGroupedByResourceType) { + $arrayGroupedBySubscription = $arrayGroupedByResourceType.where({ $_.Name -eq $resourceType.Name }).Group | Group-Object -Property subscriptionId | Select-Object -ExcludeProperty Group + $null = $arrayResourceFluctuationFinal.Add([PSCustomObject]@{ + Event = 'Removed' + ResourceType = $resourceType.Name + 'Resource count' = $resourceType.Count + 'Subscription count' = ($arrayGroupedBySubscription | Measure-Object).Count + }) + } + } + } + } + else { + Write-Host " Process Resource fluctuation skipped, no previous output (*$($ManagementGroupId)_ResourcesAll.csv) found" + } + + if ($arrayResourceFluctuationFinal.Count -gt 0 -and $doResourceFluctuation) { + #DataCollection Export of Resource fluctuation + Write-Host "Exporting ResourceFluctuation CSV '$($outputPath)$($DirectorySeparatorChar)$($fileName)_ResourceFluctuation.csv'" + $arrayResourceFluctuationFinal | Sort-Object -Property ResourceType | Export-Csv -Path "$($outputPath)$($DirectorySeparatorChar)$($fileName)_ResourceFluctuation.csv" -Delimiter "$csvDelimiter" -NoTypeInformation + } + Write-Host "Process Resource fluctuation duration: $((NEW-TIMESPAN -Start $start -End (get-date)).TotalSeconds) seconds" + #DataCollection Export of All Resources Write-Host "Exporting ResourcesAll CSV '$($outputPath)$($DirectorySeparatorChar)$($fileName)_ResourcesAll.csv'" $resourcesIdsAll | Sort-Object -Property id | Export-Csv -Path "$($outputPath)$($DirectorySeparatorChar)$($fileName)_ResourcesAll.csv" -Delimiter "$csvDelimiter" -NoTypeInformation @@ -7160,6 +7462,110 @@ extensions: [{ name: 'sort' }] #endregion ScopeInsightsResources } + #region ScopeInsightsOrphanedResources + if ($mgOrSub -eq 'sub') { + if ($arrayOrphanedResourcesGroupedBySubscription) { + $orphanedResourcesThisSubscription = $arrayOrphanedResourcesGroupedBySubscription.where({ $_.Name -eq $subscriptionId }) + if ($orphanedResourcesThisSubscription) { + $orphanedResourcesThisSubscriptionCount = $orphanedResourcesThisSubscription.Group.count + $orphanedResourcesThisSubscriptionGroupedByType = $orphanedResourcesThisSubscription.Group | Group-Object -Property type + $orphanedResourcesThisSubscriptionGroupedByTypeCount = ($orphanedResourcesThisSubscriptionGroupedByType | Measure-Object).Count + $tfCount = $orphanedResourcesThisSubscriptionGroupedByTypeCount + $htmlTableId = "ScopeInsights_OrphanedResources_$($subscriptionId -replace '-','_')" + $randomFunctionName = "func_$htmlTableId" + [void]$htmlScopeInsights.AppendLine(@" + +
ResourceType | +Resource count | +Intent | +
---|---|---|
$($resourceType.Name) | +$($resourceType.Group.Count) | +$($resourceType.Group[0].Intent) | +
No Resources orphaned
+"@) + } + } + else { + [void]$htmlScopeInsights.AppendLine(@" +No Resources orphaned
+"@) + } + [void]$htmlScopeInsights.AppendLine(@' + +Event | +ResourceType | +Resource count | +Subscription count | +
---|---|---|---|
$($entry.Event) | +$($entry.ResourceType) | +$($entry.'Resource count') | +$($entry.'Subscription count') | +
No Resource fluctuation since last run
+'@) + } + $endSUMMARYResourceFluctuation = Get-Date + Write-Host " SUMMARY Resource fluctuation processing duration: $((NEW-TIMESPAN -Start $startSUMMARYResourceFluctuation -End $endSUMMARYResourceFluctuation).TotalMinutes) minutes ($((NEW-TIMESPAN -Start $startSUMMARYResourceFluctuation -End $endSUMMARYResourceFluctuation).TotalSeconds) seconds)" + #endregion SUMMARYResourceFluctuation } + #region SUMMARYOrphanedResources + $startSUMMARYOrphanedResources = Get-Date + Write-Host ' processing TenantSummary Orphaned Resources' + if ($arrayOrphanedResources.count -gt 0) { + $script:arrayOrphanedResourcesSlim = $arrayOrphanedResources | Sort-Object -Property type | Select-Object -Property type, subscriptionId, intent + $arrayOrphanedResourcesGroupedByType = $arrayOrphanedResourcesSlim | Group-Object type + $orphanedResourceTypesCount = ($arrayOrphanedResourcesGroupedByType | Measure-Object).Count + + $tfCount = $orphanedResourceTypesCount + $htmlTableId = 'TenantSummary_orphanedResources' + [void]$htmlTenantSummary.AppendLine(@" + +ResourceType | +Resource count | +Subscriptions count | +Intent | +
---|---|---|---|
$($orphanedResourceType.Name) | +$($orphanedResourceType.count) | +$(($orphanedResourceType.Group.SubscriptionId | Sort-Object -Unique).Count) | +$($orphanedResourceType.Group[0].Intent) | +
No Resources orphaned
+'@) + } + $endSUMMARYOrphanedResources = Get-Date + Write-Host " SUMMARY Orphaned Resources processing duration: $((NEW-TIMESPAN -Start $startSUMMARYOrphanedResources -End $endSUMMARYOrphanedResources).TotalMinutes) minutes ($((NEW-TIMESPAN -Start $startSUMMARYOrphanedResources -End $endSUMMARYOrphanedResources).TotalSeconds) seconds)" + #endregion SUMMARYOrphanedResources + #region SUMMARYSubResourceProviders $startSUMMARYSubResourceProviders = Get-Date Write-Host ' processing TenantSummary Subscriptions Resource Providers' @@ -22631,7 +23233,6 @@ function verifyModules3rd { $installModuleSuccess = $false try { - if (-not $moduleVersion) { Write-Host ' Check latest module version' try { @@ -22645,17 +23246,23 @@ function verifyModules3rd { } if (-not $installModuleSuccess) { - $moduleVersionLoaded = (Get-InstalledModule -name $($module.ModuleName)).Version - if ($moduleVersionLoaded -eq $moduleVersion) { - $installModuleSuccess = $true + try { + $moduleVersionLoaded = (Get-InstalledModule -name $($module.ModuleName)).Version + if ($moduleVersionLoaded -eq $moduleVersion) { + $installModuleSuccess = $true + } + else { + Write-Host " Deviating module version $moduleVersionLoaded" + throw + } } - else { - throw " '(Get-InstalledModule -name $($module.ModuleName)).Version' returned null" + catch { + throw } } } catch { - Write-Host " '$($module.ModuleName)' not installed" + Write-Host " '$($module.ModuleName) $moduleVersion' not installed" if (($env:SYSTEM_TEAMPROJECTID -and $env:BUILD_REPOSITORY_ID) -or $env:GITHUB_ACTIONS) { Write-Host " Installing $($module.ModuleName) module ($($moduleVersion))" try { @@ -22687,9 +23294,15 @@ function verifyModules3rd { if ($installModuleUserChoice -eq 'y') { try { Install-Module -Name $module.ModuleName -RequiredVersion $moduleVersion + try { + Import-Module -Name $module.ModuleName -RequiredVersion $moduleVersion -Force + } + catch { + throw " 'Import-Module -Name $($module.ModuleName) -RequiredVersion $moduleVersion -Force' failed" + } } catch { - throw " 'Install-Module -Name $module.ModuleName -RequiredVersion $moduleVersion' failed" + throw " 'Install-Module -Name $($module.ModuleName) -RequiredVersion $moduleVersion' failed" } } elseif ($installModuleUserChoice -eq 'n') { @@ -25077,7 +25690,7 @@ function dataCollectionRoleAssignmentsMG { $method = 'GET' $roleAssignmentScheduleInstancesFromAPI = AzAPICall -AzAPICallConfiguration $azAPICallConf -uri $uri -method $method -currentTask $currentTask -caller 'CustomDataCollection' - if ($roleAssignmentScheduleInstancesFromAPI -eq 'ResourceNotOnboarded' -or $roleAssignmentScheduleInstancesFromAPI -eq 'TenantNotOnboarded' -or $roleAssignmentScheduleInstancesFromAPI -eq 'InvalidResourceType') { + if ($roleAssignmentScheduleInstancesFromAPI -eq 'ResourceNotOnboarded' -or $roleAssignmentScheduleInstancesFromAPI -eq 'TenantNotOnboarded' -or $roleAssignmentScheduleInstancesFromAPI -eq 'InvalidResourceType' -or $roleAssignmentScheduleInstancesFromAPI -eq 'RoleAssignmentScheduleInstancesError') { #Write-Host "Scope '$($scopeDisplayName)' ('$scopeId') not onboarded in PIM" } else { @@ -25353,7 +25966,7 @@ function dataCollectionRoleAssignmentsSub { $method = 'GET' $roleAssignmentScheduleInstancesFromAPI = AzAPICall -AzAPICallConfiguration $azAPICallConf -uri $uri -method $method -currentTask $currentTask -caller 'CustomDataCollection' - if ($roleAssignmentScheduleInstancesFromAPI -eq 'ResourceNotOnboarded' -or $roleAssignmentScheduleInstancesFromAPI -eq 'TenantNotOnboarded' -or $roleAssignmentScheduleInstancesFromAPI -eq 'InvalidResourceType') { + if ($roleAssignmentScheduleInstancesFromAPI -eq 'ResourceNotOnboarded' -or $roleAssignmentScheduleInstancesFromAPI -eq 'TenantNotOnboarded' -or $roleAssignmentScheduleInstancesFromAPI -eq 'InvalidResourceType' -or $roleAssignmentScheduleInstancesFromAPI -eq 'RoleAssignmentScheduleInstancesError') { #Write-Host "Scope '$($scopeDisplayName)' ('$scopeId') not onboarded in PIM" } else { @@ -26266,6 +26879,7 @@ if ($azAPICallConf['htParameters'].HierarchyMapOnly -eq $false) { $arrayPsRule = [System.Collections.ArrayList]::Synchronized((New-Object System.Collections.ArrayList)) $arrayPSRuleTracking = [System.Collections.ArrayList]::Synchronized((New-Object System.Collections.ArrayList)) $htClassicAdministrators = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} + $arrayOrphanedResources = [System.Collections.ArrayList]::Synchronized((New-Object System.Collections.ArrayList)) } getEntities @@ -26287,6 +26901,8 @@ if ($azAPICallConf['htParameters'].HierarchyMapOnly -eq $false) { getSubscriptions detailSubscriptions showMemoryUsage + getOrphanedResources + showMemoryUsage if ($azAPICallConf['htParameters'].NoMDfCSecureScore -eq $false) { getMDfCSecureScoreMG @@ -27239,6 +27855,9 @@ if ($azAPICallConf['htParameters'].HierarchyMapOnly -eq $false) { if ($arrayFeaturesAll.Count -gt 0) { $script:subFeaturesGroupedBySubscription = $arrayFeaturesAll | Group-Object -property subscriptionId } + if ($arrayOrphanedResourcesSlim.Count -gt 0) { + $arrayOrphanedResourcesGroupedBySubscription = $arrayOrphanedResourcesSlim | Group-Object subscriptionId + } processScopeInsights -mgChild $ManagementGroupId -mgChildOf $getMgParentId showMemoryUsage #[System.GC]::Collect() diff --git a/pwsh/dev/devAzGovVizParallel.ps1 b/pwsh/dev/devAzGovVizParallel.ps1 index 8498c3f7..22998421 100644 --- a/pwsh/dev/devAzGovVizParallel.ps1 +++ b/pwsh/dev/devAzGovVizParallel.ps1 @@ -283,10 +283,10 @@ Param $Product = 'AzGovViz', [string] - $AzAPICallVersion = '1.1.16', + $AzAPICallVersion = '1.1.17', [string] - $ProductVersion = 'v6_major_20220614_1', + $ProductVersion = 'v6_major_20220622_1', [string] $GithubRepository = 'aka.ms/AzGovViz', @@ -511,6 +511,7 @@ Write-Host "Start AzGovViz $($startTime) (#$($ProductVersion))" . ".\$($ScriptPath)\functions\processHierarchyMapOnly.ps1" . ".\$($ScriptPath)\functions\getSubscriptions.ps1" . ".\$($ScriptPath)\functions\detailSubscriptions.ps1" +. ".\$($ScriptPath)\functions\getOrphanedResources.ps1" . ".\$($ScriptPath)\functions\getMDfCSecureScoreMG.ps1" . ".\$($ScriptPath)\functions\getConsumption.ps1" . ".\$($ScriptPath)\functions\cacheBuiltIn.ps1" @@ -745,6 +746,7 @@ if ($azAPICallConf['htParameters'].HierarchyMapOnly -eq $false) { $arrayPsRule = [System.Collections.ArrayList]::Synchronized((New-Object System.Collections.ArrayList)) $arrayPSRuleTracking = [System.Collections.ArrayList]::Synchronized((New-Object System.Collections.ArrayList)) $htClassicAdministrators = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} + $arrayOrphanedResources = [System.Collections.ArrayList]::Synchronized((New-Object System.Collections.ArrayList)) } getEntities @@ -766,6 +768,8 @@ if ($azAPICallConf['htParameters'].HierarchyMapOnly -eq $false) { getSubscriptions detailSubscriptions showMemoryUsage + getOrphanedResources + showMemoryUsage if ($azAPICallConf['htParameters'].NoMDfCSecureScore -eq $false) { getMDfCSecureScoreMG @@ -1718,6 +1722,9 @@ if ($azAPICallConf['htParameters'].HierarchyMapOnly -eq $false) { if ($arrayFeaturesAll.Count -gt 0) { $script:subFeaturesGroupedBySubscription = $arrayFeaturesAll | Group-Object -property subscriptionId } + if ($arrayOrphanedResourcesSlim.Count -gt 0) { + $arrayOrphanedResourcesGroupedBySubscription = $arrayOrphanedResourcesSlim | Group-Object subscriptionId + } processScopeInsights -mgChild $ManagementGroupId -mgChildOf $getMgParentId showMemoryUsage #[System.GC]::Collect() diff --git a/pwsh/dev/functions/dataCollection/dataCollectionFunctions.ps1 b/pwsh/dev/functions/dataCollection/dataCollectionFunctions.ps1 index 015d4311..2aec2fca 100644 --- a/pwsh/dev/functions/dataCollection/dataCollectionFunctions.ps1 +++ b/pwsh/dev/functions/dataCollection/dataCollectionFunctions.ps1 @@ -2368,7 +2368,7 @@ function dataCollectionRoleAssignmentsMG { $method = 'GET' $roleAssignmentScheduleInstancesFromAPI = AzAPICall -AzAPICallConfiguration $azAPICallConf -uri $uri -method $method -currentTask $currentTask -caller 'CustomDataCollection' - if ($roleAssignmentScheduleInstancesFromAPI -eq 'ResourceNotOnboarded' -or $roleAssignmentScheduleInstancesFromAPI -eq 'TenantNotOnboarded' -or $roleAssignmentScheduleInstancesFromAPI -eq 'InvalidResourceType') { + if ($roleAssignmentScheduleInstancesFromAPI -eq 'ResourceNotOnboarded' -or $roleAssignmentScheduleInstancesFromAPI -eq 'TenantNotOnboarded' -or $roleAssignmentScheduleInstancesFromAPI -eq 'InvalidResourceType' -or $roleAssignmentScheduleInstancesFromAPI -eq 'RoleAssignmentScheduleInstancesError') { #Write-Host "Scope '$($scopeDisplayName)' ('$scopeId') not onboarded in PIM" } else { @@ -2644,7 +2644,7 @@ function dataCollectionRoleAssignmentsSub { $method = 'GET' $roleAssignmentScheduleInstancesFromAPI = AzAPICall -AzAPICallConfiguration $azAPICallConf -uri $uri -method $method -currentTask $currentTask -caller 'CustomDataCollection' - if ($roleAssignmentScheduleInstancesFromAPI -eq 'ResourceNotOnboarded' -or $roleAssignmentScheduleInstancesFromAPI -eq 'TenantNotOnboarded' -or $roleAssignmentScheduleInstancesFromAPI -eq 'InvalidResourceType') { + if ($roleAssignmentScheduleInstancesFromAPI -eq 'ResourceNotOnboarded' -or $roleAssignmentScheduleInstancesFromAPI -eq 'TenantNotOnboarded' -or $roleAssignmentScheduleInstancesFromAPI -eq 'InvalidResourceType' -or $roleAssignmentScheduleInstancesFromAPI -eq 'RoleAssignmentScheduleInstancesError') { #Write-Host "Scope '$($scopeDisplayName)' ('$scopeId') not onboarded in PIM" } else { diff --git a/pwsh/dev/functions/getMDfCSecureScoreMG.ps1 b/pwsh/dev/functions/getMDfCSecureScoreMG.ps1 index 3c4dedd4..15d9fa5c 100644 --- a/pwsh/dev/functions/getMDfCSecureScoreMG.ps1 +++ b/pwsh/dev/functions/getMDfCSecureScoreMG.ps1 @@ -41,7 +41,7 @@ function getMDfCSecureScoreMG { Write-Host ' Microsoft Defender for Cloud SecureScore for Management Groups will not be available' -ForegroundColor Yellow } else { - foreach ($entry in $getMgAscSecureScore.data) { + foreach ($entry in $getMgAscSecureScore) { $script:htMgASCSecureScore.($entry.mgId) = @{} if ($entry.secureScore -eq 404) { $script:htMgASCSecureScore.($entry.mgId).SecureScore = 'n/a' diff --git a/pwsh/dev/functions/getOrphanedResources.ps1 b/pwsh/dev/functions/getOrphanedResources.ps1 new file mode 100644 index 00000000..27545e2a --- /dev/null +++ b/pwsh/dev/functions/getOrphanedResources.ps1 @@ -0,0 +1,104 @@ +function getOrphanedResources { + $start = Get-Date + Write-Host 'Getting orphaned resources (ARG)' + + $queries = [System.Collections.ArrayList]@() + $intent = 'clean up' + $null = $queries.Add([PSCustomObject]@{ + queryName = 'microsoft.resources/subscriptions/resourceGroups' + query = "ResourceContainers | where type =~ 'microsoft.resources/subscriptions/resourceGroups' | extend rgAndSub = strcat(resourceGroup, '--', subscriptionId) | join kind=leftouter (Resources | extend rgAndSub = strcat(resourceGroup, '--', subscriptionId) | summarize count() by rgAndSub) on rgAndSub | where isnull(count_) | project type, subscriptionId, Resource=id, Intent='$intent'" + }) + + $intent = 'misconfiguration' + $null = $queries.Add([PSCustomObject]@{ + queryName = 'microsoft.network/networkSecurityGroups' + query = "Resources | where type =~ 'microsoft.network/networkSecurityGroups' and isnull(properties.networkInterfaces) and isnull(properties.subnets) | project type, subscriptionId, Resource=id, Intent='$intent'" + }) + + $intent = 'misconfiguration' + $null = $queries.Add([PSCustomObject]@{ + queryName = 'microsoft.network/routeTables' + query = "resources | where type =~ 'microsoft.network/routeTables' | where isnull(properties.subnets) | project type, subscriptionId, Resource=id, Intent='$intent'" + }) + + $intent = 'misconfiguration' + $null = $queries.Add([PSCustomObject]@{ + queryName = 'microsoft.network/networkInterfaces' + query = "Resources | where type =~ 'microsoft.network/networkInterfaces' | where isnull(properties.privateEndpoint) | where isnull(properties.privateLinkService) | where properties.hostedWorkloads == '[]' | where properties !has 'virtualmachine' | project type, subscriptionId, Resource=id, Intent='$intent'" + }) + + $intent = 'cost savings' + $null = $queries.Add([PSCustomObject]@{ + queryName = 'microsoft.compute/disks' + query = "Resources | where type =~ 'microsoft.compute/disks' | extend diskState = tostring(properties.diskState) | where managedBy == '' | where not(name endswith '-ASRReplica' or name startswith 'ms-asr-') | project type, subscriptionId, Resource=id, Intent='$intent'" + }) + + $intent = 'cost savings' + $null = $queries.Add([PSCustomObject]@{ + queryName = 'microsoft.network/publicIpAddresses' + query = "Resources | where type =~ 'microsoft.network/publicIpAddresses' | where properties.ipConfiguration == '' | project type, subscriptionId, Resource=id, Intent='$intent'" + }) + + $intent = 'misconfiguration' + $null = $queries.Add([PSCustomObject]@{ + queryName = 'microsoft.compute/availabilitySets' + query = "Resources | where type =~ 'microsoft.compute/availabilitySets' | where properties.virtualMachines == '[]' | project type, subscriptionId, Resource=id, Intent='$intent'" + }) + + $intent = 'misconfiguration' + $null = $queries.Add([PSCustomObject]@{ + queryName = 'microsoft.network/loadBalancers' + query = "Resources | where type =~ 'microsoft.network/loadBalancers' | where properties.backendAddressPools == '[]' | project type, subscriptionId, Resource=id, Intent='$intent'" + }) + + $intent = 'clean up' + $null = $queries.Add([PSCustomObject]@{ + queryName = 'microsoft.web/serverfarms' + query = "Resources | where type =~ 'microsoft.web/serverfarms' | where properties.numberOfSites == 0 | project type, subscriptionId, Resource=id, Intent='$intent'" + }) + + $queries | foreach-object -Parallel { + $queryDetail = $_ + $arrayOrphanedResources = $using:arrayOrphanedResources + $subsToProcessInCustomDataCollection = $using:subsToProcessInCustomDataCollection + $azAPICallConf = $using:azAPICallConf + + #Batching: https://docs.microsoft.com/en-us/azure/governance/resource-graph/troubleshoot/general#toomanysubscription + $counterBatch = [PSCustomObject] @{ Value = 0 } + $batchSize = 1000 + $subscriptionsBatch = $subsToProcessInCustomDataCollection | Group-Object -Property { [math]::Floor($counterBatch.Value++ / $batchSize) } + + $uri = "$($azAPICallConf['azAPIEndpointUrls'].ARM)/providers/Microsoft.ResourceGraph/resources?api-version=2021-03-01" + $method = "POST" + foreach ($batch in $subscriptionsBatch) { + " Getting orphaned $($queryDetail.queryName) for $($batch.Group.subscriptionId.Count) Subscriptions" + $subscriptions = '"{0}"' -f ($batch.Group.subscriptionId -join '","') + $body = @" +{ +"query": "$($queryDetail.query)", +"subscriptions": [$($subscriptions)] +} +"@ + + $res = (AzAPICall -AzAPICallConfiguration $azAPICallConf -uri $uri -method $method -body $body -listenOn 'Content' -currentTask "Getting orphaned $($queryDetail.queryName)") + #Write-Host '$res.count:' $res.count + if ($res.count -gt 0) { + foreach ($resource in $res) { + $null = $script:arrayOrphanedResources.Add($resource) + } + } + } + } -ThrottleLimit ($queries.Count) + + if ($arrayOrphanedResources.Count -gt 0) { + Write-Host " Found $($arrayOrphanedResources.Count) orphaned Resources" + Write-Host " Exporting OrphanedResources CSV '$($outputPath)$($DirectorySeparatorChar)$($fileName)_ResourcesOrphaned.csv'" + $arrayOrphanedResources | Export-Csv -Path "$($outputPath)$($DirectorySeparatorChar)$($fileName)_ResourcesOrphaned.csv" -Delimiter "$csvDelimiter" -NoTypeInformation + } + else { + Write-Host " No orphaned Resources found" + } + + $end = Get-Date + Write-Host "Getting orphaned resources (ARG) processing duration: $((NEW-TIMESPAN -Start $start -End $end).TotalMinutes) minutes ($((NEW-TIMESPAN -Start $start -End $end).TotalSeconds) seconds)" +} \ No newline at end of file diff --git a/pwsh/dev/functions/processDataCollection.ps1 b/pwsh/dev/functions/processDataCollection.ps1 index 576d7204..3656f627 100644 --- a/pwsh/dev/functions/processDataCollection.ps1 +++ b/pwsh/dev/functions/processDataCollection.ps1 @@ -579,6 +579,204 @@ function processDataCollection { if (-not $azAPICallConf['htParameters'].HierarchyMapOnly -and -not $azAPICallConf['htParameters'].ManagementGroupsOnly) { if (-not $NoCsvExport) { + + #fluctuation + Write-Host "Process Resource fluctuation" + $start = get-date + if (Test-Path -Filter "*$($ManagementGroupId)_ResourcesAll.csv" -LiteralPath "$($outputPath)") { + $startImportPrevious = get-date + $doResourceFluctuation = $true + + try { + $previous = Get-ChildItem -Path $outputPath -Filter "*$($ManagementGroupId)_ResourcesAll.csv" | Sort-Object -Descending -Property LastWriteTime | Select-Object -First 1 -ErrorAction Stop + $importPrevious = Import-Csv -LiteralPath "$($outputPath)$($DirectorySeparatorChar)$($previous.Name)" -Encoding utf8 -Delimiter ";" | Select-Object -ExpandProperty id + Write-Host " Import previous ($($previous.Name)) duration: $((NEW-TIMESPAN -Start $startImportPrevious -End (get-date)).TotalSeconds)" + } + catch { + Write-Host " FAILED: Import-Csv '$($outputPath)$($DirectorySeparatorChar)$($previous.Name)'" + $doResourceFluctuation = $false + + } + + if ($doResourceFluctuation) { + #$importPrevious.Count + + #https://gist.github.com/fatherjack/4c91cc6832b8b02d1b7319716a5fba52 + function Compare-StringSet { + <# + .SYNOPSIS + Compare two sets of strings and see the matched and unmatched elements from each input + + .DESCRIPTION + Compares sets of + + .PARAMETER Ref + The reference set of values to be compared + + .PARAMETER Diff + The difference set of values to be compared + + .PARAMETER CaseSensitive + Enables a case-sensitive comparison + + .EXAMPLE + $ref, $dif = @( + , @('a', 'b', 'c') + , @('b', 'c', 'd') + ) + $Sets = Compare-StringSet $ref $dif + $Sets.RefOnly + + $Sets.DiffOnly + + $Sets.Both + + This example sets up two arrays with some similar values and then passes them both to the Compare-StringSet function. the results of this are stored in the variable $Sets. + $Sets is an object that has three properties - RefOnly, DiffOnly, and Both. These are sets of the incoming values where they intersect or not. + + .EXAMPLE + $ref, $dif = @( + , @('tree', 'house', 'football') + , @('dog', 'cat', 'tree', 'house', 'Football') + ) + $Sets = Compare-StringSet $ref $dif -CaseSensitive + $Sets.RefOnly + $Sets.DiffOnly + $Sets.Both + + This example sets up two arrays with some similar values and then passes them both to the Compare-StringSet function using the -CaseSensitive switch. The results of this are stored in the variable $Sets. + $Sets is an object that has three properties - RefOnly, DiffOnly, and Both. + + Because of the -CaseSensitive switch usage 'football' is shown as in RefOnly and 'Football' is shown as in DiffOnly. + + .NOTES + From https://gist.github.com/IISResetMe/57ce7b76e1001974a4f7170e10775875 + #> + + param( + [string[]]$Ref, + [string[]]$Diff, + + [switch]$CaseSensitive + ) + + $Comparer = if ($CaseSensitive) { + [System.StringComparer]::InvariantCulture + } + else { + [System.StringComparer]::InvariantCultureIgnoreCase + } + + $Results = [ordered]@{ + RefOnly = @() + Both = @() + DiffOnly = @() + } + + $temp = [System.Collections.Generic.HashSet[string]]::new($Ref, $Comparer) + $temp.IntersectWith($Diff) + $Results['Both'] = $temp + + #$temp = [System.Collections.Generic.HashSet[string]]::new($Ref, [System.StringComparer]::CurrentCultureIgnoreCase) + $temp = [System.Collections.Generic.HashSet[string]]::new($Ref, $Comparer) + $temp.ExceptWith($Diff) + $Results['RefOnly'] = $temp + + #$temp = [System.Collections.Generic.HashSet[string]]::new($Diff, [System.StringComparer]::CurrentCultureIgnoreCase) + $temp = [System.Collections.Generic.HashSet[string]]::new($Diff, $Comparer) + $temp.ExceptWith($Ref) + $Results['DiffOnly'] = $temp + + return [pscustomobject]$Results + } + + Write-Host " Comparing previous ($($importPrevious.Count)) with latest ($($resourcesIdsAll.Count))" + $start = get-date + $x = Compare-StringSet $importPrevious $resourcesIdsAll.id + Write-Host " unique values in previous (deleted):" $x.RefOnly.Count + Write-Host " values that are contained in previous and latest: $($x.Both.Count)" + Write-Host " unique values in latest (added):" $x.DiffOnly.Count + $end = get-date + Write-Host " Compare previous with latest duration: $((NEW-TIMESPAN -Start $start -End $end).TotalMinutes) mins ($((NEW-TIMESPAN -Start $start -End $end).TotalSeconds) sec)" + + $script:arrayResourceFluctuationFinal = [System.Collections.ArrayList]@() + + #ADDED + $arrayAdded = [System.Collections.ArrayList]@() + foreach ($resource in $x.DiffOnly) { + $resourceSplitted = $resource.split('/') + #$resourceSplitted + + $null = $arrayAdded.Add([PSCustomObject]@{ + subscriptionId = $resourceSplitted[2] + resourceType0 = $resourceSplitted[6] + resourceType1 = $resourceSplitted[7] + resourceType2 = $resourceSplitted[9] + resourceType3 = $resourceSplitted[11] + }) + if ($resourceSplitted.Count -gt 13) { + Write-Host " Unforeseen Resource type!" + Write-Host " Please report this Resource type at $($GithubRepository): '$resource'" + } + } + + if ($arrayAdded.Count -gt 0) { + $arrayGroupedByResourceType = $arrayAdded | group-object -Property resourceType0, resourceType1, resourceType2, resourceType3 + foreach ($resourceType in $arrayGroupedByResourceType) { + $arrayGroupedBySubscription = $arrayGroupedByResourceType.where({ $_.Name -eq $resourceType.Name }).Group | Group-Object -Property subscriptionId | Select-Object -ExcludeProperty Group + $null = $arrayResourceFluctuationFinal.Add([PSCustomObject]@{ + Event = 'Added' + ResourceType = $resourceType.Name + 'Resource count' = $resourceType.Count + 'Subscription count' = ($arrayGroupedBySubscription | Measure-Object).Count + }) + } + } + + #REMOVED + $arrayRemoved = [System.Collections.ArrayList]@() + foreach ($resource in $x.RefOnly) { + $resourceSplitted = $resource.split('/') + #$resourceSplitted + + $null = $arrayRemoved.Add([PSCustomObject]@{ + subscriptionId = $resourceSplitted[2] + resourceType0 = $resourceSplitted[6] + resourceType1 = $resourceSplitted[7] + resourceType2 = $resourceSplitted[9] + resourceType3 = $resourceSplitted[11] + }) + if ($resourceSplitted.Count -gt 13) { + Write-Host " Unforeseen Resource type!" + Write-Host " Please report this Resource type at $($GithubRepository): '$resource'" + } + } + + if ($arrayRemoved.Count -gt 0) { + $arrayGroupedByResourceType = $arrayRemoved | group-object -Property resourceType0, resourceType1, resourceType2, resourceType3 + foreach ($resourceType in $arrayGroupedByResourceType) { + $arrayGroupedBySubscription = $arrayGroupedByResourceType.where({ $_.Name -eq $resourceType.Name }).Group | Group-Object -Property subscriptionId | Select-Object -ExcludeProperty Group + $null = $arrayResourceFluctuationFinal.Add([PSCustomObject]@{ + Event = 'Removed' + ResourceType = $resourceType.Name + 'Resource count' = $resourceType.Count + 'Subscription count' = ($arrayGroupedBySubscription | Measure-Object).Count + }) + } + } + } + } + else { + Write-Host " Process Resource fluctuation skipped, no previous output (*$($ManagementGroupId)_ResourcesAll.csv) found" + } + + if ($arrayResourceFluctuationFinal.Count -gt 0 -and $doResourceFluctuation) { + #DataCollection Export of Resource fluctuation + Write-Host "Exporting ResourceFluctuation CSV '$($outputPath)$($DirectorySeparatorChar)$($fileName)_ResourceFluctuation.csv'" + $arrayResourceFluctuationFinal | Sort-Object -Property ResourceType | Export-Csv -Path "$($outputPath)$($DirectorySeparatorChar)$($fileName)_ResourceFluctuation.csv" -Delimiter "$csvDelimiter" -NoTypeInformation + } + Write-Host "Process Resource fluctuation duration: $((NEW-TIMESPAN -Start $start -End (get-date)).TotalSeconds) seconds" + #DataCollection Export of All Resources Write-Host "Exporting ResourcesAll CSV '$($outputPath)$($DirectorySeparatorChar)$($fileName)_ResourcesAll.csv'" $resourcesIdsAll | Sort-Object -Property id | Export-Csv -Path "$($outputPath)$($DirectorySeparatorChar)$($fileName)_ResourcesAll.csv" -Delimiter "$csvDelimiter" -NoTypeInformation diff --git a/pwsh/dev/functions/processScopeInsightsMgOrSub.ps1 b/pwsh/dev/functions/processScopeInsightsMgOrSub.ps1 index fbfd2ce0..7b4b9213 100644 --- a/pwsh/dev/functions/processScopeInsightsMgOrSub.ps1 +++ b/pwsh/dev/functions/processScopeInsightsMgOrSub.ps1 @@ -1575,6 +1575,110 @@ extensions: [{ name: 'sort' }] #endregion ScopeInsightsResources } + #region ScopeInsightsOrphanedResources + if ($mgOrSub -eq 'sub') { + if ($arrayOrphanedResourcesGroupedBySubscription) { + $orphanedResourcesThisSubscription = $arrayOrphanedResourcesGroupedBySubscription.where({ $_.Name -eq $subscriptionId }) + if ($orphanedResourcesThisSubscription) { + $orphanedResourcesThisSubscriptionCount = $orphanedResourcesThisSubscription.Group.count + $orphanedResourcesThisSubscriptionGroupedByType = $orphanedResourcesThisSubscription.Group | Group-Object -Property type + $orphanedResourcesThisSubscriptionGroupedByTypeCount = ($orphanedResourcesThisSubscriptionGroupedByType | Measure-Object).Count + $tfCount = $orphanedResourcesThisSubscriptionGroupedByTypeCount + $htmlTableId = "ScopeInsights_OrphanedResources_$($subscriptionId -replace '-','_')" + $randomFunctionName = "func_$htmlTableId" + [void]$htmlScopeInsights.AppendLine(@" + +ResourceType | +Resource count | +Intent | +
---|---|---|
$($resourceType.Name) | +$($resourceType.Group.Count) | +$($resourceType.Group[0].Intent) | +
No Resources orphaned
+"@) + } + } + else { + [void]$htmlScopeInsights.AppendLine(@" +No Resources orphaned
+"@) + } + [void]$htmlScopeInsights.AppendLine(@' +Event | +ResourceType | +Resource count | +Subscription count | +
---|---|---|---|
$($entry.Event) | +$($entry.ResourceType) | +$($entry.'Resource count') | +$($entry.'Subscription count') | +
No Resource fluctuation since last run
+'@) + } + $endSUMMARYResourceFluctuation = Get-Date + Write-Host " SUMMARY Resource fluctuation processing duration: $((NEW-TIMESPAN -Start $startSUMMARYResourceFluctuation -End $endSUMMARYResourceFluctuation).TotalMinutes) minutes ($((NEW-TIMESPAN -Start $startSUMMARYResourceFluctuation -End $endSUMMARYResourceFluctuation).TotalSeconds) seconds)" + #endregion SUMMARYResourceFluctuation + } + + #region SUMMARYOrphanedResources + $startSUMMARYOrphanedResources = Get-Date + Write-Host ' processing TenantSummary Orphaned Resources' + if ($arrayOrphanedResources.count -gt 0) { + $script:arrayOrphanedResourcesSlim = $arrayOrphanedResources | Sort-Object -Property type | Select-Object -Property type, subscriptionId, intent + $arrayOrphanedResourcesGroupedByType = $arrayOrphanedResourcesSlim | Group-Object type + $orphanedResourceTypesCount = ($arrayOrphanedResourcesGroupedByType | Measure-Object).Count + + $tfCount = $orphanedResourceTypesCount + $htmlTableId = 'TenantSummary_orphanedResources' + [void]$htmlTenantSummary.AppendLine(@" + +ResourceType | +Resource count | +Subscriptions count | +Intent | +
---|---|---|---|
$($orphanedResourceType.Name) | +$($orphanedResourceType.count) | +$(($orphanedResourceType.Group.SubscriptionId | Sort-Object -Unique).Count) | +$($orphanedResourceType.Group[0].Intent) | +
No Resources orphaned
+'@) } + $endSUMMARYOrphanedResources = Get-Date + Write-Host " SUMMARY Orphaned Resources processing duration: $((NEW-TIMESPAN -Start $startSUMMARYOrphanedResources -End $endSUMMARYOrphanedResources).TotalMinutes) minutes ($((NEW-TIMESPAN -Start $startSUMMARYOrphanedResources -End $endSUMMARYOrphanedResources).TotalSeconds) seconds)" + #endregion SUMMARYOrphanedResources #region SUMMARYSubResourceProviders $startSUMMARYSubResourceProviders = Get-Date diff --git a/pwsh/dev/functions/verifyModules3rd.ps1 b/pwsh/dev/functions/verifyModules3rd.ps1 index f9a1beab..73202387 100644 --- a/pwsh/dev/functions/verifyModules3rd.ps1 +++ b/pwsh/dev/functions/verifyModules3rd.ps1 @@ -24,7 +24,6 @@ function verifyModules3rd { $installModuleSuccess = $false try { - if (-not $moduleVersion) { Write-Host ' Check latest module version' try { @@ -38,17 +37,23 @@ function verifyModules3rd { } if (-not $installModuleSuccess) { - $moduleVersionLoaded = (Get-InstalledModule -name $($module.ModuleName)).Version - if ($moduleVersionLoaded -eq $moduleVersion) { - $installModuleSuccess = $true + try { + $moduleVersionLoaded = (Get-InstalledModule -name $($module.ModuleName)).Version + if ($moduleVersionLoaded -eq $moduleVersion) { + $installModuleSuccess = $true + } + else { + Write-Host " Deviating module version $moduleVersionLoaded" + throw + } } - else { - throw " '(Get-InstalledModule -name $($module.ModuleName)).Version' returned null" + catch { + throw } } } catch { - Write-Host " '$($module.ModuleName)' not installed" + Write-Host " '$($module.ModuleName) $moduleVersion' not installed" if (($env:SYSTEM_TEAMPROJECTID -and $env:BUILD_REPOSITORY_ID) -or $env:GITHUB_ACTIONS) { Write-Host " Installing $($module.ModuleName) module ($($moduleVersion))" try { @@ -80,9 +85,15 @@ function verifyModules3rd { if ($installModuleUserChoice -eq 'y') { try { Install-Module -Name $module.ModuleName -RequiredVersion $moduleVersion + try { + Import-Module -Name $module.ModuleName -RequiredVersion $moduleVersion -Force + } + catch { + throw " 'Import-Module -Name $($module.ModuleName) -RequiredVersion $moduleVersion -Force' failed" + } } catch { - throw " 'Install-Module -Name $module.ModuleName -RequiredVersion $moduleVersion' failed" + throw " 'Install-Module -Name $($module.ModuleName) -RequiredVersion $moduleVersion' failed" } } elseif ($installModuleUserChoice -eq 'n') { diff --git a/version.txt b/version.txt index daa645a8..c9c380ad 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -v6_major_20220614_1 \ No newline at end of file +v6_major_20220622_1 \ No newline at end of file