Require PowerShell 7 + handle throttling errors

.build/BuildHelper/Build-PsModule.ps1

@@ -54,7 +54,7 @@ function Build-PsModule
         VariablesToExport = @()
         AliasesToExport   = @()
         Copyright         = "(c) $((Get-Date).Year) Microsoft Corporation. All rights reserved."
-        PowerShellVersion = '5.1'
+        PowerShellVersion = '7.0'
         RequiredModules   = @(
             @{
                 ModuleName    = 'Az.Accounts'

docs/_automation/powershell/README.md

@@ -34,10 +34,29 @@ The FinOps toolkit PowerShell module is a collection of commands to automate and
 
 ## 📥 Install the module
 
+The FinOps toolkit module requires PowerShell 7, which is built into [Azure Cloud Shell](https://portal.azure.com/#cloudshell) and supported on all major operating systems. 
+
+Azure Cloud Shell comes with PowerShell 7 and Azure PowerShell pre-installed. If you are not using Azure Cloud Shell, you will need to [Install PowerShell](https://learn.microsoft.com/powershell/scripting/install/installing-powershell) first and then run the following commands to install Azure PowerShell:
+
+```powershell
+Install-Module -Name Az.Accounts
+Install-Module -Name Az.Resources
+```
+
+To install the FinOps toolkit module, run the following in either Azure Cloud Shell or a PowerShell client:
+
 ```powershell
-Import-Module -Name FinOpsToolkit
+Install-Module -Name FinOpsToolkit
 ```
 
+If this is the first time using Azure PowerShell, you will also need to sign into your account and select a default subscription:
+
+```powershell
+Connect-AzAccount
+```
+
+This will show a popup window to sign in to your account. If you do not see the window, it may be on a different screen.
+
 <br>
 
 ## ⚡ Commands

docs/_automation/powershell/cost/Start-FinOpsCostExport.md

@@ -42,30 +42,52 @@ This command has been tested with the following API versions:
 Start-FinOpsCostExport `
     [-Name] <string> `
     [-Scope <string>] `
+    [-StartDate <datetime>] `
+    [-EndDate <datetime>] `
+    [-Backfill <number>] `
     [-ApiVersion <string>]
 ```
 
 <br>
 
 ## 📥 Parameters
 
-| Name          | Description                                                                                                        |
-| ------------- | ------------------------------------------------------------------------------------------------------------------ |
-| `‑Name`       | Optional. Name of the export. Supports wildcards.                                                                  |
-| `‑Scope`      | Optional. Resource ID of the scope the export was created for. If empty, defaults to current subscription context. |
-| `‑ApiVersion` | Optional. API version to use when calling the Cost Management exports API. Default = 2023-03-01.                   |
+| Name          | Description                                                                                                                                                                                                                  |
+| ------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `‑Name`       | Required. Name of the export.                                                                                                                                                                                                |
+| `‑Scope`      | Optional. Resource ID of the scope to export data for. If empty, defaults to current subscription context.                                                                                                                   |
+| `‑StartDate`  | Optional. Day to start pulling the data for. If not set, the export will use the dates defined in the export configuration.                                                                                                  |
+| `‑EndDate`    | Optional. Last day to pull data for. If not set and -StartDate is set, -EndDate will use the last day of the month. If not set and -StartDate is not set, the export will use the dates defined in the export configuration. |
+| `‑Backfill`   | Optional. Number of months to export the data for. Make note of throttling (429) errors. This is only run once. Failed exports are not re-attempted. Default = 0.                                                            |
+| `‑ApiVersion` | Optional. API version to use when calling the Cost Management Exports API. Default = 2023-07-01-preview.                                                                                                                     |
 
 <br>
 
 ## 🌟 Examples
 
-### Run export
+### Export configured period
 
 ```powershell
-Start-FinopsCostExport -Name 'July2023OneTime'
+Start-FinopsCostExport -Name 'CostExport'
 ```
 
-Runs an export called 'July2023OneTime'.
+Runs an export called 'CostExport' for the configured period.
+
+### Export specific dates
+
+```powershell
+Start-FinopsCostExport -Name 'CostExport' -StartDate '2023-01-01' -EndDate '2023-12-31'
+```
+
+Runs an export called 'CostExport' for a specific date range.
+
+### Backfill export
+
+```powershell
+Start-FinopsCostExport -Name 'CostExport' -Backfill 12
+```
+
+Runs an export called 'CostExport' for the previous 12 months.
 
 <br>
 

docs/_resources/changelog.md

@@ -94,6 +94,19 @@ Legend:
 >    - Reports will still be released as PBIX files so this change should not impact end users.
 >    - Visualizations are not being switched to [Power BI Enhanced Report (PBIR)](https://learn.microsoft.com/power-bi/developer/projects/projects-report#pbir-format) format yet due to functional limitations that would impact end users (as of June 2024).
 
+🖥️ PowerShell
+{: .fs-5 .fw-500 .mt-4 mb-0 }
+
+> ➕ Added:
+>
+> 1. Added progress tracking to [Start-FinOpsCostExport](../_automation/powershell/cost/Start-FinOpsCostExport.md) for multi-month exports.
+> 2. Added a 60-second delay when Cost Management returns throttling (429) errors in [Start-FinOpsCostExport](../_automation/powershell/cost/Start-FinOpsCostExport.md).
+>
+> 🗑️ Removed:
+>
+> 1. Removed support for Windows PowerShell.
+>    > _We discovered errors with Windows PowerShell due to incompatibilities in Windows PowerShell and PowerShell 7. Due to our limited capacity, we decided to only support [PowerShell 7](https://learn.microsoft.com/powershell/scripting/install/installing-powershell) going forward._
+
 🌐 Open data
 {: .fs-5 .fw-500 .mt-4 mb-0 }
 

src/powershell/Private/Invoke-Rest.ps1

@@ -86,7 +86,12 @@ function Invoke-Rest
     catch
     {
         $response = $_.Exception.Response
-        $content = $_.ErrorDetails.Message | ConvertFrom-Json -Depth 10
+        try
+        {
+            $content = $_.ErrorDetails.Message | ConvertFrom-Json -Depth 10
+        }
+        catch {}
+
         if ($content.error)
         {
             $errorCode = $content.error.code
@@ -104,6 +109,7 @@ function Invoke-Rest
         Success    = $response.StatusCode -ge 200 -and $response.StatusCode -lt 300
         Failure    = $response.StatusCode -ge 300
         NotFound   = $response.StatusCode -eq 404 -or $response.StatusCode -eq 'NotFound'
+        Throttled  = $response.StatusCode -eq 429 -or $response.StatusCode -eq 'ResourceRequestsThrottled'
         Content    = $content
     }
 }

src/powershell/Public/Start-FinOpsCostExport.ps1

@@ -26,16 +26,26 @@
     Optional. Last day to pull data for. If not set and -StartDate is set, -EndDate will use the last day of the month. If not set and -StartDate is not set, the export will use the dates defined in the export configuration.
 
     .PARAMETER Backfill
-    Optional. IndicLast day to pull data for. If not set and -StartDate is set, -EndDate will use the last day of the month. If not set and -StartDate is not set, the export will use the dates defined in the export configuration.
+    Optional. Number of months to export the data for. Make note of throttling (429) errors. This is only run once. Failed exports are not re-attempted. Default = 0.
 
     .PARAMETER ApiVersion
     Optional. API version to use when calling the Cost Management Exports API. Default = 2023-07-01-preview.
 
     .EXAMPLE
-    Start-FinopsCostExport -Name 'July2023OneTime'
+    Start-FinopsCostExport -Name 'CostExport'
+
+    Runs an export called 'CostExport' for the configured period.
+
+    .EXAMPLE
+    Start-FinopsCostExport -Name 'CostExport' -StartDate '2023-01-01' -EndDate '2023-12-31'
+
+    Runs an export called 'CostExport' for a specific date range.
+
+    .EXAMPLE
+    Start-FinopsCostExport -Name 'CostExport' -Backfill 12
+
+    Runs an export called 'CostExport' for the previous 12 months.
 
-    Runs an export called 'July2023OneTime'.
-
     .LINK
     https://aka.ms/ftk/Start-FinOpsCostExport
 #>
@@ -82,8 +92,22 @@ function Start-FinOpsCostExport
     # Set start date if using -Backfill
     if ($Backfill -gt 0)
     {
-        $StartDate = (Get-Date -Day 1 -Hour 0 -Minute 0 -Second 0 -Millisecond 0 -AsUTC).AddMonths($Backfill * -1)
-        $EndDate = $StartDate.AddMonths($Backfill).AddMilliseconds(-1)
+        # TODO: Consider updating this to account for one-time exports where we should copy the start date from
+
+        # If -StartDate is not set, assume the current month
+        if (-not $StartDate)
+        {
+            $StartDate = (Get-Date -Day 1 -Hour 0 -Minute 0 -Second 0 -Millisecond 0 -AsUTC)
+        }
+
+        # If -EndDate is not set, assume 1 month
+        if (-not $EndDate)
+        {
+            $EndDate = $StartDate.AddMonths(1).AddMilliseconds(-1)
+        }
+
+        # Move start date to account for the backfill period
+        $StartDate = $StartDate.AddMonths($Backfill * -1)
         Write-Verbose "Backfill $Backfill months = $($StartDate.ToUniversalTime().ToString('yyyy-MM-dd"T"HH:mm:ss"Z"')) to $($EndDate.ToUniversalTime().ToString('yyyy-MM-dd"T"HH:mm:ss"Z"'))"
     }
 
@@ -102,18 +126,43 @@ function Start-FinOpsCostExport
         Write-Verbose "Updated dates = $($StartDate.ToUniversalTime().ToString('yyyy-MM-dd"T"HH:mm:ss"Z"')) to $($EndDate.ToUniversalTime().ToString('yyyy-MM-dd"T"HH:mm:ss"Z"'))"
     }
 
+    # Start measuring progress
+    $progressActivity = "Running exports"
+    $startTime = [DateTime]::Now
+    $months = (($EndDate.Year - $StartDate.Year) * 12) + $EndDate.Month - $StartDate.Month + 1
+    if ($months -lt 1) { $months = 1 } # Assume at least 1 month to avoid errors
+    $estimatedSecPerMonth = 6 # Estimated time to trigger a single month export accounting for throttling (10 per minute)
+
     # Loop thru each month
     $monthToExport = 0
     $success = $true
     $body = $null
-    $multipleMonths = $StartDate -and $StartDate.Year -ne $EndDate.Year -or $StartDate.Month -ne $EndDate.Month
-    Write-Verbose "Exporting $($StartDate) - $($EndDate)"
+    if ($StartDate)
+    {
+        Write-Verbose "Exporting dates configured on the export definition"
+    }
+    else
+    {
+        Write-Verbose "Exporting $($StartDate) - $($EndDate)"
+    }
     do
     {
+        # Report progress
+        if ($months -gt 1)
+        {
+            $percent = [Math]::Round((1.0 * $monthToExport / $months) * 100, 0)
+            $remaining = $estimatedSecPerMonth * ($months - $monthToExport)
+            Write-Progress `
+                -Activity $progressActivity `
+                -Status "$percent% complete - $monthToExport of $months months" `
+                -PercentComplete $percent `
+                -SecondsRemaining $remaining
+        }
+
         if ($StartDate)
         {
             # If more than one month
-            if ($multipleMonths)
+            if ($months -gt 1)
             {
                 $firstDay = $EndDate.AddDays(-$EndDate.Day + 1).AddMonths($monthToExport * -1)
                 $lastDay = $firstDay.AddMonths(1).AddMilliseconds(-1)
@@ -124,7 +173,7 @@ function Start-FinOpsCostExport
                 $lastDay = $EndDate
             }
             $body = @{ timePeriod = @{ from = $firstDay.ToString("yyyy-MM-dd'T'HH:mm:ss'Z'"); to = $lastDay.ToString("yyyy-MM-dd'T'HH:mm:ss'Z'") } }
-            Write-Verbose "Executing $($firstDay.ToString("MMM d yyyy HH:mm:ss")) export $runpath"
+            Write-Verbose "Executing $($firstDay.ToString("MMM d yyyy")) export $runpath"
         }
         else
         {
@@ -140,10 +189,37 @@ function Start-FinOpsCostExport
         {
             Write-Verbose "Export failed to execute"
         }
-        $success = $success -and $response.Success
+
+        # If export throttled, wait 60 seconds and try again
+        if ($response.Throttled)
+        {
+            Write-Verbose "Export request throttled. Waiting 60 seconds and retrying."
+
+            # Report progress
+            if ($months -gt 1)
+            {
+                Write-Progress `
+                    -Activity $progressActivity `
+                    -Status "$percent% complete - Throttled by Cost Management. Waiting 60 seconds." `
+            }
+            else
+            {
+                Write-Information "Requests are being throttled by Cost Management. Waiting 60 seconds and retrying..."
+            }
+            Start-Sleep -Seconds 60
+        }
+        else
+        {
+            # If not retrying, then track the success
+            $success = $success -and $response.Success
+        }
 
         $monthToExport += 1
-    } while ($multipleMonths -and $EndDate.AddMonths($monthToExport * -1) -ge $StartDate)
+    } while ($months -gt 1 -and $EndDate.AddMonths($monthToExport * -1) -ge $StartDate)
 
+    if ($months -gt 1)
+    {
+        Write-Progress -Activity $progressActivity -Completed
+    }
     return $success
 }

src/powershell/Tests/Integration/CostExports.Tests.ps1

@@ -238,6 +238,103 @@ Describe 'CostExports' {
         }
     }
 
+    It 'Should handle progress and throttling for 12 month backfill' {
+        # Arrange
+        $startDate = (Get-Date -Day 1 -Hour 0 -Minute 0 -Second 0 -Millisecond 0)
+
+        Monitor "Export throttling tests..." -Indent '  ' {
+            Monitor "Creating $exportName export..." {
+                # Act -- create
+                $newResult = New-FinOpsCostExport -Name $exportName -Scope $scope -StorageAccountId $storage.Id -Execute -Backfill 12
+                # TODO: Run tests for all supported API versions: -ApiVersion '2023-08-01'
+
+                # Assert
+                Report -Object $newResult
+                $newResult.Name | Should -Be $exportName
+                $newResult.RunHistory | Should -BeNullOrEmpty -Because "the -RunHistory option was not specified"
+            }
+
+            Monitor "Getting $exportName..." {
+                # Act -- read
+                $getResult = Get-FinOpsCostExport -Name $exportName -Scope $scope -RunHistory
+
+                # Assert
+                Report "Found $($getResult.Count) export(s)"
+                Report -Object $getResult
+                $getResult.Count | Should -Be 1
+                $getResult.Name | Should -Be $exportName
+                $getResult.RunHistory.Count | Should -BeGreaterThan 0 -Because "-Execute -Backfill was specified during creation"
+            }
+
+            Monitor "Deleting $exportName..." {
+                # Act -- delete
+                $deleteResult = Remove-FinOpsCostExport -Name $exportName -Scope $scope
+                $confirmDeleteResult = Get-FinOpsCostExport -Name $exportName -Scope $scope
+
+                # Assert
+                Report $deleteResult
+                $deleteResult | Should -BeTrue
+                Report "$($getResult.Count) export(s) remaining"
+                $confirmDeleteResult | Should -BeNullOrEmpty
+            }
+        }
+    }
+
+    Context "Long-running unit tests" {
+        BeforeAll {
+            [Diagnostics.CodeAnalysis.SuppressMessageAttribute("PSUseDeclaredVarsMoreThanAssignments", "")]
+            $exportName = 'ftk-test-Start-FinOpsCostExport'
+
+            [Diagnostics.CodeAnalysis.SuppressMessageAttribute("PSUseDeclaredVarsMoreThanAssignments", "")]
+            $scope = "/subscriptions/$([Guid]::NewGuid())"
+
+            [Diagnostics.CodeAnalysis.SuppressMessageAttribute("PSUseDeclaredVarsMoreThanAssignments", "")]
+            $mockExport = @{
+                id   = "$scope/providers/Microsoft.CostManagement/exports/$exportName"
+                name = $exportName
+            }
+        }
+
+        It 'Should wait 60s when throttled' {
+            # NOTE: This is a unit test that mocks dependencies. It's run with integration tests due to how long it takes to run.
+
+            # Arrange
+            $waited = $false
+            $testStartTime = Get-Date
+            function CheckDate($date) {
+                $monthToThrottle = (Get-Date -Day 1 -Hour 0 -Minute 0 -Second 0 -Millisecond 0 -AsUTC).AddMonths(-3).ToUniversalTime().Date.ToString("yyyy-MM-dd'T'HH:mm:ss'Z'")
+                if ($date -eq $monthToThrottle -and -not $waited) {
+                    $waited = $true
+                    return $true
+                }
+                return $false
+            }
+            Mock -ModuleName FinOpsToolkit -CommandName 'Get-FinOpsCostExport' -MockWith { $mockExport }
+            Mock -ModuleName FinOpsToolkit -CommandName 'Invoke-Rest'          -MockWith { 
+                if (CheckDate($Body.timePeriod.from))
+                {
+                    @{ Success = $false; Throttled = $true }
+                }
+                else 
+                {
+                    @{ Success = $true }
+                }
+            }
+            Mock -ModuleName FinOpsToolkit -CommandName 'Write-Progress'       -MockWith {}
+
+            # Act
+            $success = Start-FinOpsCostExport `
+                -Name $exportName `
+                -Scope $scope `
+                -Backfill 12
+
+            # Assert
+            Assert-MockCalled -ModuleName FinOpsToolkit -CommandName 'Write-Progress' -Times 4
+            $success | Should -Be $true
+            ((Get-Date) - $testStartTime).TotalSeconds | Should -BeGreaterThan 60
+        }
+    }
+
     AfterAll {
         # Cleanup
         Remove-AzStorageAccount -ResourceGroupName $rg -Name $storageName -Force