diff --git a/src/ALZ/Private/Deploy-Accelerator-Helpers/New-FolderStructure.ps1 b/src/ALZ/Private/Deploy-Accelerator-Helpers/New-FolderStructure.ps1 index 85ab0ba3..9c876a49 100644 --- a/src/ALZ/Private/Deploy-Accelerator-Helpers/New-FolderStructure.ps1 +++ b/src/ALZ/Private/Deploy-Accelerator-Helpers/New-FolderStructure.ps1 @@ -23,7 +23,10 @@ function New-FolderStructure { [string] $overrideSourceDirectoryPath = "", [Parameter(Mandatory = $false)] - [switch] $replaceFiles + [switch] $replaceFiles, + + [Parameter(Mandatory = $false)] + [int] $maxRetryCount = 10 ) if ($PSCmdlet.ShouldProcess("ALZ-Terraform module configuration", "modify")) { @@ -51,7 +54,7 @@ function New-FolderStructure { } } else { - $releaseTag = Get-GithubRelease -githubRepoUrl $url -targetDirectory $targetDirectory -moduleSourceFolder $sourceFolder -moduleTargetFolder $targetFolder -release $release -releaseArtifactName $releaseArtifactName + $releaseTag = Get-GithubRelease -githubRepoUrl $url -targetDirectory $targetDirectory -moduleSourceFolder $sourceFolder -moduleTargetFolder $targetFolder -release $release -releaseArtifactName $releaseArtifactName -maxRetryCount $maxRetryCount $path = Join-Path $targetDirectory $targetFolder $releaseTag } diff --git a/src/ALZ/Private/Deploy-Accelerator-Helpers/New-ModuleSetup.ps1 b/src/ALZ/Private/Deploy-Accelerator-Helpers/New-ModuleSetup.ps1 index 48cce6b8..9405f445 100644 --- a/src/ALZ/Private/Deploy-Accelerator-Helpers/New-ModuleSetup.ps1 +++ b/src/ALZ/Private/Deploy-Accelerator-Helpers/New-ModuleSetup.ps1 @@ -23,7 +23,9 @@ function New-ModuleSetup { [Parameter(Mandatory = $false)] [switch]$upgrade, [Parameter(Mandatory = $false)] - [switch]$autoApprove + [switch]$autoApprove, + [Parameter(Mandatory = $false)] + [int]$maxRetryCount = 10 ) if ($PSCmdlet.ShouldProcess("Check and get module", "modify")) { @@ -50,12 +52,13 @@ function New-ModuleSetup { -targetFolder $targetFolder ` -sourceFolder $sourceFolder ` -overrideSourceDirectoryPath $moduleOverrideFolderPath ` - -replaceFiles:$replaceFiles.IsPresent + -replaceFiles:$replaceFiles.IsPresent ` + -maxRetryCount $maxRetryCount } $latestReleaseTag = $null try { - $latestResult = Get-GithubReleaseTag -githubRepoUrl $url -release "latest" + $latestResult = Get-GithubReleaseTag -githubRepoUrl $url -release "latest" -maxRetryCount $maxRetryCount $latestReleaseTag = $latestResult.ReleaseTag Write-Verbose "Latest available $targetFolder version: $latestReleaseTag" } catch { @@ -85,18 +88,18 @@ function New-ModuleSetup { $shouldDownload = $true } - if(!$shouldDownload -or $isFirstRun) { + if(!$shouldDownload -or $firstRun) { $newVersionAvailable = $false $currentCalculatedVersion = $currentVersion - if(!$isFirstRun -and $isAutoVersion -and $null -ne $latestReleaseTag -and $latestReleaseTag -ne $currentVersion) { + if(!$firstRun -and $isAutoVersion -and $null -ne $latestReleaseTag -and $latestReleaseTag -ne $currentVersion) { $newVersionAvailable = $true } - if(!$isFirstRun -and !$isAutoVersion -and $null -ne $latestReleaseTag -and $latestReleaseTag -ne $currentVersion) { + if(!$firstRun -and !$isAutoVersion -and $null -ne $latestReleaseTag -and $latestReleaseTag -ne $currentVersion) { $newVersionAvailable = $true } - if($isFirstRun -and !$isAutoVersion -and $release -ne $latestReleaseTag) { + if($firstRun -and !$isAutoVersion -and $release -ne $latestReleaseTag) { $currentCalculatedVersion = $release $newVersionAvailable = $true } @@ -110,8 +113,6 @@ function New-ModuleSetup { Write-ToConsoleLog "No upgrade required for $targetFolder module; already at latest version ($currentCalculatedVersion)." -IsWarning } Write-ToConsoleLog "Using existing $targetFolder module version ($currentCalculatedVersion)." -IsSuccess - } else { - Write-ToConsoleLog "Using specified $targetFolder module version ($currentCalculatedVersion) for the first run." -IsSuccess } } } @@ -120,14 +121,19 @@ function New-ModuleSetup { $previousVersionPath = $versionAndPath.path $desiredRelease = $isAutoVersion ? $latestReleaseTag : $release - Write-ToConsoleLog "Upgrading $targetFolder module from $currentVersion to $desiredRelease" -IsWarning - if (-not $autoApprove.IsPresent) { - $confirm = Read-Host "Do you want to proceed with the upgrade? (y/n)" - if ($confirm -ne "y" -and $confirm -ne "Y") { - Write-ToConsoleLog "Upgrade declined. Continuing with existing version $currentVersion." -IsWarning - return $versionAndPath + if (!$firstRun) { + Write-ToConsoleLog "Upgrading $targetFolder module from $currentVersion to $desiredRelease" -IsWarning + + if (-not $autoApprove.IsPresent) { + $confirm = Read-Host "Do you want to proceed with the upgrade? (y/n)" + if ($confirm -ne "y" -and $confirm -ne "Y") { + Write-ToConsoleLog "Upgrade declined. Continuing with existing version $currentVersion." -IsWarning + return $versionAndPath + } } + } else { + Write-ToConsoleLog "Downloading $targetFolder module version $desiredRelease" -IsSuccess } $versionAndPath = New-FolderStructure ` @@ -138,11 +144,12 @@ function New-ModuleSetup { -targetFolder $targetFolder ` -sourceFolder $sourceFolder ` -overrideSourceDirectoryPath $moduleOverrideFolderPath ` - -replaceFiles:$replaceFiles.IsPresent + -replaceFiles:$replaceFiles.IsPresent ` + -maxRetryCount $maxRetryCount Write-Verbose "New version: $($versionAndPath.releaseTag) at path: $($versionAndPath.path)" - if (!$isFirstRun) { + if (!$firstRun) { Write-Verbose "Checking for state files at: $previousStatePath" $previousStateFiles = Get-ChildItem $previousVersionPath -Filter "terraform.tfstate" -Recurse | Select-Object -First 1 | ForEach-Object { $_.FullName } diff --git a/src/ALZ/Private/Shared/Get-GithubRelease.ps1 b/src/ALZ/Private/Shared/Get-GithubRelease.ps1 index fc1d32b6..0e147ff4 100644 --- a/src/ALZ/Private/Shared/Get-GithubRelease.ps1 +++ b/src/ALZ/Private/Shared/Get-GithubRelease.ps1 @@ -45,14 +45,18 @@ function Get-GithubRelease { $moduleTargetFolder, [Parameter(Mandatory = $false, HelpMessage = "The name of the release artifact in the target release. Defaults to standard release zip.")] - $releaseArtifactName = "" + $releaseArtifactName = "", + + [Parameter(Mandatory = $false, HelpMessage = "Maximum number of retries for transient GitHub API errors.")] + [int] + $maxRetryCount = 10 ) $parentDirectory = $targetDirectory $targetPath = Join-Path $targetDirectory $moduleTargetFolder # Get the release tag and data from GitHub - $releaseResult = Get-GithubReleaseTag -githubRepoUrl $githubRepoUrl -release $release + $releaseResult = Get-GithubReleaseTag -githubRepoUrl $githubRepoUrl -release $release -maxRetryCount $maxRetryCount $releaseTag = $releaseResult.ReleaseTag $releaseData = $releaseResult.ReleaseData @@ -96,7 +100,7 @@ function Get-GithubRelease { Write-Verbose "===> Downloading the release artifact $releaseArtifactUrl from the GitHub repository $repoOrgPlusRepo" - Invoke-WebRequest -Uri $releaseArtifactUrl -OutFile $targetPathForZip -RetryIntervalSec 3 -MaximumRetryCount 100 | Out-String | Write-Verbose + Invoke-GitHubApiRequest -Uri $releaseArtifactUrl -OutputFile $targetPathForZip -MaxRetryCount $maxRetryCount -RetryIntervalSeconds 3 if(!(Test-Path $targetPathForZip)) { Write-ToConsoleLog "Failed to download the release $releaseTag from the GitHub repository $repoOrgPlusRepo" -IsError diff --git a/src/ALZ/Private/Shared/Get-GithubReleaseTag.ps1 b/src/ALZ/Private/Shared/Get-GithubReleaseTag.ps1 index f815b13c..73532630 100644 --- a/src/ALZ/Private/Shared/Get-GithubReleaseTag.ps1 +++ b/src/ALZ/Private/Shared/Get-GithubReleaseTag.ps1 @@ -29,7 +29,11 @@ function Get-GithubReleaseTag { [Parameter(Mandatory = $false, Position = 2, HelpMessage = "The release to check. Specify 'latest' to get the latest release tag. Defaults to 'latest'.")] [string] - $release = "latest" + $release = "latest", + + [Parameter(Mandatory = $false, HelpMessage = "Maximum number of retries for transient GitHub API errors.")] + [int] + $maxRetryCount = 10 ) # Split Repo URL into parts @@ -44,7 +48,9 @@ function Get-GithubReleaseTag { } # Query the GitHub API - $releaseData = Invoke-RestMethod $repoReleaseUrl -SkipHttpErrorCheck -StatusCodeVariable "statusCode" + $response = Invoke-GitHubApiRequest -Uri $repoReleaseUrl -SkipHttpErrorCheck -MaxRetryCount $maxRetryCount -RetryIntervalSeconds 3 + $releaseData = $response.Result + $statusCode = $response.StatusCode Write-Verbose "Status code: $statusCode" @@ -53,12 +59,6 @@ function Get-GithubReleaseTag { throw "The release $release does not exist in the GitHub repository $githubRepoUrl - $repoReleaseUrl" } - # Handle transient errors like throttling - if ($statusCode -ge 400 -and $statusCode -le 599) { - Write-ToConsoleLog "Retrying as got the Status Code $statusCode, which may be a transient error." -IsWarning - $releaseData = Invoke-RestMethod $repoReleaseUrl -RetryIntervalSec 3 -MaximumRetryCount 100 - } - if ($statusCode -ne 200) { throw "Unable to query repository version, please check your internet connection and try again..." } diff --git a/src/ALZ/Private/Shared/Invoke-GitHubApiRequest.ps1 b/src/ALZ/Private/Shared/Invoke-GitHubApiRequest.ps1 new file mode 100644 index 00000000..e65dd185 --- /dev/null +++ b/src/ALZ/Private/Shared/Invoke-GitHubApiRequest.ps1 @@ -0,0 +1,135 @@ +#################################### +# Invoke-GitHubApiRequest.ps1 # +#################################### +# Version: 0.1.0 + +<# +.SYNOPSIS +Invokes a GitHub API request with optional authentication and retry logic. + +.DESCRIPTION +Makes HTTP requests to GitHub APIs or downloads files from GitHub. +If the GitHub CLI (gh) is installed and authenticated, the auth token is +automatically included in request headers to increase rate limits. +Transient errors (HTTP 408, 429, 500, 502, 503, 504) are retried up to +a configurable number of attempts. + +.PARAMETER Uri +The URI to send the request to. + +.PARAMETER Method +The HTTP method for the request. Defaults to GET. + +.PARAMETER MaxRetryCount +Maximum number of retries for transient errors. Defaults to 10. + +.PARAMETER RetryIntervalSeconds +Seconds to wait between retries. Defaults to 3. + +.PARAMETER OutputFile +If specified, downloads the response to this file path using Invoke-WebRequest. + +.PARAMETER SkipHttpErrorCheck +If specified, does not throw on HTTP error status codes. +Returns a hashtable with Result and StatusCode properties. + +.EXAMPLE +Invoke-GitHubApiRequest -Uri "https://api.github.com/repos/Azure/ALZ/releases/latest" + +.EXAMPLE +Invoke-GitHubApiRequest -Uri "https://api.github.com/repos/Azure/ALZ/releases/latest" -SkipHttpErrorCheck + +.EXAMPLE +Invoke-GitHubApiRequest -Uri "https://github.com/Azure/ALZ/archive/refs/tags/v1.0.0.zip" -OutputFile "./release.zip" + +.NOTES +# Release notes 25/03/2026 - V0.1.0: +- Initial release. +#> + +function Invoke-GitHubApiRequest { + [CmdletBinding()] + param ( + [Parameter(Mandatory = $true, Position = 1, HelpMessage = "The URI to send the request to.")] + [string] $Uri, + + [Parameter(Mandatory = $false, HelpMessage = "The HTTP method for the request.")] + [string] $Method = "GET", + + [Parameter(Mandatory = $false, HelpMessage = "Maximum number of retries for transient errors.")] + [int] $MaxRetryCount = 10, + + [Parameter(Mandatory = $false, HelpMessage = "Seconds to wait between retries.")] + [int] $RetryIntervalSeconds = 3, + + [Parameter(Mandatory = $false, HelpMessage = "If specified, downloads the response to this file path.")] + [string] $OutputFile, + + [Parameter(Mandatory = $false, HelpMessage = "If specified, does not throw on HTTP error status codes.")] + [switch] $SkipHttpErrorCheck + ) + + # Build auth headers from gh CLI if available + $headers = @{} + $ghCommand = Get-Command "gh" -ErrorAction SilentlyContinue + if ($null -ne $ghCommand) { + $null = & gh auth status 2>&1 + if ($LASTEXITCODE -eq 0) { + $token = & gh auth token 2>&1 + if ($LASTEXITCODE -eq 0 -and -not [string]::IsNullOrWhiteSpace($token)) { + $headers["Authorization"] = "Bearer $($token.Trim())" + Write-Verbose "GitHub CLI authentication token found. Using authenticated requests." + } + } else { + Write-Verbose "GitHub CLI is installed but not authenticated. Proceeding without authentication." + } + } else { + Write-Verbose "GitHub CLI is not installed. Proceeding without authentication." + } + + $isDownload = -not [string]::IsNullOrEmpty($OutputFile) + $transientStatusCodes = @(408, 429, 500, 502, 503, 504) + $maxAttempts = $MaxRetryCount + 1 + + for ($attempt = 1; $attempt -le $maxAttempts; $attempt++) { + try { + if ($isDownload) { + Invoke-WebRequest -Uri $Uri -Method $Method -Headers $headers -OutFile $OutputFile -ErrorAction Stop + return + } + + if ($SkipHttpErrorCheck) { + $result = Invoke-RestMethod -Uri $Uri -Method $Method -Headers $headers -SkipHttpErrorCheck -StatusCodeVariable "responseStatusCode" + + $code = [int]$responseStatusCode + + if ($code -in $transientStatusCodes -and $attempt -lt $maxAttempts) { + Write-Warning "Request to $Uri returned status $code (attempt $attempt of $maxAttempts). Retrying in $RetryIntervalSeconds seconds..." + Start-Sleep -Seconds $RetryIntervalSeconds + continue + } + + return @{ + Result = $result + StatusCode = $code + } + } + + return (Invoke-RestMethod -Uri $Uri -Method $Method -Headers $headers -ErrorAction Stop) + } catch { + $responseCode = $null + if ($_.Exception.Response) { + $responseCode = [int]$_.Exception.Response.StatusCode + } + + $isTransient = $responseCode -in $transientStatusCodes + + if ($isTransient -and $attempt -lt $maxAttempts) { + Write-Warning "Request to $Uri failed with status $responseCode (attempt $attempt of $maxAttempts). Retrying in $RetryIntervalSeconds seconds..." + Start-Sleep -Seconds $RetryIntervalSeconds + } else { + throw + } + } + } +} diff --git a/src/ALZ/Private/Tools/Checks/Test-NetworkConnectivity.ps1 b/src/ALZ/Private/Tools/Checks/Test-NetworkConnectivity.ps1 index a12e8667..8e3439a3 100644 --- a/src/ALZ/Private/Tools/Checks/Test-NetworkConnectivity.ps1 +++ b/src/ALZ/Private/Tools/Checks/Test-NetworkConnectivity.ps1 @@ -17,7 +17,11 @@ function Test-NetworkConnectivity { foreach ($endpoint in $endpoints) { Write-Verbose "Testing network connectivity to $($endpoint.Uri)" try { - Invoke-WebRequest -Uri $endpoint.Uri -Method Head -TimeoutSec 10 -SkipHttpErrorCheck -ErrorAction Stop -UseBasicParsing | Out-Null + if ($endpoint.Uri -eq "https://api.github.com") { + Invoke-GitHubApiRequest -Uri $endpoint.Uri -Method Head -SkipHttpErrorCheck -MaxRetryCount 0 | Out-Null + } else { + Invoke-WebRequest -Uri $endpoint.Uri -Method Head -TimeoutSec 10 -SkipHttpErrorCheck -ErrorAction Stop -UseBasicParsing | Out-Null + } $results += @{ message = "Network connectivity to $($endpoint.Description) ($($endpoint.Uri)) is available." result = "Success" diff --git a/src/ALZ/Private/Tools/Get-HCLParserTool.ps1 b/src/ALZ/Private/Tools/Get-HCLParserTool.ps1 index 9c01d04b..450453c7 100644 --- a/src/ALZ/Private/Tools/Get-HCLParserTool.ps1 +++ b/src/ALZ/Private/Tools/Get-HCLParserTool.ps1 @@ -5,7 +5,10 @@ function Get-HCLParserTool { [string] $toolsPath, [Parameter(Mandatory = $false)] - [string] $toolVersion + [string] $toolVersion, + + [Parameter(Mandatory = $false)] + [int] $maxRetryCount = 10 ) if ($PSCmdlet.ShouldProcess("Download Terraform Tools", "modify")) { @@ -29,7 +32,7 @@ function Get-HCLParserTool { $uri = "https://github.com/tmccombs/hcl2json/releases/download/$($toolVersion)/$($toolFileName)" Write-Verbose "Downloading Terraform HCL parser Tool from $uri" - Invoke-WebRequest -Uri $uri -OutFile "$toolFilePath" | Out-String | Write-Verbose + Invoke-GitHubApiRequest -Uri $uri -OutputFile $toolFilePath -MaxRetryCount $maxRetryCount } if($osArchitecture.os -ne "windows") { diff --git a/src/ALZ/Public/Deploy-Accelerator.ps1 b/src/ALZ/Public/Deploy-Accelerator.ps1 index 45a9ee02..6ca158f0 100644 --- a/src/ALZ/Public/Deploy-Accelerator.ps1 +++ b/src/ALZ/Public/Deploy-Accelerator.ps1 @@ -194,7 +194,15 @@ function Deploy-Accelerator { HelpMessage = "[OPTIONAL] Clears the cached Azure context (management groups, subscriptions, regions) and fetches fresh data from Azure." )] [Alias("cc")] - [switch] $clear_cache + [switch] $clear_cache, + + [Parameter( + Mandatory = $false, + HelpMessage = "[OPTIONAL] Maximum number of retries for transient GitHub API errors. Defaults to 10. Environment variable: ALZ_github_max_retry_count. Config file input: github_max_retry_count." + )] + [Alias("gmrc")] + [Alias("githubMaxRetryCount")] + [int] $github_max_retry_count = 10 ) $ProgressPreference = "SilentlyContinue" @@ -321,7 +329,7 @@ function Deploy-Accelerator { } else { Write-ToConsoleLog "Checking you have the latest version of Terraform installed..." -IsSuccess Get-TerraformTool -version "latest" -toolsPath $toolsPath - $hclParserToolPath = Get-HCLParserTool -toolVersion "v0.6.0" -toolsPath $toolsPath + $hclParserToolPath = Get-HCLParserTool -toolVersion "v0.6.0" -toolsPath $toolsPath -maxRetryCount $github_max_retry_count } # Get User Inputs from the input config file diff --git a/src/Tests/Unit/Private/Test-NetworkConnectivity.Tests.ps1 b/src/Tests/Unit/Private/Test-NetworkConnectivity.Tests.ps1 index bb3a65d3..354576f6 100644 --- a/src/Tests/Unit/Private/Test-NetworkConnectivity.Tests.ps1 +++ b/src/Tests/Unit/Private/Test-NetworkConnectivity.Tests.ps1 @@ -23,6 +23,9 @@ InModuleScope 'ALZ' { Mock -CommandName Invoke-WebRequest -MockWith { [PSCustomObject]@{ StatusCode = 200 } } + Mock -CommandName Invoke-GitHubApiRequest -MockWith { + @{ StatusCode = 200; Result = $null } + } } It 'returns HasFailure = $false when all endpoints succeed' { @@ -45,7 +48,7 @@ InModuleScope 'ALZ' { Context 'One endpoint is unreachable' { BeforeAll { - Mock -CommandName Invoke-WebRequest -ParameterFilter { $Uri -eq "https://api.github.com" } -MockWith { + Mock -CommandName Invoke-GitHubApiRequest -MockWith { throw "Unable to connect to the remote server" } Mock -CommandName Invoke-WebRequest -MockWith { @@ -83,6 +86,9 @@ InModuleScope 'ALZ' { Mock -CommandName Invoke-WebRequest -MockWith { throw "Network unreachable" } + Mock -CommandName Invoke-GitHubApiRequest -MockWith { + throw "Network unreachable" + } } It 'returns HasFailure = $true' { @@ -104,7 +110,8 @@ InModuleScope 'ALZ' { It 'checks all endpoints and does not stop at the first failure' { $result = Test-NetworkConnectivity - Should -Invoke -CommandName Invoke-WebRequest -Times 6 -Scope It + Should -Invoke -CommandName Invoke-WebRequest -Times 5 -Scope It + Should -Invoke -CommandName Invoke-GitHubApiRequest -Times 1 -Scope It } } }