From a8a48bda4e872219a4f6fe8f2bd093927275ebbd Mon Sep 17 00:00:00 2001 From: Rajesh Kamal Date: Fri, 3 Apr 2026 10:18:06 -0700 Subject: [PATCH 01/10] Add daily extension builds to storage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mirrors the core CLI daily build pattern. On CI builds (push to main), sign and upload extension binaries to Azure Storage. - publish-extension-daily.yml — new stage, depends on Sign, uploads to {ext-id}/daily (latest) and {ext-id}/daily/archive/{version} (history) - publish-extension.yml — add CreateGitHubRelease param (default true) so daily can skip GitHub Release and just upload to storage - release-azd-extension.yml — wire daily publish for CI builds - update-daily-registry.yml — download existing registry-daily.json from storage, merge in the current extension entry with checksums and storage URLs, upload back. Self-maintaining across extension builds. Closes #7317 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../stages/publish-extension-daily.yml | 60 ++++++++ .../stages/release-azd-extension.yml | 11 ++ .../templates/steps/publish-extension.yml | 103 ++++++++------ .../templates/steps/update-daily-registry.yml | 129 ++++++++++++++++++ 4 files changed, 259 insertions(+), 44 deletions(-) create mode 100644 eng/pipelines/templates/stages/publish-extension-daily.yml create mode 100644 eng/pipelines/templates/steps/update-daily-registry.yml diff --git a/eng/pipelines/templates/stages/publish-extension-daily.yml b/eng/pipelines/templates/stages/publish-extension-daily.yml new file mode 100644 index 00000000000..ffa6e39e98e --- /dev/null +++ b/eng/pipelines/templates/stages/publish-extension-daily.yml @@ -0,0 +1,60 @@ +parameters: + - name: SanitizedExtensionId + type: string + - name: AzdExtensionId + type: string + +stages: + - stage: PublishDaily + dependsOn: Sign + condition: >- + and( + succeeded(), + ne(variables['Skip.Release'], 'true'), + or( + in(variables['BuildReasonOverride'], 'IndividualCI', 'BatchedCI'), + and( + eq('', variables['BuildReasonOverride']), + in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI') + ) + ) + ) + + variables: + - template: /eng/pipelines/templates/variables/image.yml + - template: /eng/pipelines/templates/variables/globals.yml + + jobs: + - deployment: Publish_Daily + environment: none + + pool: + name: azsdk-pool + image: ubuntu-22.04 + os: linux + + templateContext: + type: releaseJob + isProduction: false + inputs: + - input: pipelineArtifact + artifactName: release + targetPath: release + + strategy: + runOnce: + deploy: + steps: + - template: /eng/pipelines/templates/steps/extension-set-metadata-variables.yml + parameters: + Use1ESArtifactTask: true + + - template: /eng/pipelines/templates/steps/publish-extension.yml + parameters: + PublishUploadLocations: ${{ parameters.SanitizedExtensionId }}/daily;${{ parameters.SanitizedExtensionId }}/daily/archive/$(EXT_VERSION) + CreateGitHubRelease: false + + - template: /eng/pipelines/templates/steps/update-daily-registry.yml + parameters: + SanitizedExtensionId: ${{ parameters.SanitizedExtensionId }} + AzdExtensionId: ${{ parameters.AzdExtensionId }} diff --git a/eng/pipelines/templates/stages/release-azd-extension.yml b/eng/pipelines/templates/stages/release-azd-extension.yml index 11442c20342..0319c625d33 100644 --- a/eng/pipelines/templates/stages/release-azd-extension.yml +++ b/eng/pipelines/templates/stages/release-azd-extension.yml @@ -84,3 +84,14 @@ stages: - template: /eng/pipelines/templates/stages/publish-extension.yml parameters: SanitizedExtensionId: ${{ parameters.SanitizedExtensionId }} + + # Publish daily builds to storage on CI (push to main) + - ${{ if and(eq(variables['System.TeamProject'], 'internal'), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI')) }}: + - template: /eng/pipelines/templates/stages/sign-extension.yml + parameters: + SanitizedExtensionId: ${{ parameters.SanitizedExtensionId }} + + - template: /eng/pipelines/templates/stages/publish-extension-daily.yml + parameters: + SanitizedExtensionId: ${{ parameters.SanitizedExtensionId }} + AzdExtensionId: ${{ parameters.AzdExtensionId }} diff --git a/eng/pipelines/templates/steps/publish-extension.yml b/eng/pipelines/templates/steps/publish-extension.yml index 6608941e932..343cbda11e3 100644 --- a/eng/pipelines/templates/steps/publish-extension.yml +++ b/eng/pipelines/templates/steps/publish-extension.yml @@ -6,63 +6,78 @@ parameters: default: '`$web' - name: TagPrefix type: string + default: '' - name: TagVersion type: string + default: '' + - name: CreateGitHubRelease + type: boolean + default: true steps: - # This step must run first because a duplicated tag means we don't need to - # continue with any of the subsequent steps. - - pwsh: | - $tag = "${{ parameters.TagPrefix }}_${{ parameters.TagVersion}}" - Write-Host "Release tag: $tag" + - ${{ if eq(parameters.CreateGitHubRelease, true) }}: + # This step must run first because a duplicated tag means we don't need to + # continue with any of the subsequent steps. + - pwsh: | + $tag = "${{ parameters.TagPrefix }}_${{ parameters.TagVersion}}" + Write-Host "Release tag: $tag" - # Check for tag using gh API - $existingTag = gh api /repos/$(Build.Repository.Name)/tags | ConvertFrom-Json | Where-Object { $_.name -eq $tag } - if ($existingTag) { - Write-Host "Tag $tag already exists. Exiting." + # Check for tag using gh API + $existingTag = gh api /repos/$(Build.Repository.Name)/tags | ConvertFrom-Json | Where-Object { $_.name -eq $tag } + if ($existingTag) { + Write-Host "Tag $tag already exists. Exiting." + exit 1 + } + + gh release view $tag --repo $(Build.Repository.Name) + if ($LASTEXITCODE -eq 0) { + Write-Host "Release ($tag) already exists. Exiting." exit 1 - } + } - gh release view $tag --repo $(Build.Repository.Name) - if ($LASTEXITCODE -eq 0) { - Write-Host "Release ($tag) already exists. Exiting." - exit 1 - } + Write-Host "##vso[task.setvariable variable=GH_RELEASE_TAG;]$tag" - Write-Host "##vso[task.setvariable variable=GH_RELEASE_TAG;]$tag" + # Exit with 0 (otherwise $LASTEXITCODE will not be 0 and the pipeline + # will fail) + exit 0 + displayName: Check for existing GitHub release + env: + GH_TOKEN: $(azuresdk-github-pat) - # Exit with 0 (otherwise $LASTEXITCODE will not be 0 and the pipeline - # will fail) - exit 0 - displayName: Check for existing GitHub release - env: - GH_TOKEN: $(azuresdk-github-pat) + - pwsh: | + Remove-Item -Path release/_manifest -Recurse -Force + Write-Host "Release:" + Get-ChildItem -Recurse release/ | Select-Object -Property Length,FullName + displayName: Remove _manifest folder - - pwsh: | - Remove-Item -Path release/_manifest -Recurse -Force - Write-Host "Release:" - Get-ChildItem -Recurse release/ | Select-Object -Property Length,FullName - displayName: Remove _manifest folder + - pwsh: | + $version = "${{ parameters.TagVersion }}" + $createArgs = @( + "$(GH_RELEASE_TAG)", + "--title", "$(GH_RELEASE_TAG)", + "--notes-file", "changelog/CHANGELOG.md", + "--repo", "$(Build.Repository.Name)" + ) - - pwsh: | - $version = "${{ parameters.TagVersion }}" - $createArgs = @( - "$(GH_RELEASE_TAG)", - "--title", "$(GH_RELEASE_TAG)", - "--notes-file", "changelog/CHANGELOG.md", - "--repo", "$(Build.Repository.Name)" - ) + if ($version -match "^0\." -or $version -match "-(alpha|beta|preview)") { + $createArgs += "--prerelease" + } - if ($version -match "^0\." -or $version -match "-(alpha|beta|preview)") { - $createArgs += "--prerelease" - } + gh release create @createArgs - gh release create @createArgs + gh release upload $(GH_RELEASE_TAG) release/* --repo $(Build.Repository.Name) + displayName: Create GitHub Release and upload artifacts + env: + GH_TOKEN: $(azuresdk-github-pat) - gh release upload $(GH_RELEASE_TAG) release/* --repo $(Build.Repository.Name) - displayName: Create GitHub Release and upload artifacts - env: - GH_TOKEN: $(azuresdk-github-pat) + - ${{ if eq(parameters.CreateGitHubRelease, false) }}: + - pwsh: | + if (Test-Path release/_manifest) { + Remove-Item -Path release/_manifest -Recurse -Force + } + Write-Host "Release:" + Get-ChildItem -Recurse release/ | Select-Object -Property Length,FullName + displayName: Remove _manifest folder - task: AzurePowerShell@5 displayName: Upload release to storage account @@ -77,7 +92,7 @@ steps: Get-ChildItem release/ foreach ($folder in $uploadLocations) { Write-Host "Upload to ${{ parameters.StorageContainerName }}/azd/extensions/$folder" - azcopy copy "release/*" "$(publish-storage-location)/${{ parameters.StorageContainerName }}/azd/extensions/$folder" + azcopy copy "release/*" "$(publish-storage-location)/${{ parameters.StorageContainerName }}/azd/extensions/$folder" --overwrite=true if ($LASTEXITCODE) { Write-Error "Upload failed" exit 1 diff --git a/eng/pipelines/templates/steps/update-daily-registry.yml b/eng/pipelines/templates/steps/update-daily-registry.yml new file mode 100644 index 00000000000..ab03f55b8bf --- /dev/null +++ b/eng/pipelines/templates/steps/update-daily-registry.yml @@ -0,0 +1,129 @@ +parameters: + - name: SanitizedExtensionId + type: string + - name: AzdExtensionId + type: string + +steps: + - task: AzurePowerShell@5 + displayName: Update daily registry + inputs: + azureSubscription: 'Azure SDK Artifacts' + azurePowerShellVersion: LatestVersion + pwsh: true + ScriptType: InlineScript + Inline: | + $extId = "${{ parameters.AzdExtensionId }}" + $sanitizedId = "${{ parameters.SanitizedExtensionId }}" + $version = "$(EXT_VERSION)" + $storageHost = "$(publish-storage-static-host)" + $registryBlobPath = "$(publish-storage-location)/`$web/azd/extensions/registry-daily.json" + $dailyBaseUrl = "$storageHost/azd/extensions/$sanitizedId/daily" + + # Download existing registry or create empty one + $registryFile = "registry-daily.json" + azcopy copy $registryBlobPath $registryFile 2>$null + if ($LASTEXITCODE -ne 0 -or !(Test-Path $registryFile)) { + Write-Host "No existing registry found, creating new one" + @{ extensions = @() } | ConvertTo-Json -Depth 10 | Set-Content $registryFile + } + + $registry = Get-Content $registryFile -Raw | ConvertFrom-Json -Depth 20 + + # Read extension.yaml from the release-metadata artifact + $extYaml = Get-Content "release-metadata/extension.yaml" -Raw + $extMeta = @{} + foreach ($line in $extYaml -split "`n") { + if ($line -match "^(\w[\w\-]*):\s*(.+)$") { + $extMeta[$matches[1]] = $matches[2].Trim() + } + } + + # Parse list fields from extension.yaml + function Get-YamlList($yaml, $field) { + $items = @() + $inField = $false + foreach ($line in $yaml -split "`n") { + if ($line -match "^${field}:") { $inField = $true; continue } + if ($inField -and $line -match "^\s+-\s+(.+)$") { + $items += $matches[1].Trim() + } elseif ($inField -and $line -match "^\S") { + break + } + } + return $items + } + + $capabilities = Get-YamlList $extYaml "capabilities" + + # Compute checksums from the release artifacts + $platforms = @( + @{ key = "darwin/amd64"; file = "$sanitizedId-darwin-amd64.zip"; entry = "$sanitizedId-darwin-amd64" }, + @{ key = "darwin/arm64"; file = "$sanitizedId-darwin-arm64.zip"; entry = "$sanitizedId-darwin-arm64" }, + @{ key = "linux/amd64"; file = "$sanitizedId-linux-amd64.tar.gz"; entry = "$sanitizedId-linux-amd64" }, + @{ key = "linux/arm64"; file = "$sanitizedId-linux-arm64.tar.gz"; entry = "$sanitizedId-linux-arm64" }, + @{ key = "windows/amd64"; file = "$sanitizedId-windows-amd64.zip"; entry = "$sanitizedId-windows-amd64.exe" }, + @{ key = "windows/arm64"; file = "$sanitizedId-windows-arm64.zip"; entry = "$sanitizedId-windows-arm64.exe" } + ) + + $artifacts = @{} + foreach ($p in $platforms) { + $filePath = "release/$($p.file)" + if (Test-Path $filePath) { + $hash = (Get-FileHash -Path $filePath -Algorithm SHA256).Hash.ToLower() + $artifacts[$p.key] = @{ + checksum = @{ algorithm = "sha256"; value = $hash } + entryPoint = $p.entry + url = "$dailyBaseUrl/$($p.file)" + } + } + } + + # Build the version entry + $versionEntry = @{ + version = $version + capabilities = $capabilities + artifacts = $artifacts + } + + if ($extMeta["usage"]) { $versionEntry.usage = $extMeta["usage"] } + if ($extMeta["requiredAzdVersion"]) { + $versionEntry.requiredAzdVersion = $extMeta["requiredAzdVersion"] + } + + # Build the extension entry + $extEntry = @{ + id = $extId + namespace = $extMeta["namespace"] + displayName = $extMeta["displayName"] + description = $extMeta["description"] + versions = @($versionEntry) + } + + # Merge into registry: replace if extension exists, add if new + $found = $false + for ($i = 0; $i -lt $registry.extensions.Count; $i++) { + if ($registry.extensions[$i].id -eq $extId) { + $registry.extensions[$i] = $extEntry + $found = $true + break + } + } + if (-not $found) { + $registry.extensions += $extEntry + } + + # Write and upload + $registry | ConvertTo-Json -Depth 20 | Set-Content $registryFile -Encoding utf8 + Write-Host "Updated registry for $extId ($version):" + Get-Content $registryFile + + azcopy copy $registryFile $registryBlobPath --overwrite=true + if ($LASTEXITCODE) { + Write-Error "Failed to upload registry" + exit 1 + } + + Write-Host "Registry uploaded to $registryBlobPath" + env: + AZCOPY_AUTO_LOGIN_TYPE: 'PSCRED' From 1ba909d0fa7d74f7684dfb23ee40a56da9da4ffd Mon Sep 17 00:00:00 2001 From: Rajesh Kamal Date: Fri, 3 Apr 2026 16:57:45 -0700 Subject: [PATCH 02/10] Add daily extension builds to storage with registry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pipeline changes: - publish-extension-daily.yml — daily stage, depends on Sign, uploads to {ext-id}/daily and {ext-id}/daily/archive/{version} - publish-extension.yml — add CreateGitHubRelease param so daily can skip GitHub Release and just upload to storage - release-azd-extension.yml — wire daily publish for CI builds - update-extension-daily-registry.yml — calls Update-ExtensionDailyRegistry.ps1 to merge extension entry into registry-daily.json on storage - extension-registry-daily-template.json — JSON template with placeholders Version bumps: - Bump all extension version.txt and extension.yaml so daily builds are ahead of the last released versions Closes #7317 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../azure.ai.finetune/extension.yaml | 2 +- .../extensions/azure.ai.finetune/version.txt | 2 +- .../extensions/azure.ai.models/extension.yaml | 2 +- .../extensions/azure.ai.models/version.txt | 2 +- .../azure.appservice/extension.yaml | 2 +- .../extensions/azure.appservice/version.txt | 2 +- .../azure.coding-agent/extension.yaml | 2 +- .../extensions/azure.coding-agent/version.txt | 2 +- .../microsoft.azd.ai.builder/extension.yaml | 2 +- .../microsoft.azd.ai.builder/version.txt | 2 +- .../microsoft.azd.concurx/extension.yaml | 2 +- .../microsoft.azd.concurx/version.txt | 2 +- .../microsoft.azd.demo/extension.yaml | 2 +- .../extensions/microsoft.azd.demo/version.txt | 2 +- .../microsoft.azd.extensions/extension.yaml | 2 +- .../microsoft.azd.extensions/version.txt | 2 +- .../extension-registry-daily-template.json | 38 +++ .../stages/publish-extension-daily.yml | 2 +- .../templates/steps/update-daily-registry.yml | 129 ---------- .../steps/update-extension-daily-registry.yml | 29 +++ eng/scripts/Update-ExtensionDailyRegistry.ps1 | 227 ++++++++++++++++++ 21 files changed, 311 insertions(+), 146 deletions(-) create mode 100644 eng/pipelines/templates/json/extension-registry-daily-template.json delete mode 100644 eng/pipelines/templates/steps/update-daily-registry.yml create mode 100644 eng/pipelines/templates/steps/update-extension-daily-registry.yml create mode 100644 eng/scripts/Update-ExtensionDailyRegistry.ps1 diff --git a/cli/azd/extensions/azure.ai.finetune/extension.yaml b/cli/azd/extensions/azure.ai.finetune/extension.yaml index afcac71f338..2eea05b6047 100644 --- a/cli/azd/extensions/azure.ai.finetune/extension.yaml +++ b/cli/azd/extensions/azure.ai.finetune/extension.yaml @@ -3,7 +3,7 @@ namespace: ai.finetuning displayName: Foundry Fine Tuning (Preview) description: Extension for Foundry Fine Tuning. (Preview) usage: azd ai finetuning [options] -version: 0.0.17-preview +version: 0.0.18-preview language: go capabilities: - custom-commands diff --git a/cli/azd/extensions/azure.ai.finetune/version.txt b/cli/azd/extensions/azure.ai.finetune/version.txt index 4a615f30a4e..575667c628c 100644 --- a/cli/azd/extensions/azure.ai.finetune/version.txt +++ b/cli/azd/extensions/azure.ai.finetune/version.txt @@ -1 +1 @@ -0.0.17-preview +0.0.18-preview diff --git a/cli/azd/extensions/azure.ai.models/extension.yaml b/cli/azd/extensions/azure.ai.models/extension.yaml index 43f17641435..27029d82468 100644 --- a/cli/azd/extensions/azure.ai.models/extension.yaml +++ b/cli/azd/extensions/azure.ai.models/extension.yaml @@ -3,7 +3,7 @@ namespace: ai.models displayName: Foundry Custom Models (Preview) description: Extension for managing custom models in Azure AI Foundry. (Preview) usage: azd ai models [options] -version: 0.0.5-preview +version: 0.0.6-preview language: go capabilities: - custom-commands diff --git a/cli/azd/extensions/azure.ai.models/version.txt b/cli/azd/extensions/azure.ai.models/version.txt index f23e8a72a0f..a2c0ee7a53c 100644 --- a/cli/azd/extensions/azure.ai.models/version.txt +++ b/cli/azd/extensions/azure.ai.models/version.txt @@ -1 +1 @@ -0.0.5-preview +0.0.6-preview diff --git a/cli/azd/extensions/azure.appservice/extension.yaml b/cli/azd/extensions/azure.appservice/extension.yaml index b9f0e67be76..3f7d5cda966 100644 --- a/cli/azd/extensions/azure.appservice/extension.yaml +++ b/cli/azd/extensions/azure.appservice/extension.yaml @@ -5,7 +5,7 @@ displayName: Azure App Service description: Extension for managing Azure App Service resources. usage: azd appservice [options] # NOTE: Make sure version.txt is in sync with this version. -version: 0.1.0 +version: 0.1.1 language: go capabilities: - custom-commands diff --git a/cli/azd/extensions/azure.appservice/version.txt b/cli/azd/extensions/azure.appservice/version.txt index 6e8bf73aa55..17e51c385ea 100644 --- a/cli/azd/extensions/azure.appservice/version.txt +++ b/cli/azd/extensions/azure.appservice/version.txt @@ -1 +1 @@ -0.1.0 +0.1.1 diff --git a/cli/azd/extensions/azure.coding-agent/extension.yaml b/cli/azd/extensions/azure.coding-agent/extension.yaml index 0c42c3de738..f84701e809f 100644 --- a/cli/azd/extensions/azure.coding-agent/extension.yaml +++ b/cli/azd/extensions/azure.coding-agent/extension.yaml @@ -8,4 +8,4 @@ language: go namespace: coding-agent usage: azd coding-agent [options] # NOTE: Make sure version.txt is in sync with this version. -version: 0.6.1 +version: 0.6.2 diff --git a/cli/azd/extensions/azure.coding-agent/version.txt b/cli/azd/extensions/azure.coding-agent/version.txt index ee6cdce3c29..b6160487433 100644 --- a/cli/azd/extensions/azure.coding-agent/version.txt +++ b/cli/azd/extensions/azure.coding-agent/version.txt @@ -1 +1 @@ -0.6.1 +0.6.2 diff --git a/cli/azd/extensions/microsoft.azd.ai.builder/extension.yaml b/cli/azd/extensions/microsoft.azd.ai.builder/extension.yaml index 4271102ea81..d58b8acfc84 100644 --- a/cli/azd/extensions/microsoft.azd.ai.builder/extension.yaml +++ b/cli/azd/extensions/microsoft.azd.ai.builder/extension.yaml @@ -4,7 +4,7 @@ namespace: ai.builder displayName: azd AI Builder description: This extension provides custom commands for building AI applications using Azure Developer CLI. usage: azd ai builder [options] -version: 0.2.0 +version: 0.2.1 language: go capabilities: - custom-commands diff --git a/cli/azd/extensions/microsoft.azd.ai.builder/version.txt b/cli/azd/extensions/microsoft.azd.ai.builder/version.txt index 341cf11faf9..0c62199f16a 100644 --- a/cli/azd/extensions/microsoft.azd.ai.builder/version.txt +++ b/cli/azd/extensions/microsoft.azd.ai.builder/version.txt @@ -1 +1 @@ -0.2.0 \ No newline at end of file +0.2.1 diff --git a/cli/azd/extensions/microsoft.azd.concurx/extension.yaml b/cli/azd/extensions/microsoft.azd.concurx/extension.yaml index 35b5e66c1fb..3f3cc37d552 100644 --- a/cli/azd/extensions/microsoft.azd.concurx/extension.yaml +++ b/cli/azd/extensions/microsoft.azd.concurx/extension.yaml @@ -7,4 +7,4 @@ id: microsoft.azd.concurx language: go namespace: concurx usage: azd concurx [options] -version: 0.1.0 +version: 0.1.1 diff --git a/cli/azd/extensions/microsoft.azd.concurx/version.txt b/cli/azd/extensions/microsoft.azd.concurx/version.txt index 6c6aa7cb091..17e51c385ea 100644 --- a/cli/azd/extensions/microsoft.azd.concurx/version.txt +++ b/cli/azd/extensions/microsoft.azd.concurx/version.txt @@ -1 +1 @@ -0.1.0 \ No newline at end of file +0.1.1 diff --git a/cli/azd/extensions/microsoft.azd.demo/extension.yaml b/cli/azd/extensions/microsoft.azd.demo/extension.yaml index 05e570efb95..96d5a942b61 100644 --- a/cli/azd/extensions/microsoft.azd.demo/extension.yaml +++ b/cli/azd/extensions/microsoft.azd.demo/extension.yaml @@ -4,7 +4,7 @@ namespace: demo displayName: Demo Extension description: This extension provides examples of the azd extension framework. usage: azd demo [options] -version: 0.6.0 +version: 0.6.1 language: go capabilities: - custom-commands diff --git a/cli/azd/extensions/microsoft.azd.demo/version.txt b/cli/azd/extensions/microsoft.azd.demo/version.txt index 09a3acfa138..ee6cdce3c29 100644 --- a/cli/azd/extensions/microsoft.azd.demo/version.txt +++ b/cli/azd/extensions/microsoft.azd.demo/version.txt @@ -1 +1 @@ -0.6.0 \ No newline at end of file +0.6.1 diff --git a/cli/azd/extensions/microsoft.azd.extensions/extension.yaml b/cli/azd/extensions/microsoft.azd.extensions/extension.yaml index 51317a17e2a..b978c255986 100644 --- a/cli/azd/extensions/microsoft.azd.extensions/extension.yaml +++ b/cli/azd/extensions/microsoft.azd.extensions/extension.yaml @@ -5,7 +5,7 @@ language: go displayName: azd extensions Developer Kit description: This extension provides a set of tools for azd extension developers to test and debug their extensions. usage: azd x [options] -version: 0.10.0 +version: 0.10.1 capabilities: - custom-commands - metadata diff --git a/cli/azd/extensions/microsoft.azd.extensions/version.txt b/cli/azd/extensions/microsoft.azd.extensions/version.txt index 2774f8587f4..571215736a6 100644 --- a/cli/azd/extensions/microsoft.azd.extensions/version.txt +++ b/cli/azd/extensions/microsoft.azd.extensions/version.txt @@ -1 +1 @@ -0.10.0 \ No newline at end of file +0.10.1 diff --git a/eng/pipelines/templates/json/extension-registry-daily-template.json b/eng/pipelines/templates/json/extension-registry-daily-template.json new file mode 100644 index 00000000000..d27d134075b --- /dev/null +++ b/eng/pipelines/templates/json/extension-registry-daily-template.json @@ -0,0 +1,38 @@ +{ + "version": "${EXT_VERSION}", + "requiredAzdVersion": "${REQUIRED_AZD_VERSION}", + "capabilities": [], + "usage": "${USAGE}", + "artifacts": { + "darwin/amd64": { + "checksum": { "algorithm": "sha256", "value": "${CHECKSUM_DARWIN_AMD64}" }, + "entryPoint": "${SANITIZED_ID}-darwin-amd64", + "url": "${STORAGE_BASE_URL}/${SANITIZED_ID}-darwin-amd64.zip" + }, + "darwin/arm64": { + "checksum": { "algorithm": "sha256", "value": "${CHECKSUM_DARWIN_ARM64}" }, + "entryPoint": "${SANITIZED_ID}-darwin-arm64", + "url": "${STORAGE_BASE_URL}/${SANITIZED_ID}-darwin-arm64.zip" + }, + "linux/amd64": { + "checksum": { "algorithm": "sha256", "value": "${CHECKSUM_LINUX_AMD64}" }, + "entryPoint": "${SANITIZED_ID}-linux-amd64", + "url": "${STORAGE_BASE_URL}/${SANITIZED_ID}-linux-amd64.tar.gz" + }, + "linux/arm64": { + "checksum": { "algorithm": "sha256", "value": "${CHECKSUM_LINUX_ARM64}" }, + "entryPoint": "${SANITIZED_ID}-linux-arm64", + "url": "${STORAGE_BASE_URL}/${SANITIZED_ID}-linux-arm64.tar.gz" + }, + "windows/amd64": { + "checksum": { "algorithm": "sha256", "value": "${CHECKSUM_WINDOWS_AMD64}" }, + "entryPoint": "${SANITIZED_ID}-windows-amd64.exe", + "url": "${STORAGE_BASE_URL}/${SANITIZED_ID}-windows-amd64.zip" + }, + "windows/arm64": { + "checksum": { "algorithm": "sha256", "value": "${CHECKSUM_WINDOWS_ARM64}" }, + "entryPoint": "${SANITIZED_ID}-windows-arm64.exe", + "url": "${STORAGE_BASE_URL}/${SANITIZED_ID}-windows-arm64.zip" + } + } +} diff --git a/eng/pipelines/templates/stages/publish-extension-daily.yml b/eng/pipelines/templates/stages/publish-extension-daily.yml index ffa6e39e98e..b793c4f5f54 100644 --- a/eng/pipelines/templates/stages/publish-extension-daily.yml +++ b/eng/pipelines/templates/stages/publish-extension-daily.yml @@ -54,7 +54,7 @@ stages: PublishUploadLocations: ${{ parameters.SanitizedExtensionId }}/daily;${{ parameters.SanitizedExtensionId }}/daily/archive/$(EXT_VERSION) CreateGitHubRelease: false - - template: /eng/pipelines/templates/steps/update-daily-registry.yml + - template: /eng/pipelines/templates/steps/update-extension-daily-registry.yml parameters: SanitizedExtensionId: ${{ parameters.SanitizedExtensionId }} AzdExtensionId: ${{ parameters.AzdExtensionId }} diff --git a/eng/pipelines/templates/steps/update-daily-registry.yml b/eng/pipelines/templates/steps/update-daily-registry.yml deleted file mode 100644 index ab03f55b8bf..00000000000 --- a/eng/pipelines/templates/steps/update-daily-registry.yml +++ /dev/null @@ -1,129 +0,0 @@ -parameters: - - name: SanitizedExtensionId - type: string - - name: AzdExtensionId - type: string - -steps: - - task: AzurePowerShell@5 - displayName: Update daily registry - inputs: - azureSubscription: 'Azure SDK Artifacts' - azurePowerShellVersion: LatestVersion - pwsh: true - ScriptType: InlineScript - Inline: | - $extId = "${{ parameters.AzdExtensionId }}" - $sanitizedId = "${{ parameters.SanitizedExtensionId }}" - $version = "$(EXT_VERSION)" - $storageHost = "$(publish-storage-static-host)" - $registryBlobPath = "$(publish-storage-location)/`$web/azd/extensions/registry-daily.json" - $dailyBaseUrl = "$storageHost/azd/extensions/$sanitizedId/daily" - - # Download existing registry or create empty one - $registryFile = "registry-daily.json" - azcopy copy $registryBlobPath $registryFile 2>$null - if ($LASTEXITCODE -ne 0 -or !(Test-Path $registryFile)) { - Write-Host "No existing registry found, creating new one" - @{ extensions = @() } | ConvertTo-Json -Depth 10 | Set-Content $registryFile - } - - $registry = Get-Content $registryFile -Raw | ConvertFrom-Json -Depth 20 - - # Read extension.yaml from the release-metadata artifact - $extYaml = Get-Content "release-metadata/extension.yaml" -Raw - $extMeta = @{} - foreach ($line in $extYaml -split "`n") { - if ($line -match "^(\w[\w\-]*):\s*(.+)$") { - $extMeta[$matches[1]] = $matches[2].Trim() - } - } - - # Parse list fields from extension.yaml - function Get-YamlList($yaml, $field) { - $items = @() - $inField = $false - foreach ($line in $yaml -split "`n") { - if ($line -match "^${field}:") { $inField = $true; continue } - if ($inField -and $line -match "^\s+-\s+(.+)$") { - $items += $matches[1].Trim() - } elseif ($inField -and $line -match "^\S") { - break - } - } - return $items - } - - $capabilities = Get-YamlList $extYaml "capabilities" - - # Compute checksums from the release artifacts - $platforms = @( - @{ key = "darwin/amd64"; file = "$sanitizedId-darwin-amd64.zip"; entry = "$sanitizedId-darwin-amd64" }, - @{ key = "darwin/arm64"; file = "$sanitizedId-darwin-arm64.zip"; entry = "$sanitizedId-darwin-arm64" }, - @{ key = "linux/amd64"; file = "$sanitizedId-linux-amd64.tar.gz"; entry = "$sanitizedId-linux-amd64" }, - @{ key = "linux/arm64"; file = "$sanitizedId-linux-arm64.tar.gz"; entry = "$sanitizedId-linux-arm64" }, - @{ key = "windows/amd64"; file = "$sanitizedId-windows-amd64.zip"; entry = "$sanitizedId-windows-amd64.exe" }, - @{ key = "windows/arm64"; file = "$sanitizedId-windows-arm64.zip"; entry = "$sanitizedId-windows-arm64.exe" } - ) - - $artifacts = @{} - foreach ($p in $platforms) { - $filePath = "release/$($p.file)" - if (Test-Path $filePath) { - $hash = (Get-FileHash -Path $filePath -Algorithm SHA256).Hash.ToLower() - $artifacts[$p.key] = @{ - checksum = @{ algorithm = "sha256"; value = $hash } - entryPoint = $p.entry - url = "$dailyBaseUrl/$($p.file)" - } - } - } - - # Build the version entry - $versionEntry = @{ - version = $version - capabilities = $capabilities - artifacts = $artifacts - } - - if ($extMeta["usage"]) { $versionEntry.usage = $extMeta["usage"] } - if ($extMeta["requiredAzdVersion"]) { - $versionEntry.requiredAzdVersion = $extMeta["requiredAzdVersion"] - } - - # Build the extension entry - $extEntry = @{ - id = $extId - namespace = $extMeta["namespace"] - displayName = $extMeta["displayName"] - description = $extMeta["description"] - versions = @($versionEntry) - } - - # Merge into registry: replace if extension exists, add if new - $found = $false - for ($i = 0; $i -lt $registry.extensions.Count; $i++) { - if ($registry.extensions[$i].id -eq $extId) { - $registry.extensions[$i] = $extEntry - $found = $true - break - } - } - if (-not $found) { - $registry.extensions += $extEntry - } - - # Write and upload - $registry | ConvertTo-Json -Depth 20 | Set-Content $registryFile -Encoding utf8 - Write-Host "Updated registry for $extId ($version):" - Get-Content $registryFile - - azcopy copy $registryFile $registryBlobPath --overwrite=true - if ($LASTEXITCODE) { - Write-Error "Failed to upload registry" - exit 1 - } - - Write-Host "Registry uploaded to $registryBlobPath" - env: - AZCOPY_AUTO_LOGIN_TYPE: 'PSCRED' diff --git a/eng/pipelines/templates/steps/update-extension-daily-registry.yml b/eng/pipelines/templates/steps/update-extension-daily-registry.yml new file mode 100644 index 00000000000..61761ce368b --- /dev/null +++ b/eng/pipelines/templates/steps/update-extension-daily-registry.yml @@ -0,0 +1,29 @@ +parameters: + - name: SanitizedExtensionId + type: string + - name: AzdExtensionId + type: string + +steps: + - task: AzurePowerShell@5 + displayName: Update daily registry + inputs: + azureSubscription: 'Azure SDK Artifacts' + azurePowerShellVersion: LatestVersion + pwsh: true + ScriptType: InlineScript + Inline: | + $storageHost = "$(publish-storage-static-host)" + $dailyBaseUrl = "$storageHost/azd/extensions/${{ parameters.SanitizedExtensionId }}/daily" + $registryBlobPath = "$(publish-storage-location)/`$web/azd/extensions/registry-daily.json" + $templatePath = "$(Build.SourcesDirectory)/eng/pipelines/templates/json/extension-registry-daily-template.json" + + & "$(Build.SourcesDirectory)/eng/scripts/Update-ExtensionDailyRegistry.ps1" ` + -SanitizedExtensionId "${{ parameters.SanitizedExtensionId }}" ` + -AzdExtensionId "${{ parameters.AzdExtensionId }}" ` + -Version "$(EXT_VERSION)" ` + -StorageBaseUrl $dailyBaseUrl ` + -RegistryBlobPath $registryBlobPath ` + -TemplatePath $templatePath + env: + AZCOPY_AUTO_LOGIN_TYPE: 'PSCRED' diff --git a/eng/scripts/Update-ExtensionDailyRegistry.ps1 b/eng/scripts/Update-ExtensionDailyRegistry.ps1 new file mode 100644 index 00000000000..de2c2a3db4b --- /dev/null +++ b/eng/scripts/Update-ExtensionDailyRegistry.ps1 @@ -0,0 +1,227 @@ +<# +.SYNOPSIS + Updates registry-daily.json on Azure Storage with the current extension's daily build entry. + +.DESCRIPTION + 1. Computes checksums from signed release artifacts + 2. Reads extension.yaml for metadata (id, namespace, displayName, etc.) + 3. Loads the JSON template, replaces placeholders with actual values + 4. Downloads existing registry-daily.json from storage (or creates empty) + 5. Merges the new entry and uploads back + +.PARAMETER SanitizedExtensionId + Hyphenated extension id (e.g. azure-ai-agents) + +.PARAMETER AzdExtensionId + Dotted extension id (e.g. azure.ai.agents) + +.PARAMETER Version + Extension version from version.txt + +.PARAMETER StorageBaseUrl + Static storage host URL for daily artifacts + +.PARAMETER RegistryBlobPath + Full blob path for registry-daily.json + +.PARAMETER ReleasePath + Path to the signed release artifacts + +.PARAMETER MetadataPath + Path to the release-metadata directory containing extension.yaml + +.PARAMETER TemplatePath + Path to the extension-registry-daily-template.json +#> + +param( + [Parameter(Mandatory)] [string] $SanitizedExtensionId, + [Parameter(Mandatory)] [string] $AzdExtensionId, + [Parameter(Mandatory)] [string] $Version, + [Parameter(Mandatory)] [string] $StorageBaseUrl, + [Parameter(Mandatory)] [string] $RegistryBlobPath, + [Parameter(Mandatory)] [string] $TemplatePath, + [string] $ReleasePath = "release", + [string] $MetadataPath = "release-metadata" +) + +$ErrorActionPreference = 'Stop' + +# Validate required files exist +$extYamlPath = Join-Path $MetadataPath "extension.yaml" +if (!(Test-Path $extYamlPath)) { + Write-Error "extension.yaml not found at $extYamlPath" + exit 1 +} +if (!(Test-Path $TemplatePath)) { + Write-Error "Template not found at $TemplatePath" + exit 1 +} + +# Compute checksums from signed artifacts +$checksums = @{} +$missingArtifacts = @() +$artifactFiles = @( + @{ key = "DARWIN_AMD64"; file = "$SanitizedExtensionId-darwin-amd64.zip" }, + @{ key = "DARWIN_ARM64"; file = "$SanitizedExtensionId-darwin-arm64.zip" }, + @{ key = "LINUX_AMD64"; file = "$SanitizedExtensionId-linux-amd64.tar.gz" }, + @{ key = "LINUX_ARM64"; file = "$SanitizedExtensionId-linux-arm64.tar.gz" }, + @{ key = "WINDOWS_AMD64"; file = "$SanitizedExtensionId-windows-amd64.zip" }, + @{ key = "WINDOWS_ARM64"; file = "$SanitizedExtensionId-windows-arm64.zip" } +) + +foreach ($artifact in $artifactFiles) { + $filePath = Join-Path $ReleasePath $artifact.file + if (Test-Path $filePath) { + $hash = (Get-FileHash -Path $filePath -Algorithm SHA256).Hash.ToLower() + $checksums[$artifact.key] = $hash + Write-Host "Checksum $($artifact.key): $hash" + } else { + $missingArtifacts += $artifact.file + } +} + +if ($missingArtifacts.Count -gt 0) { + Write-Error "Missing release artifacts: $($missingArtifacts -join ', ')" + exit 1 +} + +# Read extension.yaml for metadata +# Simple line-by-line parsing for top-level scalar fields, capabilities list, +# and providers list. Sufficient for the known extension.yaml schema. +$extYaml = Get-Content $extYamlPath -Raw +$extMeta = @{} +foreach ($line in $extYaml -split "`n") { + if ($line -match "^(\w[\w\-]*):\s*(.+)$") { + $extMeta[$matches[1]] = $matches[2].Trim().Trim('"') + } +} + +# Parse capabilities list +$capabilities = @() +$inCapabilities = $false +foreach ($line in $extYaml -split "`n") { + if ($line -match "^capabilities:") { $inCapabilities = $true; continue } + if ($inCapabilities -and $line -match "^\s+-\s+(.+)$") { + $capabilities += $matches[1].Trim() + } elseif ($inCapabilities -and $line -match "^\S") { + break + } +} + +# Parse providers list +$providers = @() +$inProviders = $false +$currentProvider = $null +foreach ($line in $extYaml -split "`n") { + if ($line -match "^providers:") { $inProviders = $true; continue } + if ($inProviders -and $line -match "^\s+-\s+name:\s*(.+)$") { + if ($currentProvider) { $providers += $currentProvider } + $currentProvider = [ordered]@{ name = $matches[1].Trim() } + } elseif ($inProviders -and $currentProvider -and $line -match "^\s+(\w+):\s*(.+)$") { + $currentProvider[$matches[1].Trim()] = $matches[2].Trim() + } elseif ($inProviders -and $line -match "^\S") { + break + } +} +if ($currentProvider) { $providers += $currentProvider } + +# Load template and replace placeholders +$template = Get-Content $TemplatePath -Raw +$replacements = @{ + '${EXT_VERSION}' = $Version + '${REQUIRED_AZD_VERSION}' = if ($extMeta.requiredAzdVersion) { $extMeta.requiredAzdVersion } else { "" } + '${USAGE}' = if ($extMeta.usage) { $extMeta.usage } else { "" } + '${SANITIZED_ID}' = $SanitizedExtensionId + '${STORAGE_BASE_URL}' = $StorageBaseUrl + '${CHECKSUM_DARWIN_AMD64}' = $checksums["DARWIN_AMD64"] + '${CHECKSUM_DARWIN_ARM64}' = $checksums["DARWIN_ARM64"] + '${CHECKSUM_LINUX_AMD64}' = $checksums["LINUX_AMD64"] + '${CHECKSUM_LINUX_ARM64}' = $checksums["LINUX_ARM64"] + '${CHECKSUM_WINDOWS_AMD64}' = $checksums["WINDOWS_AMD64"] + '${CHECKSUM_WINDOWS_ARM64}' = $checksums["WINDOWS_ARM64"] +} + +foreach ($placeholder in $replacements.Keys) { + $template = $template.Replace($placeholder, $replacements[$placeholder]) +} + +$versionEntry = $template | ConvertFrom-Json + +# Add capabilities and providers (can't template arrays/objects easily) +$versionEntry.capabilities = $capabilities +if ($providers.Count -gt 0) { + $versionEntry | Add-Member -NotePropertyName "providers" -NotePropertyValue $providers +} + +# Build the full extension entry +$extEntry = [ordered]@{ + id = $AzdExtensionId + namespace = $extMeta.namespace + displayName = $extMeta.displayName + description = $extMeta.description + versions = @($versionEntry) +} + +# Download existing registry or create empty +# Use ErrorActionPreference Continue for azcopy since "not found" is expected on first run +$registryFile = "registry-daily.json" +$prevErrorPref = $ErrorActionPreference +$ErrorActionPreference = 'Continue' +$azcopyOutput = azcopy copy $RegistryBlobPath $registryFile 2>&1 +$azcopyExitCode = $LASTEXITCODE +$ErrorActionPreference = $prevErrorPref + +if ($azcopyExitCode -ne 0) { + # Check if this is a "not found" (expected) vs an actual error + $outputStr = $azcopyOutput | Out-String + if ($outputStr -match "BlobNotFound|404|does not exist") { + Write-Host "No existing registry found, creating new one" + } else { + Write-Warning "azcopy download failed (exit code $azcopyExitCode), creating new registry" + Write-Warning $outputStr + } + [ordered]@{ extensions = @() } | ConvertTo-Json -Depth 10 | Set-Content $registryFile +} + +$registry = Get-Content $registryFile -Raw | ConvertFrom-Json -Depth 20 + +# Merge: replace existing extension entry or add new +$found = $false +for ($i = 0; $i -lt $registry.extensions.Count; $i++) { + if ($registry.extensions[$i].id -eq $AzdExtensionId) { + $registry.extensions[$i] = $extEntry + $found = $true + Write-Host "Updated existing entry for $AzdExtensionId" + break + } +} +if (-not $found) { + $registry.extensions += $extEntry + Write-Host "Added new entry for $AzdExtensionId" +} + +# Write registry and validate JSON round-trip before uploading +$registryJson = $registry | ConvertTo-Json -Depth 20 +$registryJson | Set-Content $registryFile -Encoding utf8 + +# Validate the output is valid JSON +try { + $null = Get-Content $registryFile -Raw | ConvertFrom-Json -Depth 20 +} catch { + Write-Error "Generated registry JSON is invalid: $_" + exit 1 +} + +Write-Host "Registry contents:" +Get-Content $registryFile + +# Upload to storage +$ErrorActionPreference = 'Continue' +azcopy copy $registryFile $RegistryBlobPath --overwrite=true +if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to upload registry to $RegistryBlobPath (exit code $LASTEXITCODE)" + exit 1 +} + +Write-Host "Registry uploaded to $RegistryBlobPath" From c39a5b65a79b1529a5329d2921ce5542658500a3 Mon Sep 17 00:00:00 2001 From: Rajesh Kamal Date: Fri, 3 Apr 2026 17:34:31 -0700 Subject: [PATCH 03/10] Add daily version suffix for extension CI builds Mirrors core CLI pattern: appends -daily. for CI builds, -pr. for PR builds, skips for Manual (release) builds. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../stages/build-and-test-azd-extension.yml | 4 ++ .../steps/set-extension-version-cd.yml | 15 +++++++ eng/scripts/Set-ExtensionVersionInBuild.ps1 | 41 +++++++++++++++++++ 3 files changed, 60 insertions(+) create mode 100644 eng/pipelines/templates/steps/set-extension-version-cd.yml create mode 100644 eng/scripts/Set-ExtensionVersionInBuild.ps1 diff --git a/eng/pipelines/templates/stages/build-and-test-azd-extension.yml b/eng/pipelines/templates/stages/build-and-test-azd-extension.yml index 703f6e822e0..54f654dc847 100644 --- a/eng/pipelines/templates/stages/build-and-test-azd-extension.yml +++ b/eng/pipelines/templates/stages/build-and-test-azd-extension.yml @@ -110,6 +110,10 @@ stages: steps: - checkout: self + - template: /eng/pipelines/templates/steps/set-extension-version-cd.yml + parameters: + AzdExtensionDirectory: ${{ parameters.AzdExtensionDirectory }} + - task: PowerShell@2 displayName: Set extension version variable inputs: diff --git a/eng/pipelines/templates/steps/set-extension-version-cd.yml b/eng/pipelines/templates/steps/set-extension-version-cd.yml new file mode 100644 index 00000000000..2e402aeca06 --- /dev/null +++ b/eng/pipelines/templates/steps/set-extension-version-cd.yml @@ -0,0 +1,15 @@ +parameters: + - name: AzdExtensionDirectory + type: string + +steps: + - task: PowerShell@2 + displayName: Set extension version for CD release + inputs: + pwsh: true + targetType: filePath + filePath: eng/scripts/Set-ExtensionVersionInBuild.ps1 + arguments: >- + -ExtensionDirectory ${{ parameters.AzdExtensionDirectory }} + -BuildReason ($env:BUILDREASONOVERRIDE ?? '$(Build.Reason)') + -BuildId $(Build.BuildId) diff --git a/eng/scripts/Set-ExtensionVersionInBuild.ps1 b/eng/scripts/Set-ExtensionVersionInBuild.ps1 new file mode 100644 index 00000000000..0ddbbf13cf9 --- /dev/null +++ b/eng/scripts/Set-ExtensionVersionInBuild.ps1 @@ -0,0 +1,41 @@ +<# + .SYNOPSIS + Appends a prerelease suffix to the extension's version.txt for CI/PR builds. + Skips for Manual (release) builds. + + .PARAMETER ExtensionDirectory + Path to the extension directory containing version.txt. + + .PARAMETER BuildReason + The build reason from CI (e.g. Build.Reason). + + .PARAMETER BuildId + A unique build ID from CI (e.g. Build.BuildId). + #> +param( + [Parameter(Mandatory)] [string] $ExtensionDirectory, + [string] $BuildReason, + [string] $BuildId +) + +Write-Host "Build reason: $BuildReason" + +$prereleaseCategory = "" + +if ($BuildReason -eq "Manual") { + Write-Host "Skipping prerelease tagging for release build." + exit 0 +} +elseif ($BuildReason -eq "PullRequest") { + $prereleaseCategory = "pr" +} +else { + $prereleaseCategory = "daily" +} + +$versionFile = Join-Path $ExtensionDirectory "version.txt" +$version = Get-Content $versionFile +$newVersion = "$version-$prereleaseCategory.$BuildId" + +Set-Content $versionFile -Value $newVersion +Write-Host "Set version.txt contents to: $newVersion" From 9736f329694fb14d5a17316b4ad1a9c23e56aef3 Mon Sep 17 00:00:00 2001 From: Rajesh Kamal Date: Fri, 3 Apr 2026 17:39:56 -0700 Subject: [PATCH 04/10] Address hostile review: input validation, error handling, logging - Trim version.txt on read (Set-ExtensionVersionVariable, Set-ExtensionVersionInBuild) - Validate empty version.txt in Set-ExtensionVersionInBuild - Make BuildReason/BuildId mandatory params - Wrap Get-FileHash in try/catch - Validate required YAML fields (namespace, displayName, description) after parsing - Log parsed extension metadata for pipeline diagnostics - Fix upload error handling (don't set ErrorActionPreference=Continue before upload) - Guard CreateGitHubRelease against empty TagPrefix/TagVersion Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../templates/steps/publish-extension.yml | 2 +- eng/scripts/Set-ExtensionVersionInBuild.ps1 | 10 +++++-- eng/scripts/Set-ExtensionVersionVariable.ps1 | 2 +- eng/scripts/Update-ExtensionDailyRegistry.ps1 | 28 ++++++++++++++++--- 4 files changed, 33 insertions(+), 9 deletions(-) diff --git a/eng/pipelines/templates/steps/publish-extension.yml b/eng/pipelines/templates/steps/publish-extension.yml index 343cbda11e3..ce3997bcf8d 100644 --- a/eng/pipelines/templates/steps/publish-extension.yml +++ b/eng/pipelines/templates/steps/publish-extension.yml @@ -15,7 +15,7 @@ parameters: default: true steps: - - ${{ if eq(parameters.CreateGitHubRelease, true) }}: + - ${{ if and(eq(parameters.CreateGitHubRelease, true), ne(parameters.TagPrefix, ''), ne(parameters.TagVersion, '')) }}: # This step must run first because a duplicated tag means we don't need to # continue with any of the subsequent steps. - pwsh: | diff --git a/eng/scripts/Set-ExtensionVersionInBuild.ps1 b/eng/scripts/Set-ExtensionVersionInBuild.ps1 index 0ddbbf13cf9..3c31b6f7f6d 100644 --- a/eng/scripts/Set-ExtensionVersionInBuild.ps1 +++ b/eng/scripts/Set-ExtensionVersionInBuild.ps1 @@ -14,8 +14,8 @@ #> param( [Parameter(Mandatory)] [string] $ExtensionDirectory, - [string] $BuildReason, - [string] $BuildId + [Parameter(Mandatory)] [string] $BuildReason, + [Parameter(Mandatory)] [string] $BuildId ) Write-Host "Build reason: $BuildReason" @@ -34,7 +34,11 @@ else { } $versionFile = Join-Path $ExtensionDirectory "version.txt" -$version = Get-Content $versionFile +$version = (Get-Content $versionFile).Trim() +if ([string]::IsNullOrWhiteSpace($version)) { + Write-Error "version.txt is empty at $versionFile" + exit 1 +} $newVersion = "$version-$prereleaseCategory.$BuildId" Set-Content $versionFile -Value $newVersion diff --git a/eng/scripts/Set-ExtensionVersionVariable.ps1 b/eng/scripts/Set-ExtensionVersionVariable.ps1 index 34277f57182..36b2a348758 100644 --- a/eng/scripts/Set-ExtensionVersionVariable.ps1 +++ b/eng/scripts/Set-ExtensionVersionVariable.ps1 @@ -2,6 +2,6 @@ param( [string] $ExtensionDirectory ) -$extVersion = Get-Content "$ExtensionDirectory/version.txt" +$extVersion = (Get-Content "$ExtensionDirectory/version.txt").Trim() Write-Host "Extension Version: $extVersion" Write-Host "##vso[task.setvariable variable=EXT_VERSION;]$extVersion" diff --git a/eng/scripts/Update-ExtensionDailyRegistry.ps1 b/eng/scripts/Update-ExtensionDailyRegistry.ps1 index de2c2a3db4b..d8413767e09 100644 --- a/eng/scripts/Update-ExtensionDailyRegistry.ps1 +++ b/eng/scripts/Update-ExtensionDailyRegistry.ps1 @@ -73,9 +73,14 @@ $artifactFiles = @( foreach ($artifact in $artifactFiles) { $filePath = Join-Path $ReleasePath $artifact.file if (Test-Path $filePath) { - $hash = (Get-FileHash -Path $filePath -Algorithm SHA256).Hash.ToLower() - $checksums[$artifact.key] = $hash - Write-Host "Checksum $($artifact.key): $hash" + try { + $hash = (Get-FileHash -Path $filePath -Algorithm SHA256).Hash.ToLower() + $checksums[$artifact.key] = $hash + Write-Host "Checksum $($artifact.key): $hash" + } catch { + Write-Error "Failed to compute checksum for ${filePath}: $_" + exit 1 + } } else { $missingArtifacts += $artifact.file } @@ -126,6 +131,22 @@ foreach ($line in $extYaml -split "`n") { } if ($currentProvider) { $providers += $currentProvider } +# Validate required fields were parsed +$requiredFields = @('namespace', 'displayName', 'description') +foreach ($field in $requiredFields) { + if (-not $extMeta[$field] -or $extMeta[$field] -eq '') { + Write-Error "Required field '$field' missing or empty in extension.yaml" + exit 1 + } +} + +Write-Host "Parsed extension metadata:" +Write-Host " namespace: $($extMeta.namespace)" +Write-Host " displayName: $($extMeta.displayName)" +Write-Host " description: $($extMeta.description)" +Write-Host " capabilities: $($capabilities -join ', ')" +Write-Host " providers: $($providers.Count)" + # Load template and replace placeholders $template = Get-Content $TemplatePath -Raw $replacements = @{ @@ -217,7 +238,6 @@ Write-Host "Registry contents:" Get-Content $registryFile # Upload to storage -$ErrorActionPreference = 'Continue' azcopy copy $registryFile $RegistryBlobPath --overwrite=true if ($LASTEXITCODE -ne 0) { Write-Error "Failed to upload registry to $RegistryBlobPath (exit code $LASTEXITCODE)" From 32e10051cd5dc8dfb2eb26165c0b2397767a445b Mon Sep 17 00:00:00 2001 From: Rajesh Kamal Date: Mon, 6 Apr 2026 20:19:57 -0700 Subject: [PATCH 05/10] Address PR review: version override, per-extension registry, manifest fix, usage validation - Add set-extension-version-cd.yml to build and cross-build jobs so binaries get the daily version suffix (not just release artifacts) - Switch from shared registry-daily.json to per-extension entries (daily-registry-entries/{id}.json) to avoid race condition when multiple extension pipelines run concurrently - Make _manifest removal unconditional before conditional blocks - Add usage to required fields validation in registry script - Document YAML regex parsing limitation in code comment Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../templates/jobs/build-azd-extension.yml | 4 + .../jobs/cross-build-azd-extension.yml | 4 + .../templates/steps/publish-extension.yml | 24 ++--- .../steps/update-extension-daily-registry.yml | 6 +- eng/scripts/Update-ExtensionDailyRegistry.ps1 | 94 +++++++------------ 5 files changed, 52 insertions(+), 80 deletions(-) diff --git a/eng/pipelines/templates/jobs/build-azd-extension.yml b/eng/pipelines/templates/jobs/build-azd-extension.yml index a12ae98d977..e77e8385eb6 100644 --- a/eng/pipelines/templates/jobs/build-azd-extension.yml +++ b/eng/pipelines/templates/jobs/build-azd-extension.yml @@ -40,6 +40,10 @@ jobs: - template: /eng/pipelines/templates/steps/setup-go.yml + - template: /eng/pipelines/templates/steps/set-extension-version-cd.yml + parameters: + AzdExtensionDirectory: ${{ parameters.AzdExtensionDirectory }} + - task: PowerShell@2 inputs: pwsh: true diff --git a/eng/pipelines/templates/jobs/cross-build-azd-extension.yml b/eng/pipelines/templates/jobs/cross-build-azd-extension.yml index b8a711bfa19..db2c8a303a1 100644 --- a/eng/pipelines/templates/jobs/cross-build-azd-extension.yml +++ b/eng/pipelines/templates/jobs/cross-build-azd-extension.yml @@ -49,6 +49,10 @@ jobs: parameters: Condition: false + - template: /eng/pipelines/templates/steps/set-extension-version-cd.yml + parameters: + AzdExtensionDirectory: ${{ parameters.AzdExtensionDirectory }} + - task: PowerShell@2 displayName: Set extension version variable inputs: diff --git a/eng/pipelines/templates/steps/publish-extension.yml b/eng/pipelines/templates/steps/publish-extension.yml index ce3997bcf8d..5631830da3f 100644 --- a/eng/pipelines/templates/steps/publish-extension.yml +++ b/eng/pipelines/templates/steps/publish-extension.yml @@ -15,6 +15,15 @@ parameters: default: true steps: + # Remove _manifest folder unconditionally before upload/release + - pwsh: | + if (Test-Path release/_manifest) { + Remove-Item -Path release/_manifest -Recurse -Force + } + Write-Host "Release:" + Get-ChildItem -Recurse release/ | Select-Object -Property Length,FullName + displayName: Remove _manifest folder + - ${{ if and(eq(parameters.CreateGitHubRelease, true), ne(parameters.TagPrefix, ''), ne(parameters.TagVersion, '')) }}: # This step must run first because a duplicated tag means we don't need to # continue with any of the subsequent steps. @@ -44,12 +53,6 @@ steps: env: GH_TOKEN: $(azuresdk-github-pat) - - pwsh: | - Remove-Item -Path release/_manifest -Recurse -Force - Write-Host "Release:" - Get-ChildItem -Recurse release/ | Select-Object -Property Length,FullName - displayName: Remove _manifest folder - - pwsh: | $version = "${{ parameters.TagVersion }}" $createArgs = @( @@ -70,15 +73,6 @@ steps: env: GH_TOKEN: $(azuresdk-github-pat) - - ${{ if eq(parameters.CreateGitHubRelease, false) }}: - - pwsh: | - if (Test-Path release/_manifest) { - Remove-Item -Path release/_manifest -Recurse -Force - } - Write-Host "Release:" - Get-ChildItem -Recurse release/ | Select-Object -Property Length,FullName - displayName: Remove _manifest folder - - task: AzurePowerShell@5 displayName: Upload release to storage account inputs: diff --git a/eng/pipelines/templates/steps/update-extension-daily-registry.yml b/eng/pipelines/templates/steps/update-extension-daily-registry.yml index 61761ce368b..2cab7f74a3f 100644 --- a/eng/pipelines/templates/steps/update-extension-daily-registry.yml +++ b/eng/pipelines/templates/steps/update-extension-daily-registry.yml @@ -6,7 +6,7 @@ parameters: steps: - task: AzurePowerShell@5 - displayName: Update daily registry + displayName: Upload per-extension daily registry entry inputs: azureSubscription: 'Azure SDK Artifacts' azurePowerShellVersion: LatestVersion @@ -15,7 +15,7 @@ steps: Inline: | $storageHost = "$(publish-storage-static-host)" $dailyBaseUrl = "$storageHost/azd/extensions/${{ parameters.SanitizedExtensionId }}/daily" - $registryBlobPath = "$(publish-storage-location)/`$web/azd/extensions/registry-daily.json" + $entryBlobPath = "$(publish-storage-location)/`$web/azd/extensions/daily-registry-entries/${{ parameters.AzdExtensionId }}.json" $templatePath = "$(Build.SourcesDirectory)/eng/pipelines/templates/json/extension-registry-daily-template.json" & "$(Build.SourcesDirectory)/eng/scripts/Update-ExtensionDailyRegistry.ps1" ` @@ -23,7 +23,7 @@ steps: -AzdExtensionId "${{ parameters.AzdExtensionId }}" ` -Version "$(EXT_VERSION)" ` -StorageBaseUrl $dailyBaseUrl ` - -RegistryBlobPath $registryBlobPath ` + -RegistryEntryBlobPath $entryBlobPath ` -TemplatePath $templatePath env: AZCOPY_AUTO_LOGIN_TYPE: 'PSCRED' diff --git a/eng/scripts/Update-ExtensionDailyRegistry.ps1 b/eng/scripts/Update-ExtensionDailyRegistry.ps1 index d8413767e09..2d07d481e38 100644 --- a/eng/scripts/Update-ExtensionDailyRegistry.ps1 +++ b/eng/scripts/Update-ExtensionDailyRegistry.ps1 @@ -1,13 +1,17 @@ <# .SYNOPSIS - Updates registry-daily.json on Azure Storage with the current extension's daily build entry. + Writes a per-extension daily registry entry to Azure Storage. .DESCRIPTION 1. Computes checksums from signed release artifacts 2. Reads extension.yaml for metadata (id, namespace, displayName, etc.) 3. Loads the JSON template, replaces placeholders with actual values - 4. Downloads existing registry-daily.json from storage (or creates empty) - 5. Merges the new entry and uploads back + 4. Uploads the entry as a standalone per-extension JSON blob + + Each extension writes its own file to avoid race conditions when multiple + extension pipelines run concurrently. A separate unification script + (Build-UnifiedDailyRegistry.ps1) combines all per-extension entries into + the final registry-daily.json. .PARAMETER SanitizedExtensionId Hyphenated extension id (e.g. azure-ai-agents) @@ -21,8 +25,9 @@ .PARAMETER StorageBaseUrl Static storage host URL for daily artifacts -.PARAMETER RegistryBlobPath - Full blob path for registry-daily.json +.PARAMETER RegistryEntryBlobPath + Full blob path for the per-extension entry JSON + (e.g. .../azd/extensions/daily-registry-entries/azure.ai.agents.json) .PARAMETER ReleasePath Path to the signed release artifacts @@ -39,7 +44,7 @@ param( [Parameter(Mandatory)] [string] $AzdExtensionId, [Parameter(Mandatory)] [string] $Version, [Parameter(Mandatory)] [string] $StorageBaseUrl, - [Parameter(Mandatory)] [string] $RegistryBlobPath, + [Parameter(Mandatory)] [string] $RegistryEntryBlobPath, [Parameter(Mandatory)] [string] $TemplatePath, [string] $ReleasePath = "release", [string] $MetadataPath = "release-metadata" @@ -91,9 +96,12 @@ if ($missingArtifacts.Count -gt 0) { exit 1 } -# Read extension.yaml for metadata -# Simple line-by-line parsing for top-level scalar fields, capabilities list, -# and providers list. Sufficient for the known extension.yaml schema. +# Read extension.yaml for metadata. +# Uses simple line-by-line regex parsing — handles top-level scalar fields, +# capabilities list, and providers list. This is intentionally not a full YAML +# parser. It works for the known extension.yaml schema where all values are +# single-line scalars or simple lists. If extension.yaml grows multi-line +# values or complex nesting, switch to powershell-yaml. $extYaml = Get-Content $extYamlPath -Raw $extMeta = @{} foreach ($line in $extYaml -split "`n") { @@ -132,7 +140,7 @@ foreach ($line in $extYaml -split "`n") { if ($currentProvider) { $providers += $currentProvider } # Validate required fields were parsed -$requiredFields = @('namespace', 'displayName', 'description') +$requiredFields = @('namespace', 'displayName', 'description', 'usage') foreach ($field in $requiredFields) { if (-not $extMeta[$field] -or $extMeta[$field] -eq '') { Write-Error "Required field '$field' missing or empty in extension.yaml" @@ -144,6 +152,7 @@ Write-Host "Parsed extension metadata:" Write-Host " namespace: $($extMeta.namespace)" Write-Host " displayName: $($extMeta.displayName)" Write-Host " description: $($extMeta.description)" +Write-Host " usage: $($extMeta.usage)" Write-Host " capabilities: $($capabilities -join ', ')" Write-Host " providers: $($providers.Count)" @@ -152,7 +161,7 @@ $template = Get-Content $TemplatePath -Raw $replacements = @{ '${EXT_VERSION}' = $Version '${REQUIRED_AZD_VERSION}' = if ($extMeta.requiredAzdVersion) { $extMeta.requiredAzdVersion } else { "" } - '${USAGE}' = if ($extMeta.usage) { $extMeta.usage } else { "" } + '${USAGE}' = $extMeta.usage '${SANITIZED_ID}' = $SanitizedExtensionId '${STORAGE_BASE_URL}' = $StorageBaseUrl '${CHECKSUM_DARWIN_AMD64}' = $checksums["DARWIN_AMD64"] @@ -175,7 +184,7 @@ if ($providers.Count -gt 0) { $versionEntry | Add-Member -NotePropertyName "providers" -NotePropertyValue $providers } -# Build the full extension entry +# Build the per-extension entry $extEntry = [ordered]@{ id = $AzdExtensionId namespace = $extMeta.namespace @@ -184,64 +193,25 @@ $extEntry = [ordered]@{ versions = @($versionEntry) } -# Download existing registry or create empty -# Use ErrorActionPreference Continue for azcopy since "not found" is expected on first run -$registryFile = "registry-daily.json" -$prevErrorPref = $ErrorActionPreference -$ErrorActionPreference = 'Continue' -$azcopyOutput = azcopy copy $RegistryBlobPath $registryFile 2>&1 -$azcopyExitCode = $LASTEXITCODE -$ErrorActionPreference = $prevErrorPref - -if ($azcopyExitCode -ne 0) { - # Check if this is a "not found" (expected) vs an actual error - $outputStr = $azcopyOutput | Out-String - if ($outputStr -match "BlobNotFound|404|does not exist") { - Write-Host "No existing registry found, creating new one" - } else { - Write-Warning "azcopy download failed (exit code $azcopyExitCode), creating new registry" - Write-Warning $outputStr - } - [ordered]@{ extensions = @() } | ConvertTo-Json -Depth 10 | Set-Content $registryFile -} - -$registry = Get-Content $registryFile -Raw | ConvertFrom-Json -Depth 20 - -# Merge: replace existing extension entry or add new -$found = $false -for ($i = 0; $i -lt $registry.extensions.Count; $i++) { - if ($registry.extensions[$i].id -eq $AzdExtensionId) { - $registry.extensions[$i] = $extEntry - $found = $true - Write-Host "Updated existing entry for $AzdExtensionId" - break - } -} -if (-not $found) { - $registry.extensions += $extEntry - Write-Host "Added new entry for $AzdExtensionId" -} - -# Write registry and validate JSON round-trip before uploading -$registryJson = $registry | ConvertTo-Json -Depth 20 -$registryJson | Set-Content $registryFile -Encoding utf8 +# Write per-extension entry and validate JSON +$entryFile = "$AzdExtensionId.json" +$extEntry | ConvertTo-Json -Depth 20 | Set-Content $entryFile -Encoding utf8 -# Validate the output is valid JSON try { - $null = Get-Content $registryFile -Raw | ConvertFrom-Json -Depth 20 + $null = Get-Content $entryFile -Raw | ConvertFrom-Json -Depth 20 } catch { - Write-Error "Generated registry JSON is invalid: $_" + Write-Error "Generated entry JSON is invalid: $_" exit 1 } -Write-Host "Registry contents:" -Get-Content $registryFile +Write-Host "Extension entry:" +Get-Content $entryFile -# Upload to storage -azcopy copy $registryFile $RegistryBlobPath --overwrite=true +# Upload per-extension entry to storage +azcopy copy $entryFile $RegistryEntryBlobPath --overwrite=true if ($LASTEXITCODE -ne 0) { - Write-Error "Failed to upload registry to $RegistryBlobPath (exit code $LASTEXITCODE)" + Write-Error "Failed to upload entry to $RegistryEntryBlobPath (exit code $LASTEXITCODE)" exit 1 } -Write-Host "Registry uploaded to $RegistryBlobPath" +Write-Host "Entry uploaded to $RegistryEntryBlobPath" From fdb24a5f6b80083cfc62aa63274b216b8207c254 Mon Sep 17 00:00:00 2001 From: Rajesh Kamal Date: Tue, 7 Apr 2026 20:06:46 -0700 Subject: [PATCH 06/10] Address PR review feedback: powershell-yaml, registry format, error handling - Replace regex YAML parsing with powershell-yaml for proper parsing - Wrap JSON output in { extensions: [...] } registry format for azd compatibility - Move template to eng/templates/, copy scripts into release-metadata artifact - Flip CreateGitHubRelease default to false, explicit true in release caller - Add idempotency guard in version script for pipeline retries - Add placeholder validation, null checks, and file-exists guards - Split copy steps for clearer error attribution in CI Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../stages/build-and-test-azd-extension.yml | 12 +++ .../stages/publish-extension-daily.yml | 3 + .../templates/stages/publish-extension.yml | 1 + .../templates/steps/publish-extension.yml | 2 +- .../steps/update-extension-daily-registry.yml | 8 +- eng/scripts/Set-ExtensionVersionInBuild.ps1 | 11 +++ eng/scripts/Update-ExtensionDailyRegistry.ps1 | 83 ++++++++++--------- .../extensions-registry-daily.json.template} | 0 8 files changed, 76 insertions(+), 44 deletions(-) rename eng/{pipelines/templates/json/extension-registry-daily-template.json => templates/extensions-registry-daily.json.template} (100%) diff --git a/eng/pipelines/templates/stages/build-and-test-azd-extension.yml b/eng/pipelines/templates/stages/build-and-test-azd-extension.yml index 54f654dc847..b66db0f4c58 100644 --- a/eng/pipelines/templates/stages/build-and-test-azd-extension.yml +++ b/eng/pipelines/templates/stages/build-and-test-azd-extension.yml @@ -161,6 +161,18 @@ stages: Copy-Item NOTICE.txt release-metadata/NOTICE.txt displayName: Copy NOTICE.txt to release-metadata + - pwsh: | + Copy-Item eng/templates/extensions-registry-daily.json.template release-metadata/extensions-registry-daily.json.template + displayName: Copy registry template to release-metadata + + - pwsh: | + Copy-Item eng/scripts/Update-ExtensionDailyRegistry.ps1 release-metadata/Update-ExtensionDailyRegistry.ps1 + displayName: Copy Update-ExtensionDailyRegistry.ps1 to release-metadata + + - pwsh: | + Copy-Item eng/common/scripts/Helpers/PSModule-Helpers.ps1 release-metadata/PSModule-Helpers.ps1 + displayName: Copy PSModule-Helpers.ps1 to release-metadata + templateContext: outputs: - output: pipelineArtifact diff --git a/eng/pipelines/templates/stages/publish-extension-daily.yml b/eng/pipelines/templates/stages/publish-extension-daily.yml index b793c4f5f54..1d367d1b781 100644 --- a/eng/pipelines/templates/stages/publish-extension-daily.yml +++ b/eng/pipelines/templates/stages/publish-extension-daily.yml @@ -40,6 +40,9 @@ stages: - input: pipelineArtifact artifactName: release targetPath: release + - input: pipelineArtifact + artifactName: release-metadata + targetPath: release-metadata strategy: runOnce: diff --git a/eng/pipelines/templates/stages/publish-extension.yml b/eng/pipelines/templates/stages/publish-extension.yml index 704d4e2e076..f355ca9a5c5 100644 --- a/eng/pipelines/templates/stages/publish-extension.yml +++ b/eng/pipelines/templates/stages/publish-extension.yml @@ -67,5 +67,6 @@ stages: - template: /eng/pipelines/templates/steps/publish-extension.yml parameters: PublishUploadLocations: $(StorageUploadLocations) + CreateGitHubRelease: true TagPrefix: azd-ext-${{ parameters.SanitizedExtensionId }} TagVersion: $(EXT_VERSION) diff --git a/eng/pipelines/templates/steps/publish-extension.yml b/eng/pipelines/templates/steps/publish-extension.yml index 5631830da3f..f024e21674a 100644 --- a/eng/pipelines/templates/steps/publish-extension.yml +++ b/eng/pipelines/templates/steps/publish-extension.yml @@ -12,7 +12,7 @@ parameters: default: '' - name: CreateGitHubRelease type: boolean - default: true + default: false steps: # Remove _manifest folder unconditionally before upload/release diff --git a/eng/pipelines/templates/steps/update-extension-daily-registry.yml b/eng/pipelines/templates/steps/update-extension-daily-registry.yml index 2cab7f74a3f..f27f29a27a7 100644 --- a/eng/pipelines/templates/steps/update-extension-daily-registry.yml +++ b/eng/pipelines/templates/steps/update-extension-daily-registry.yml @@ -16,14 +16,16 @@ steps: $storageHost = "$(publish-storage-static-host)" $dailyBaseUrl = "$storageHost/azd/extensions/${{ parameters.SanitizedExtensionId }}/daily" $entryBlobPath = "$(publish-storage-location)/`$web/azd/extensions/daily-registry-entries/${{ parameters.AzdExtensionId }}.json" - $templatePath = "$(Build.SourcesDirectory)/eng/pipelines/templates/json/extension-registry-daily-template.json" + $templatePath = "release-metadata/extensions-registry-daily.json.template" - & "$(Build.SourcesDirectory)/eng/scripts/Update-ExtensionDailyRegistry.ps1" ` + & "release-metadata/Update-ExtensionDailyRegistry.ps1" ` -SanitizedExtensionId "${{ parameters.SanitizedExtensionId }}" ` -AzdExtensionId "${{ parameters.AzdExtensionId }}" ` -Version "$(EXT_VERSION)" ` -StorageBaseUrl $dailyBaseUrl ` -RegistryEntryBlobPath $entryBlobPath ` - -TemplatePath $templatePath + -TemplatePath $templatePath ` + -ReleasePath "release" ` + -MetadataPath "release-metadata" env: AZCOPY_AUTO_LOGIN_TYPE: 'PSCRED' diff --git a/eng/scripts/Set-ExtensionVersionInBuild.ps1 b/eng/scripts/Set-ExtensionVersionInBuild.ps1 index 3c31b6f7f6d..fbe5cf5307d 100644 --- a/eng/scripts/Set-ExtensionVersionInBuild.ps1 +++ b/eng/scripts/Set-ExtensionVersionInBuild.ps1 @@ -34,11 +34,22 @@ else { } $versionFile = Join-Path $ExtensionDirectory "version.txt" +if (!(Test-Path $versionFile)) { + Write-Error "version.txt not found at $versionFile" + exit 1 +} $version = (Get-Content $versionFile).Trim() if ([string]::IsNullOrWhiteSpace($version)) { Write-Error "version.txt is empty at $versionFile" exit 1 } + +# Guard against pipeline retries — skip if suffix already applied +if ($version -match "-${prereleaseCategory}\.\d+$") { + Write-Host "Version '$version' already has $prereleaseCategory suffix, skipping." + exit 0 +} + $newVersion = "$version-$prereleaseCategory.$BuildId" Set-Content $versionFile -Value $newVersion diff --git a/eng/scripts/Update-ExtensionDailyRegistry.ps1 b/eng/scripts/Update-ExtensionDailyRegistry.ps1 index 2d07d481e38..ea0a9a836cd 100644 --- a/eng/scripts/Update-ExtensionDailyRegistry.ps1 +++ b/eng/scripts/Update-ExtensionDailyRegistry.ps1 @@ -9,9 +9,8 @@ 4. Uploads the entry as a standalone per-extension JSON blob Each extension writes its own file to avoid race conditions when multiple - extension pipelines run concurrently. A separate unification script - (Build-UnifiedDailyRegistry.ps1) combines all per-extension entries into - the final registry-daily.json. + extension pipelines run concurrently. The azd CLI reads each per-extension + entry directly via the registry source URL. .PARAMETER SanitizedExtensionId Hyphenated extension id (e.g. azure-ai-agents) @@ -96,48 +95,43 @@ if ($missingArtifacts.Count -gt 0) { exit 1 } -# Read extension.yaml for metadata. -# Uses simple line-by-line regex parsing — handles top-level scalar fields, -# capabilities list, and providers list. This is intentionally not a full YAML -# parser. It works for the known extension.yaml schema where all values are -# single-line scalars or simple lists. If extension.yaml grows multi-line -# values or complex nesting, switch to powershell-yaml. -$extYaml = Get-Content $extYamlPath -Raw -$extMeta = @{} -foreach ($line in $extYaml -split "`n") { - if ($line -match "^(\w[\w\-]*):\s*(.+)$") { - $extMeta[$matches[1]] = $matches[2].Trim().Trim('"') - } +# Install powershell-yaml for proper YAML parsing +$psModuleHelpers = Join-Path $PSScriptRoot "PSModule-Helpers.ps1" +if (!(Test-Path $psModuleHelpers)) { + # Fallback to repo location when running from source checkout + $psModuleHelpers = Join-Path $PSScriptRoot "../common/scripts/Helpers/PSModule-Helpers.ps1" +} +if (!(Test-Path $psModuleHelpers)) { + Write-Error "PSModule-Helpers.ps1 not found at $PSScriptRoot or repo fallback path" + exit 1 +} +. $psModuleHelpers +Install-ModuleIfNotInstalled "powershell-yaml" "0.4.7" | Import-Module + +# Parse extension.yaml +$extData = ConvertFrom-Yaml (Get-Content $extYamlPath -Raw) +if ($null -eq $extData) { + Write-Error "Failed to parse extension.yaml at $extYamlPath — file may be empty or malformed" + exit 1 } -# Parse capabilities list -$capabilities = @() -$inCapabilities = $false -foreach ($line in $extYaml -split "`n") { - if ($line -match "^capabilities:") { $inCapabilities = $true; continue } - if ($inCapabilities -and $line -match "^\s+-\s+(.+)$") { - $capabilities += $matches[1].Trim() - } elseif ($inCapabilities -and $line -match "^\S") { - break +$extMeta = @{} +foreach ($key in @('namespace', 'displayName', 'description', 'usage', 'requiredAzdVersion')) { + if ($extData.ContainsKey($key)) { + $extMeta[$key] = $extData[$key] } } -# Parse providers list +$capabilities = if ($extData.ContainsKey('capabilities')) { @($extData['capabilities']) } else { @() } + $providers = @() -$inProviders = $false -$currentProvider = $null -foreach ($line in $extYaml -split "`n") { - if ($line -match "^providers:") { $inProviders = $true; continue } - if ($inProviders -and $line -match "^\s+-\s+name:\s*(.+)$") { - if ($currentProvider) { $providers += $currentProvider } - $currentProvider = [ordered]@{ name = $matches[1].Trim() } - } elseif ($inProviders -and $currentProvider -and $line -match "^\s+(\w+):\s*(.+)$") { - $currentProvider[$matches[1].Trim()] = $matches[2].Trim() - } elseif ($inProviders -and $line -match "^\S") { - break +if ($extData.ContainsKey('providers')) { + foreach ($p in $extData['providers']) { + $provider = [ordered]@{} + foreach ($k in $p.Keys) { $provider[$k] = $p[$k] } + $providers += $provider } } -if ($currentProvider) { $providers += $currentProvider } # Validate required fields were parsed $requiredFields = @('namespace', 'displayName', 'description', 'usage') @@ -176,6 +170,12 @@ foreach ($placeholder in $replacements.Keys) { $template = $template.Replace($placeholder, $replacements[$placeholder]) } +# Verify all placeholders were replaced +if ($template -match '\$\{[A-Za-z0-9_]+\}') { + Write-Error "Unreplaced placeholder found: $($matches[0])" + exit 1 +} + $versionEntry = $template | ConvertFrom-Json # Add capabilities and providers (can't template arrays/objects easily) @@ -184,7 +184,8 @@ if ($providers.Count -gt 0) { $versionEntry | Add-Member -NotePropertyName "providers" -NotePropertyValue $providers } -# Build the per-extension entry +# Build the per-extension entry wrapped in registry format +# azd ext source add -t url expects { "extensions": [...] } $extEntry = [ordered]@{ id = $AzdExtensionId namespace = $extMeta.namespace @@ -193,9 +194,11 @@ $extEntry = [ordered]@{ versions = @($versionEntry) } -# Write per-extension entry and validate JSON +$registry = [ordered]@{ extensions = @($extEntry) } + +# Write registry entry and validate JSON $entryFile = "$AzdExtensionId.json" -$extEntry | ConvertTo-Json -Depth 20 | Set-Content $entryFile -Encoding utf8 +$registry | ConvertTo-Json -Depth 20 | Set-Content $entryFile -Encoding utf8 try { $null = Get-Content $entryFile -Raw | ConvertFrom-Json -Depth 20 diff --git a/eng/pipelines/templates/json/extension-registry-daily-template.json b/eng/templates/extensions-registry-daily.json.template similarity index 100% rename from eng/pipelines/templates/json/extension-registry-daily-template.json rename to eng/templates/extensions-registry-daily.json.template From 49986eca66668063e3cc38b66bdd359dddcce6eb Mon Sep 17 00:00:00 2001 From: Rajesh Kamal Date: Thu, 9 Apr 2026 21:31:27 -0700 Subject: [PATCH 07/10] Add extension PR builds with registry and install comment - Add publish-extension-pr.yml stage for PR builds - Add update-extension-pr-registry.yml step for per-extension PR registry - Wire PR publish into release-azd-extension.yml pipeline - Add PR number validation guard for empty/unexpanded values - Add unsigned binary warning to PR comment - Add ConvertTo-JsonSafeString for JSON-safe template replacements - Bump azure.ai.agents version to 0.1.22-preview Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../extensions/azure.ai.agents/extension.yaml | 2 +- .../extensions/azure.ai.agents/version.txt | 2 +- .../templates/stages/publish-extension-pr.yml | 120 ++++++++++++++++++ .../stages/release-azd-extension.yml | 11 ++ .../steps/update-extension-pr-registry.yml | 33 +++++ eng/scripts/Update-ExtensionDailyRegistry.ps1 | 18 ++- 6 files changed, 179 insertions(+), 7 deletions(-) create mode 100644 eng/pipelines/templates/stages/publish-extension-pr.yml create mode 100644 eng/pipelines/templates/steps/update-extension-pr-registry.yml diff --git a/cli/azd/extensions/azure.ai.agents/extension.yaml b/cli/azd/extensions/azure.ai.agents/extension.yaml index 939c5e33bde..7e8ce5a957f 100644 --- a/cli/azd/extensions/azure.ai.agents/extension.yaml +++ b/cli/azd/extensions/azure.ai.agents/extension.yaml @@ -5,7 +5,7 @@ displayName: Foundry agents (Preview) description: Ship agents with Microsoft Foundry from your terminal. (Preview) usage: azd ai agent [options] # NOTE: Make sure version.txt is in sync with this version. -version: 0.1.21-preview +version: 0.1.22-preview requiredAzdVersion: ">1.23.13" language: go capabilities: diff --git a/cli/azd/extensions/azure.ai.agents/version.txt b/cli/azd/extensions/azure.ai.agents/version.txt index 84c8f2d9e45..440ea160368 100644 --- a/cli/azd/extensions/azure.ai.agents/version.txt +++ b/cli/azd/extensions/azure.ai.agents/version.txt @@ -1 +1 @@ -0.1.21-preview +0.1.22-preview diff --git a/eng/pipelines/templates/stages/publish-extension-pr.yml b/eng/pipelines/templates/stages/publish-extension-pr.yml new file mode 100644 index 00000000000..84a0048db69 --- /dev/null +++ b/eng/pipelines/templates/stages/publish-extension-pr.yml @@ -0,0 +1,120 @@ +parameters: + - name: SanitizedExtensionId + type: string + - name: AzdExtensionId + type: string + +stages: + - stage: PublishForPR + dependsOn: Sign + condition: >- + and( + succeeded(), + ne(variables['Skip.Release'], 'true'), + or( + eq('PullRequest', variables['BuildReasonOverride']), + and( + eq('', variables['BuildReasonOverride']), + eq(variables['Build.Reason'], 'PullRequest') + ) + ) + ) + + variables: + - template: /eng/pipelines/templates/variables/image.yml + - template: /eng/pipelines/templates/variables/globals.yml + + jobs: + - deployment: Publish_Extension_For_PR + environment: none + + pool: + name: azsdk-pool + image: ubuntu-22.04 + os: linux + + templateContext: + type: releaseJob + isProduction: false + inputs: + - input: pipelineArtifact + artifactName: release + targetPath: release + - input: pipelineArtifact + artifactName: release-metadata + targetPath: release-metadata + + strategy: + runOnce: + deploy: + steps: + - pwsh: | + $PRNumber = '$(System.PullRequest.PullRequestNumber)' + if ($env:PRNUMBEROVERRIDE) { + Write-Host "PR Number override: $($env:PRNUMBEROVERRIDE)" + $PRNumber = "$($env:PRNUMBEROVERRIDE)" + } + if (-not $PRNumber -or $PRNumber -match '^\$\(') { + Write-Error "PR number could not be determined. Ensure this runs in a PR build or set PRNUMBEROVERRIDE." + exit 1 + } + Write-Host "##vso[task.setvariable variable=PRNumber]$PRNumber" + displayName: Set PR Number Variable + + - template: /eng/pipelines/templates/steps/extension-set-metadata-variables.yml + parameters: + Use1ESArtifactTask: true + + - template: /eng/pipelines/templates/steps/publish-extension.yml + parameters: + PublishUploadLocations: ${{ parameters.SanitizedExtensionId }}/pr/$(PRNumber) + CreateGitHubRelease: false + + - template: /eng/pipelines/templates/steps/update-extension-pr-registry.yml + parameters: + SanitizedExtensionId: ${{ parameters.SanitizedExtensionId }} + AzdExtensionId: ${{ parameters.AzdExtensionId }} + PRNumber: $(PRNumber) + + - pwsh: | + $storageHost = "$(publish-storage-static-host)" + $extId = "${{ parameters.AzdExtensionId }}" + $sanitizedId = "${{ parameters.SanitizedExtensionId }}" + $prNumber = "$(PRNumber)" + $registryUrl = "$storageHost/azd/extensions/pr-registry-entries/$prNumber/$extId.json" + $binaryBase = "$storageHost/azd/extensions/$sanitizedId/pr/$prNumber" + + $content = @" + + ## azd Extension Install Instructions — ``$extId`` + + > :warning: **These are unsigned PR builds for testing only.** Do not use in production. + + ### Install from PR build + + ``````bash + azd ext source add -n pr-$prNumber -t url -l "$registryUrl" + azd ext install $extId --source pr-$prNumber + `````` + + ### Standalone Binaries + + | Platform | Download | + |----------|----------| + | Linux x86_64 | $binaryBase/$sanitizedId-linux-amd64.tar.gz | + | Linux ARM64 | $binaryBase/$sanitizedId-linux-arm64.tar.gz | + | macOS x86_64 | $binaryBase/$sanitizedId-darwin-amd64.zip | + | macOS ARM64 | $binaryBase/$sanitizedId-darwin-arm64.zip | + | Windows x86_64 | $binaryBase/$sanitizedId-windows-amd64.zip | + | Windows ARM64 | $binaryBase/$sanitizedId-windows-arm64.zip | + "@ + $file = New-TemporaryFile + Set-Content -Path $file -Value $content + Write-Host "##vso[task.setvariable variable=CommentBodyFile]$file" + displayName: Generate PR comment body + + - template: /eng/pipelines/templates/steps/update-prcomment.yml + parameters: + PrNumber: $(PRNumber) + BodyFile: $(CommentBodyFile) + Tag: '' diff --git a/eng/pipelines/templates/stages/release-azd-extension.yml b/eng/pipelines/templates/stages/release-azd-extension.yml index 0319c625d33..9852400a0eb 100644 --- a/eng/pipelines/templates/stages/release-azd-extension.yml +++ b/eng/pipelines/templates/stages/release-azd-extension.yml @@ -95,3 +95,14 @@ stages: parameters: SanitizedExtensionId: ${{ parameters.SanitizedExtensionId }} AzdExtensionId: ${{ parameters.AzdExtensionId }} + + # Publish PR builds to storage and post install instructions as PR comment + - ${{ if and(eq(variables['System.TeamProject'], 'internal'), eq(variables['Build.Reason'], 'PullRequest')) }}: + - template: /eng/pipelines/templates/stages/sign-extension.yml + parameters: + SanitizedExtensionId: ${{ parameters.SanitizedExtensionId }} + + - template: /eng/pipelines/templates/stages/publish-extension-pr.yml + parameters: + SanitizedExtensionId: ${{ parameters.SanitizedExtensionId }} + AzdExtensionId: ${{ parameters.AzdExtensionId }} diff --git a/eng/pipelines/templates/steps/update-extension-pr-registry.yml b/eng/pipelines/templates/steps/update-extension-pr-registry.yml new file mode 100644 index 00000000000..bc01049db1c --- /dev/null +++ b/eng/pipelines/templates/steps/update-extension-pr-registry.yml @@ -0,0 +1,33 @@ +parameters: + - name: SanitizedExtensionId + type: string + - name: AzdExtensionId + type: string + - name: PRNumber + type: string + +steps: + - task: AzurePowerShell@5 + displayName: Upload per-extension PR registry entry + inputs: + azureSubscription: 'Azure SDK Artifacts' + azurePowerShellVersion: LatestVersion + pwsh: true + ScriptType: InlineScript + Inline: | + $storageHost = "$(publish-storage-static-host)" + $prBaseUrl = "$storageHost/azd/extensions/${{ parameters.SanitizedExtensionId }}/pr/${{ parameters.PRNumber }}" + $entryBlobPath = "$(publish-storage-location)/`$web/azd/extensions/pr-registry-entries/${{ parameters.PRNumber }}/${{ parameters.AzdExtensionId }}.json" + $templatePath = "release-metadata/extensions-registry-daily.json.template" + + & "release-metadata/Update-ExtensionDailyRegistry.ps1" ` + -SanitizedExtensionId "${{ parameters.SanitizedExtensionId }}" ` + -AzdExtensionId "${{ parameters.AzdExtensionId }}" ` + -Version "$(EXT_VERSION)" ` + -StorageBaseUrl $prBaseUrl ` + -RegistryEntryBlobPath $entryBlobPath ` + -TemplatePath $templatePath ` + -ReleasePath "release" ` + -MetadataPath "release-metadata" + env: + AZCOPY_AUTO_LOGIN_TYPE: 'PSCRED' diff --git a/eng/scripts/Update-ExtensionDailyRegistry.ps1 b/eng/scripts/Update-ExtensionDailyRegistry.ps1 index ea0a9a836cd..f080ef0ffeb 100644 --- a/eng/scripts/Update-ExtensionDailyRegistry.ps1 +++ b/eng/scripts/Update-ExtensionDailyRegistry.ps1 @@ -151,13 +151,21 @@ Write-Host " capabilities: $($capabilities -join ', ')" Write-Host " providers: $($providers.Count)" # Load template and replace placeholders +# JSON-escape string values that are inserted into JSON string literals. +# This prevents characters like " \ and control chars from producing invalid JSON. +function ConvertTo-JsonSafeString([string]$value) { + # Use ConvertTo-Json to get a properly escaped JSON string, then strip the surrounding quotes + $escaped = $value | ConvertTo-Json + return $escaped.Substring(1, $escaped.Length - 2) +} + $template = Get-Content $TemplatePath -Raw $replacements = @{ - '${EXT_VERSION}' = $Version - '${REQUIRED_AZD_VERSION}' = if ($extMeta.requiredAzdVersion) { $extMeta.requiredAzdVersion } else { "" } - '${USAGE}' = $extMeta.usage - '${SANITIZED_ID}' = $SanitizedExtensionId - '${STORAGE_BASE_URL}' = $StorageBaseUrl + '${EXT_VERSION}' = ConvertTo-JsonSafeString $Version + '${REQUIRED_AZD_VERSION}' = ConvertTo-JsonSafeString ($(if ($extMeta.requiredAzdVersion) { $extMeta.requiredAzdVersion } else { "" })) + '${USAGE}' = ConvertTo-JsonSafeString $extMeta.usage + '${SANITIZED_ID}' = ConvertTo-JsonSafeString $SanitizedExtensionId + '${STORAGE_BASE_URL}' = ConvertTo-JsonSafeString $StorageBaseUrl '${CHECKSUM_DARWIN_AMD64}' = $checksums["DARWIN_AMD64"] '${CHECKSUM_DARWIN_ARM64}' = $checksums["DARWIN_ARM64"] '${CHECKSUM_LINUX_AMD64}' = $checksums["LINUX_AMD64"] From eb8b3a96e98dfc242b563897e53c186c91cbec3f Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Fri, 10 Apr 2026 15:43:14 -0700 Subject: [PATCH 08/10] Use runtime conditions for publishing based on build reason or overrides --- eng/pipelines/templates/stages/release-azd-extension.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/eng/pipelines/templates/stages/release-azd-extension.yml b/eng/pipelines/templates/stages/release-azd-extension.yml index 9852400a0eb..6ba8cb45252 100644 --- a/eng/pipelines/templates/stages/release-azd-extension.yml +++ b/eng/pipelines/templates/stages/release-azd-extension.yml @@ -76,7 +76,7 @@ stages: AZURE_DEV_CI_OS: mac-arm64 # Only sign and release on manual builds from internal - - ${{ if and(eq(variables['System.TeamProject'], 'internal'), eq(variables['Build.Reason'], 'Manual')) }}: + - ${{ if and(eq(variables['System.TeamProject'], 'internal'), in(variables['Build.Reason'], 'Manual', 'IndividualCI', 'BatchedCI')) }}: - template: /eng/pipelines/templates/stages/sign-extension.yml parameters: SanitizedExtensionId: ${{ parameters.SanitizedExtensionId }} @@ -85,12 +85,6 @@ stages: parameters: SanitizedExtensionId: ${{ parameters.SanitizedExtensionId }} - # Publish daily builds to storage on CI (push to main) - - ${{ if and(eq(variables['System.TeamProject'], 'internal'), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI')) }}: - - template: /eng/pipelines/templates/stages/sign-extension.yml - parameters: - SanitizedExtensionId: ${{ parameters.SanitizedExtensionId }} - - template: /eng/pipelines/templates/stages/publish-extension-daily.yml parameters: SanitizedExtensionId: ${{ parameters.SanitizedExtensionId }} From 08a30f9258863748b7dc63b083699948c15291a6 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Fri, 10 Apr 2026 15:45:18 -0700 Subject: [PATCH 09/10] Skip tests --- eng/pipelines/release-ext-microsoft-azd-demo.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eng/pipelines/release-ext-microsoft-azd-demo.yml b/eng/pipelines/release-ext-microsoft-azd-demo.yml index a5995f1602e..8e938c8c7bf 100644 --- a/eng/pipelines/release-ext-microsoft-azd-demo.yml +++ b/eng/pipelines/release-ext-microsoft-azd-demo.yml @@ -29,6 +29,8 @@ extends: stages: - template: /eng/pipelines/templates/stages/release-azd-extension.yml parameters: + # TODO: Revert before merging + SkipTests: true AzdExtensionId: microsoft.azd.demo SanitizedExtensionId: microsoft-azd-demo AzdExtensionDirectory: cli/azd/extensions/microsoft.azd.demo From 0c7bc3a21fff77dea801cdb5cdcb72ca5299af02 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Fri, 10 Apr 2026 16:05:33 -0700 Subject: [PATCH 10/10] Skip ValidateCrossCompile --- .../templates/stages/build-and-test-azd-extension.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/eng/pipelines/templates/stages/build-and-test-azd-extension.yml b/eng/pipelines/templates/stages/build-and-test-azd-extension.yml index b66db0f4c58..5a913e61b2c 100644 --- a/eng/pipelines/templates/stages/build-and-test-azd-extension.yml +++ b/eng/pipelines/templates/stages/build-and-test-azd-extension.yml @@ -95,7 +95,10 @@ stages: OSVmImage: ${{ build.value.OSVmImage }} OS: ${{ build.value.OS }} Variables: ${{ build.value.Variables }} - ValidateCrossCompile: ${{ build.value.ValidateCrossCompile }} + # TODO: Revert this before merge + # ValidateCrossCompile: ${{ build.value.ValidateCrossCompile }} + # TODO: Remove this line before merge + ValidateCrossCompile: false ValidateVm: ${{ build.value.ValidateVm }} ValidationTask: ${{ build.value.ValidationTask }} ValidationScript: ${{ build.value.ValidationScript }}