diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 5a584ea6..b5b243e0 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 65e09c040143048211dcf6b2bd69336cbf27eec6 + 55c97a45af8fd75bc87e9e4b4f61aef3fead254d diff --git a/eng/common/core-templates/job/publish-build-assets.yml b/eng/common/core-templates/job/publish-build-assets.yml index 8b5c635f..c9ee8ffd 100644 --- a/eng/common/core-templates/job/publish-build-assets.yml +++ b/eng/common/core-templates/job/publish-build-assets.yml @@ -80,7 +80,7 @@ jobs: # If it's not devdiv, it's dnceng ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: name: NetCore1ESPool-Publishing-Internal - image: windows.vs2019.amd64 + image: windows.vs2022.amd64 os: windows steps: - ${{ if eq(parameters.is1ESPipeline, '') }}: diff --git a/eng/common/core-templates/job/renovate.yml b/eng/common/core-templates/job/renovate.yml new file mode 100644 index 00000000..b662269d --- /dev/null +++ b/eng/common/core-templates/job/renovate.yml @@ -0,0 +1,135 @@ +# -------------------------------------------------------------------------------------- +# Renovate Bot Job Template +# -------------------------------------------------------------------------------------- +# This Azure DevOps pipeline job template runs Renovate (https://docs.renovatebot.com/) +# to automatically update dependencies in a GitHub repository. +# +# Renovate scans the repository for dependency files and creates pull requests to update +# outdated dependencies based on the configuration specified in the renovateConfigPath +# parameter. +# +# Usage: +# For each product repo wanting to make use of Renovate, this template is called from +# an internal Azure DevOps pipeline, typically with a schedule trigger, to check for +# and propose dependency updates. +# +# For more info, see https://github.com/dotnet/arcade/blob/main/Documentation/Renovate.md +# -------------------------------------------------------------------------------------- + +parameters: + +# Path to the Renovate configuration file within the repository. +- name: renovateConfigPath + type: string + default: 'eng/renovate.json' + +# GitHub repository to run Renovate against, in the format 'owner/repo'. +# This could technically be any repo but convention is to target the same +# repo that contains the calling pipeline. The Renovate config file would +# be co-located with the pipeline's repo and, in most cases, the config +# file is specific to the repo being targeted. +- name: gitHubRepo + type: string + +# List of base branches to target for Renovate PRs. +# NOTE: The Renovate configuration file is always read from the branch where the +# pipeline is run, NOT from the target branches specified here. If you need different +# configurations for different branches, run the pipeline from each branch separately. +- name: baseBranches + type: object + default: + - main + +# When true, Renovate will run in dry run mode, which previews changes without creating PRs. +# See the 'Run Renovate' step log output for details of what would have been changed. +- name: dryRun + type: boolean + default: false + +# By default, Renovate will not recreate a PR for a given dependency/version pair that was +# previously closed. This allows opting in to always recreating PRs even if they were +# previously closed. +- name: forceRecreatePR + type: boolean + default: false + +# Pool configuration for the job. +- name: pool + type: object + default: + name: NetCore1ESPool-Internal + image: build.azurelinux.3.amd64 + os: linux + +jobs: +- job: Renovate + displayName: Run Renovate + container: RenovateContainer + variables: + - group: dotnet-renovate-bot + # The Renovate version is automatically updated by https://github.com/dotnet/arcade/blob/main/azure-pipelines-renovate.yml. + # Changing the variable name here would require updating the name in https://github.com/dotnet/arcade/blob/main/eng/renovate.json as well. + - name: renovateVersion + value: '42' + - name: dryRunArg + ${{ if eq(parameters.dryRun, true) }}: + value: 'full' + ${{ else }}: + value: '' + - name: recreateWhenArg + ${{ if eq(parameters.forceRecreatePR, true) }}: + value: 'always' + ${{ else }}: + value: '' + pool: ${{ parameters.pool }} + + templateContext: + outputParentDirectory: $(Build.ArtifactStagingDirectory) + outputs: + - output: pipelineArtifact + displayName: Publish Renovate Log + condition: succeededOrFailed() + targetPath: $(Build.ArtifactStagingDirectory) + artifactName: $(Agent.JobName)_Logs_Attempt$(System.JobAttempt) + sbomEnabled: false + + steps: + - checkout: self + fetchDepth: 1 + + - script: renovate-config-validator $(Build.SourcesDirectory)/${{parameters.renovateConfigPath}} + displayName: Validate Renovate config + env: + LOG_LEVEL: info + LOG_FILE_LEVEL: debug + LOG_FILE: $(Build.ArtifactStagingDirectory)/renovate-config-validator.json + + - script: | + . $(Build.SourcesDirectory)/eng/common/renovate.env + renovate + displayName: Run Renovate + env: + RENOVATE_FORK_TOKEN: $(BotAccount-dotnet-renovate-bot-PAT) + RENOVATE_TOKEN: $(BotAccount-dotnet-renovate-bot-PAT) + RENOVATE_REPOSITORIES: ${{parameters.gitHubRepo}} + RENOVATE_BASE_BRANCHES: ${{ convertToJson(parameters.baseBranches) }} + RENOVATE_DRY_RUN: $(dryRunArg) + RENOVATE_RECREATE_WHEN: $(recreateWhenArg) + LOG_LEVEL: info + LOG_FILE_LEVEL: debug + LOG_FILE: $(Build.ArtifactStagingDirectory)/renovate.json + RENOVATE_CONFIG_FILE: $(Build.SourcesDirectory)/${{parameters.renovateConfigPath}} + + - script: | + echo "PRs created by Renovate:" + if [ -s "$(Build.ArtifactStagingDirectory)/renovate-log.json" ]; then + if ! jq -r 'select(.msg == "PR created" and .pr != null) | "https://github.com/\(.repository)/pull/\(.pr)"' "$(Build.ArtifactStagingDirectory)/renovate-log.json" | sort -u; then + echo "##vso[task.logissue type=warning]Failed to parse Renovate log file with jq." + echo "##vso[task.complete result=SucceededWithIssues]" + fi + else + echo "##vso[task.logissue type=warning]No Renovate log file found or file is empty." + echo "##vso[task.complete result=SucceededWithIssues]" + fi + displayName: List created PRs + condition: and(succeededOrFailed(), eq('${{ parameters.dryRun }}', false)) diff --git a/eng/common/core-templates/job/source-build.yml b/eng/common/core-templates/job/source-build.yml index 9d820f97..1997c2ae 100644 --- a/eng/common/core-templates/job/source-build.yml +++ b/eng/common/core-templates/job/source-build.yml @@ -60,19 +60,19 @@ jobs: pool: ${{ if eq(variables['System.TeamProject'], 'public') }}: name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore-Svc-Public' ), False, 'NetCore-Public')] - demands: ImageOverride -equals build.ubuntu.2204.amd64 + demands: ImageOverride -equals build.azurelinux.3.amd64.open ${{ if eq(variables['System.TeamProject'], 'internal') }}: name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore1ESPool-Svc-Internal'), False, 'NetCore1ESPool-Internal')] - image: 1es-azurelinux-3 + image: build.azurelinux.3.amd64 os: linux ${{ else }}: pool: ${{ if eq(variables['System.TeamProject'], 'public') }}: name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore-Svc-Public' ), False, 'NetCore-Public')] - demands: ImageOverride -equals Build.Ubuntu.2204.Amd64.Open + demands: ImageOverride -equals build.azurelinux.3.amd64.open ${{ if eq(variables['System.TeamProject'], 'internal') }}: name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore1ESPool-Svc-Internal'), False, 'NetCore1ESPool-Internal')] - demands: ImageOverride -equals Build.Ubuntu.2204.Amd64 + demands: ImageOverride -equals build.azurelinux.3.amd64 ${{ if ne(parameters.platform.pool, '') }}: pool: ${{ parameters.platform.pool }} diff --git a/eng/common/core-templates/job/source-index-stage1.yml b/eng/common/core-templates/job/source-index-stage1.yml index 76baf5c2..cf02b82d 100644 --- a/eng/common/core-templates/job/source-index-stage1.yml +++ b/eng/common/core-templates/job/source-index-stage1.yml @@ -25,10 +25,10 @@ jobs: pool: ${{ if eq(variables['System.TeamProject'], 'public') }}: name: $(DncEngPublicBuildPool) - image: windows.vs2026preview.scout.amd64.open + image: windows.vs2026.amd64.open ${{ if eq(variables['System.TeamProject'], 'internal') }}: name: $(DncEngInternalBuildPool) - image: windows.vs2026preview.scout.amd64 + image: windows.vs2026.amd64 steps: - ${{ if eq(parameters.is1ESPipeline, '') }}: diff --git a/eng/common/core-templates/post-build/post-build.yml b/eng/common/core-templates/post-build/post-build.yml index 06864cd1..9438429c 100644 --- a/eng/common/core-templates/post-build/post-build.yml +++ b/eng/common/core-templates/post-build/post-build.yml @@ -127,11 +127,11 @@ stages: ${{ else }}: ${{ if eq(parameters.is1ESPipeline, true) }}: name: $(DncEngInternalBuildPool) - image: windows.vs2026preview.scout.amd64 + image: windows.vs2026.amd64 os: windows ${{ else }}: name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals windows.vs2026preview.scout.amd64 + demands: ImageOverride -equals windows.vs2026.amd64 steps: - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml @@ -175,7 +175,7 @@ stages: os: windows ${{ else }}: name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals windows.vs2026preview.scout.amd64 + demands: ImageOverride -equals windows.vs2026.amd64 steps: - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml parameters: @@ -236,7 +236,7 @@ stages: os: windows ${{ else }}: name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals windows.vs2026preview.scout.amd64 + demands: ImageOverride -equals windows.vs2026.amd64 steps: - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml parameters: @@ -293,11 +293,11 @@ stages: ${{ else }}: ${{ if eq(parameters.is1ESPipeline, true) }}: name: NetCore1ESPool-Publishing-Internal - image: windows.vs2019.amd64 + image: windows.vs2022.amd64 os: windows ${{ else }}: name: NetCore1ESPool-Publishing-Internal - demands: ImageOverride -equals windows.vs2019.amd64 + demands: ImageOverride -equals windows.vs2022.amd64 steps: - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml parameters: diff --git a/eng/common/core-templates/stages/renovate.yml b/eng/common/core-templates/stages/renovate.yml new file mode 100644 index 00000000..383e397e --- /dev/null +++ b/eng/common/core-templates/stages/renovate.yml @@ -0,0 +1,86 @@ +# -------------------------------------------------------------------------------------- +# Renovate Pipeline Template +# -------------------------------------------------------------------------------------- +# This template provides a complete reusable pipeline definition for running Renovate +# in a 1ES Official pipeline. Pipelines can extend from this template and only need +# to pass the Renovate job parameters. +# +# For more info, see https://github.com/dotnet/arcade/blob/main/Documentation/Renovate.md +# -------------------------------------------------------------------------------------- + +parameters: + +# Path to the Renovate configuration file within the repository. +- name: renovateConfigPath + type: string + default: 'eng/renovate.json' + +# GitHub repository to run Renovate against, in the format 'owner/repo'. +- name: gitHubRepo + type: string + +# List of base branches to target for Renovate PRs. +- name: baseBranches + type: object + default: + - main + +# When true, Renovate will run in dry run mode. +- name: dryRun + type: boolean + default: false + +# When true, Renovate will recreate PRs even if they were previously closed. +- name: forceRecreatePR + type: boolean + default: false + +# Pool configuration for the pipeline. +- name: pool + type: object + default: + name: NetCore1ESPool-Internal + image: build.azurelinux.3.amd64 + os: linux + +# Renovate version used in the container image tag. +- name: renovateVersion + default: 43 + type: number + +# Pool configuration for SDL analysis. +- name: sdlPool + type: object + default: + name: NetCore1ESPool-Internal + image: 1es-windows-2022 + os: windows + +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release + +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + parameters: + pool: ${{ parameters.pool }} + sdl: + sourceAnalysisPool: ${{ parameters.sdlPool }} + containers: + RenovateContainer: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:azurelinux-3.0-renovate-${{ parameters.renovateVersion }}-amd64 + stages: + - stage: Renovate + displayName: Run Renovate + jobs: + - template: /eng/common/core-templates/job/renovate.yml@self + parameters: + renovateConfigPath: ${{ parameters.renovateConfigPath }} + gitHubRepo: ${{ parameters.gitHubRepo }} + baseBranches: ${{ parameters.baseBranches }} + dryRun: ${{ parameters.dryRun }} + forceRecreatePR: ${{ parameters.forceRecreatePR }} + pool: ${{ parameters.pool }} diff --git a/eng/common/core-templates/steps/install-microbuild-impl.yml b/eng/common/core-templates/steps/install-microbuild-impl.yml index b9e0143e..da22beb3 100644 --- a/eng/common/core-templates/steps/install-microbuild-impl.yml +++ b/eng/common/core-templates/steps/install-microbuild-impl.yml @@ -18,7 +18,7 @@ parameters: type: boolean steps: -- ${{ if eq(parameters.enablePreviewMicrobuild, 'true') }}: +- ${{ if eq(parameters.enablePreviewMicrobuild, true) }}: - task: MicroBuildSigningPluginPreview@4 displayName: Install Preview MicroBuild plugin inputs: ${{ parameters.microbuildTaskInputs }} diff --git a/eng/common/core-templates/steps/source-index-stage1-publish.yml b/eng/common/core-templates/steps/source-index-stage1-publish.yml index ac019e2d..3ad83b8c 100644 --- a/eng/common/core-templates/steps/source-index-stage1-publish.yml +++ b/eng/common/core-templates/steps/source-index-stage1-publish.yml @@ -14,8 +14,8 @@ steps: workingDirectory: $(Agent.TempDirectory) - script: | - $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version ${{parameters.sourceIndexProcessBinlogPackageVersion}} --source ${{parameters.SourceIndexPackageSource}} --tool-path $(Agent.TempDirectory)/.source-index/tools - $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version ${{parameters.sourceIndexUploadPackageVersion}} --source ${{parameters.SourceIndexPackageSource}} --tool-path $(Agent.TempDirectory)/.source-index/tools + $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version ${{parameters.sourceIndexProcessBinlogPackageVersion}} --source ${{parameters.sourceIndexPackageSource}} --tool-path $(Agent.TempDirectory)/.source-index/tools + $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version ${{parameters.sourceIndexUploadPackageVersion}} --source ${{parameters.sourceIndexPackageSource}} --tool-path $(Agent.TempDirectory)/.source-index/tools displayName: "Source Index: Download netsourceindex Tools" # Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk. workingDirectory: $(Agent.TempDirectory) diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh index 8abfb71f..ef97671f 100755 --- a/eng/common/cross/build-rootfs.sh +++ b/eng/common/cross/build-rootfs.sh @@ -72,7 +72,7 @@ __AlpinePackages+=" krb5-dev" __AlpinePackages+=" openssl-dev" __AlpinePackages+=" zlib-dev" -__FreeBSDBase="13.4-RELEASE" +__FreeBSDBase="13.5-RELEASE" __FreeBSDPkg="1.21.3" __FreeBSDABI="13" __FreeBSDPackages="libunwind" @@ -295,9 +295,7 @@ while :; do ;; noble) # Ubuntu 24.04 __CodeName=noble - if [[ -z "$__LLDB_Package" ]]; then - __LLDB_Package="liblldb-19-dev" - fi + __LLDB_Package="liblldb-19-dev" ;; stretch) # Debian 9 __CodeName=stretch @@ -383,7 +381,7 @@ while :; do ;; freebsd14) __CodeName=freebsd - __FreeBSDBase="14.2-RELEASE" + __FreeBSDBase="14.3-RELEASE" __FreeBSDABI="14" __SkipUnmount=1 ;; diff --git a/eng/common/internal-feed-operations.ps1 b/eng/common/internal-feed-operations.ps1 index 92b77347..c282d3ae 100644 --- a/eng/common/internal-feed-operations.ps1 +++ b/eng/common/internal-feed-operations.ps1 @@ -26,7 +26,7 @@ function SetupCredProvider { $url = 'https://raw.githubusercontent.com/microsoft/artifacts-credprovider/master/helpers/installcredprovider.ps1' Write-Host "Writing the contents of 'installcredprovider.ps1' locally..." - Invoke-WebRequest $url -OutFile installcredprovider.ps1 + Invoke-WebRequest $url -UseBasicParsing -OutFile installcredprovider.ps1 Write-Host 'Installing plugin...' .\installcredprovider.ps1 -Force diff --git a/eng/common/native/install-dependencies.sh b/eng/common/native/install-dependencies.sh index 64b87d0b..4742177a 100644 --- a/eng/common/native/install-dependencies.sh +++ b/eng/common/native/install-dependencies.sh @@ -24,16 +24,16 @@ case "$os" in apt update apt install -y build-essential gettext locales cmake llvm clang lld lldb liblldb-dev libunwind8-dev libicu-dev liblttng-ust-dev \ - libssl-dev libkrb5-dev pigz cpio + libssl-dev libkrb5-dev pigz cpio ninja-build localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 - elif [ "$ID" = "fedora" ] || [ "$ID" = "rhel" ] || [ "$ID" = "azurelinux" ] || [ "$ID" = "centos"]; then + elif [ "$ID" = "fedora" ] || [ "$ID" = "rhel" ] || [ "$ID" = "azurelinux" ] || [ "$ID" = "centos" ]; then pkg_mgr="$(command -v tdnf 2>/dev/null || command -v dnf)" - $pkg_mgr install -y cmake llvm lld lldb clang python curl libicu-devel openssl-devel krb5-devel lttng-ust-devel pigz cpio + $pkg_mgr install -y cmake llvm lld lldb clang python curl libicu-devel openssl-devel krb5-devel lttng-ust-devel pigz cpio ninja-build elif [ "$ID" = "amzn" ]; then - dnf install -y cmake llvm lld lldb clang python libicu-devel openssl-devel krb5-devel lttng-ust-devel pigz cpio + dnf install -y cmake llvm lld lldb clang python libicu-devel openssl-devel krb5-devel lttng-ust-devel pigz cpio ninja-build elif [ "$ID" = "alpine" ]; then - apk add build-base cmake bash curl clang llvm-dev lld lldb krb5-dev lttng-ust-dev icu-dev openssl-dev pigz cpio + apk add build-base cmake bash curl clang llvm llvm-dev lld lldb-dev krb5-dev lttng-ust-dev icu-dev openssl-dev pigz cpio ninja else echo "Unsupported distro. distro: $ID" exit 1 @@ -54,6 +54,7 @@ brew "openssl@3" brew "pkgconf" brew "python3" brew "pigz" +brew "ninja" EOF ;; diff --git a/eng/common/post-build/nuget-verification.ps1 b/eng/common/post-build/nuget-verification.ps1 index ac5c69ff..eea88e65 100644 --- a/eng/common/post-build/nuget-verification.ps1 +++ b/eng/common/post-build/nuget-verification.ps1 @@ -65,7 +65,7 @@ if ($NuGetExePath) { Write-Host "Downloading nuget.exe from $nugetExeUrl..." $ProgressPreference = 'SilentlyContinue' try { - Invoke-WebRequest $nugetExeUrl -OutFile $downloadedNuGetExe + Invoke-WebRequest $nugetExeUrl -UseBasicParsing -OutFile $downloadedNuGetExe $ProgressPreference = 'Continue' } catch { $ProgressPreference = 'Continue' diff --git a/eng/common/renovate.env b/eng/common/renovate.env new file mode 100644 index 00000000..9f79dbc6 --- /dev/null +++ b/eng/common/renovate.env @@ -0,0 +1,39 @@ +# Renovate Global Configuration +# https://docs.renovatebot.com/self-hosted-configuration/ +# +# NOTE: This file uses bash/shell format and is sourced via `. renovate.env`. +# Values containing spaces or special characters must be quoted. + +# Author to use for git commits made by Renovate +# https://docs.renovatebot.com/configuration-options/#gitauthor +export RENOVATE_GIT_AUTHOR='.NET Renovate ' + +# Disable rate limiting for PR creation (0 = unlimited) +# https://docs.renovatebot.com/presets-default/#prhourlylimitnone +# https://docs.renovatebot.com/presets-default/#prconcurrentlimitnone +export RENOVATE_PR_HOURLY_LIMIT=0 +export RENOVATE_PR_CONCURRENT_LIMIT=0 + +# Skip the onboarding PR that Renovate normally creates for new repos +# https://docs.renovatebot.com/config-overview/#onboarding +export RENOVATE_ONBOARDING=false + +# Any Renovate config file in the cloned repository is ignored. Only +# the Renovate config file from the repo where the pipeline is running +# is used (yes, those are the same repo but the sources may be different). +# https://docs.renovatebot.com/self-hosted-configuration/#requireconfig +export RENOVATE_REQUIRE_CONFIG=ignored + +# Customize the PR body content. This removes some of the default +# sections that aren't relevant in a self-hosted config. +# https://docs.renovatebot.com/configuration-options/#prheader +# https://docs.renovatebot.com/configuration-options/#prbodynotes +# https://docs.renovatebot.com/configuration-options/#prbodytemplate +export RENOVATE_PR_HEADER='## Automated Dependency Update' +export RENOVATE_PR_BODY_NOTES='["This PR has been created automatically by the [.NET Renovate Bot](https://github.com/dotnet/arcade/blob/main/Documentation/Renovate.md) to update one or more dependencies in your repo. Please review the changes and merge the PR if everything looks good."]' +export RENOVATE_PR_BODY_TEMPLATE='{{{header}}}{{{table}}}{{{warnings}}}{{{notes}}}{{{changelogs}}}' + +# Extend the global config with additional presets +# https://docs.renovatebot.com/self-hosted-configuration/#globalextends +# Disable the Dependency Dashboard issue that tracks all updates +export RENOVATE_GLOBAL_EXTENDS='[":disableDependencyDashboard"]' diff --git a/eng/common/templates/steps/vmr-sync.yml b/eng/common/templates/steps/vmr-sync.yml index 599afb61..eb619c50 100644 --- a/eng/common/templates/steps/vmr-sync.yml +++ b/eng/common/templates/steps/vmr-sync.yml @@ -38,27 +38,6 @@ steps: displayName: Label PR commit workingDirectory: $(Agent.BuildDirectory)/repo -- script: | - vmr_sha=$(grep -oP '(?<=Sha=")[^"]*' $(Agent.BuildDirectory)/repo/eng/Version.Details.xml) - echo "##vso[task.setvariable variable=vmr_sha]$vmr_sha" - displayName: Obtain the vmr sha from Version.Details.xml (Unix) - condition: ne(variables['Agent.OS'], 'Windows_NT') - workingDirectory: $(Agent.BuildDirectory)/repo - -- powershell: | - [xml]$xml = Get-Content -Path $(Agent.BuildDirectory)/repo/eng/Version.Details.xml - $vmr_sha = $xml.SelectSingleNode("//Source").Sha - Write-Output "##vso[task.setvariable variable=vmr_sha]$vmr_sha" - displayName: Obtain the vmr sha from Version.Details.xml (Windows) - condition: eq(variables['Agent.OS'], 'Windows_NT') - workingDirectory: $(Agent.BuildDirectory)/repo - -- script: | - git fetch --all - git checkout $(vmr_sha) - displayName: Checkout VMR at correct sha for repo flow - workingDirectory: ${{ parameters.vmrPath }} - - script: | git config --global user.name "dotnet-maestro[bot]" git config --global user.email "dotnet-maestro[bot]@users.noreply.github.com" diff --git a/eng/common/templates/variables/pool-providers.yml b/eng/common/templates/variables/pool-providers.yml index e0b19c14..18693ea1 100644 --- a/eng/common/templates/variables/pool-providers.yml +++ b/eng/common/templates/variables/pool-providers.yml @@ -23,7 +23,7 @@ # # pool: # name: $(DncEngInternalBuildPool) -# demands: ImageOverride -equals windows.vs2019.amd64 +# demands: ImageOverride -equals windows.vs2022.amd64 variables: - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - template: /eng/common/templates-official/variables/pool-providers.yml diff --git a/eng/common/templates/vmr-build-pr.yml b/eng/common/templates/vmr-build-pr.yml index ce3c29a6..2f3694fa 100644 --- a/eng/common/templates/vmr-build-pr.yml +++ b/eng/common/templates/vmr-build-pr.yml @@ -34,6 +34,7 @@ resources: type: github name: dotnet/dotnet endpoint: dotnet + ref: refs/heads/main # Set to whatever VMR branch the PR build should insert into stages: - template: /eng/pipelines/templates/stages/vmr-build.yml@vmr diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 1556562c..01296ee6 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -273,7 +273,7 @@ function GetDotNetInstallScript([string] $dotnetRoot) { Retry({ Write-Host "GET $uri" - Invoke-WebRequest $uri -OutFile $installScript + Invoke-WebRequest $uri -UseBasicParsing -OutFile $installScript }) } @@ -506,7 +506,7 @@ function InitializeXCopyMSBuild([string]$packageVersion, [bool]$install) { Write-Host "Downloading $packageName $packageVersion" $ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit Retry({ - Invoke-WebRequest "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/flat2/$packageName/$packageVersion/$packageName.$packageVersion.nupkg" -OutFile $packagePath + Invoke-WebRequest "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/flat2/$packageName/$packageVersion/$packageName.$packageVersion.nupkg" -UseBasicParsing -OutFile $packagePath }) if (!(Test-Path $packagePath)) { @@ -552,23 +552,30 @@ function LocateVisualStudio([object]$vsRequirements = $null){ Write-Host "Downloading vswhere $vswhereVersion" $ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit Retry({ - Invoke-WebRequest "https://netcorenativeassets.blob.core.windows.net/resource-packages/external/windows/vswhere/$vswhereVersion/vswhere.exe" -OutFile $vswhereExe + Invoke-WebRequest "https://netcorenativeassets.blob.core.windows.net/resource-packages/external/windows/vswhere/$vswhereVersion/vswhere.exe" -UseBasicParsing -OutFile $vswhereExe }) } - if (!$vsRequirements) { $vsRequirements = $GlobalJson.tools.vs } + if (!$vsRequirements) { + if (Get-Member -InputObject $GlobalJson.tools -Name 'vs' -ErrorAction SilentlyContinue) { + $vsRequirements = $GlobalJson.tools.vs + } else { + $vsRequirements = $null + } + } + $args = @('-latest', '-format', 'json', '-requires', 'Microsoft.Component.MSBuild', '-products', '*') if (!$excludePrereleaseVS) { $args += '-prerelease' } - if (Get-Member -InputObject $vsRequirements -Name 'version') { + if ($vsRequirements -and (Get-Member -InputObject $vsRequirements -Name 'version' -ErrorAction SilentlyContinue)) { $args += '-version' $args += $vsRequirements.version } - if (Get-Member -InputObject $vsRequirements -Name 'components') { + if ($vsRequirements -and (Get-Member -InputObject $vsRequirements -Name 'components' -ErrorAction SilentlyContinue)) { foreach ($component in $vsRequirements.components) { $args += '-requires' $args += $component @@ -581,6 +588,11 @@ function LocateVisualStudio([object]$vsRequirements = $null){ return $null } + if ($null -eq $vsInfo -or $vsInfo.Count -eq 0) { + throw "No instance of Visual Studio meeting the requirements specified was found. Requirements: $($args -join ' ')" + return $null + } + # use first matching instance return $vsInfo[0] } @@ -813,6 +825,11 @@ function MSBuild-Core() { $cmdArgs = "$($buildTool.Command) /m /nologo /clp:Summary /v:$verbosity /nr:$nodeReuse /p:ContinuousIntegrationBuild=$ci" + # Add -mt flag for MSBuild multithreaded mode if enabled via environment variable + if ($env:MSBUILD_MT_ENABLED -eq "1") { + $cmdArgs += ' -mt' + } + if ($warnAsError) { $cmdArgs += ' /warnaserror /p:TreatWarningsAsErrors=true' } diff --git a/eng/common/tools.sh b/eng/common/tools.sh index 6c121300..edc7b128 100755 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -522,7 +522,13 @@ function MSBuild-Core { } } - RunBuildTool "$_InitializeBuildToolCommand" /m /nologo /clp:Summary /v:$verbosity /nr:$node_reuse $warnaserror_switch /p:TreatWarningsAsErrors=$warn_as_error /p:ContinuousIntegrationBuild=$ci "$@" + # Add -mt flag for MSBuild multithreaded mode if enabled via environment variable + local mt_switch="" + if [[ "${MSBUILD_MT_ENABLED:-}" == "1" ]]; then + mt_switch="-mt" + fi + + RunBuildTool "$_InitializeBuildToolCommand" /m /nologo /clp:Summary /v:$verbosity /nr:$node_reuse $warnaserror_switch $mt_switch /p:TreatWarningsAsErrors=$warn_as_error /p:ContinuousIntegrationBuild=$ci "$@" } function GetDarc { diff --git a/eng/common/vmr-sync.ps1 b/eng/common/vmr-sync.ps1 index 97302f32..b37992d9 100644 --- a/eng/common/vmr-sync.ps1 +++ b/eng/common/vmr-sync.ps1 @@ -103,12 +103,20 @@ Set-StrictMode -Version Latest Highlight 'Installing .NET, preparing the tooling..' . .\eng\common\tools.ps1 $dotnetRoot = InitializeDotNetCli -install:$true +$env:DOTNET_ROOT = $dotnetRoot $darc = Get-Darc -$dotnet = "$dotnetRoot\dotnet.exe" Highlight "Starting the synchronization of VMR.." # Synchronize the VMR +$versionDetailsPath = Resolve-Path (Join-Path $PSScriptRoot '..\Version.Details.xml') | Select-Object -ExpandProperty Path +[xml]$versionDetails = Get-Content -Path $versionDetailsPath +$repoName = $versionDetails.SelectSingleNode('//Source').Mapping +if (-not $repoName) { + Fail "Failed to resolve repo mapping from $versionDetailsPath" + exit 1 +} + $darcArgs = ( "vmr", "forwardflow", "--tmp", $tmpDir, @@ -130,9 +138,27 @@ if ($LASTEXITCODE -eq 0) { Highlight "Synchronization succeeded" } else { - Fail "Synchronization of repo to VMR failed!" - Fail "'$vmrDir' is left in its last state (re-run of this script will reset it)." - Fail "Please inspect the logs which contain path to the failing patch file (use -debugOutput to get all the details)." - Fail "Once you make changes to the conflicting VMR patch, commit it locally and re-run this script." - exit 1 + Highlight "Failed to flow code into the local VMR. Falling back to resetting the VMR to match repo contents..." + git -C $vmrDir reset --hard + + $resetArgs = ( + "vmr", "reset", + "${repoName}:HEAD", + "--vmr", $vmrDir, + "--tmp", $tmpDir, + "--additional-remotes", "${repoName}:${repoRoot}" + ) + + & "$darc" $resetArgs + + if ($LASTEXITCODE -eq 0) { + Highlight "Successfully reset the VMR using 'darc vmr reset'" + } + else { + Fail "Synchronization of repo to VMR failed!" + Fail "'$vmrDir' is left in its last state (re-run of this script will reset it)." + Fail "Please inspect the logs which contain path to the failing patch file (use -debugOutput to get all the details)." + Fail "Once you make changes to the conflicting VMR patch, commit it locally and re-run this script." + exit 1 + } } diff --git a/eng/common/vmr-sync.sh b/eng/common/vmr-sync.sh index 44239e33..198caec5 100644 --- a/eng/common/vmr-sync.sh +++ b/eng/common/vmr-sync.sh @@ -186,6 +186,13 @@ fi # Synchronize the VMR +version_details_path=$(cd "$scriptroot/.."; pwd -P)/Version.Details.xml +repo_name=$(grep -m 1 '