diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 1f97e733f58..9e5e3b9a8f2 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,15 +1,15 @@ - + https://github.com/dotnet/source-build-externals - 3dc05150cf234f76f6936dcb2853d31a0da1f60e + d64f0dd8dea2a4cd6e956f1f84137a3c58d70bd9 - - https://github.com/dotnet/source-build-reference-packages - 1bffa52c9f78c19618fa929eea4916b0c6d27403 - + + https://github.com/dotnet/source-build-assets + c91650e32f386a2ac2fa3ee0eb17541de52e89d4 + https://github.com/dotnet/command-line-api @@ -17,9 +17,9 @@ - + https://github.com/dotnet/arcade - b9abaa942c5224cf7196918cdf25a4db6bf99487 + 7945b1b660dff1927e4d71aca0a11e5d067ba4a5 diff --git a/eng/common/Get-GitHubAppToken.ps1 b/eng/common/Get-GitHubAppToken.ps1 new file mode 100644 index 00000000000..ded1a6595cc --- /dev/null +++ b/eng/common/Get-GitHubAppToken.ps1 @@ -0,0 +1,155 @@ +# Mints a short-lived GitHub App installation access token by signing a JWT +# with a private key stored in Azure Key Vault (RSA, RS256). The signed JWT is +# exchanged with the GitHub API for a token scoped to a single installation. +# +# Requirements: +# - A GitHub App whose private key has been uploaded into Key Vault as an RSA +# key (the PEM converted to a Key Vault *key*, NOT stored as a secret). +# - The caller (the federated Azure service connection used to run this script) +# must have the `Key Vault Crypto User` role (or at minimum the `Sign` +# action) on that key. +# - The App must be installed on the target organization/account +# (`InstallationOwner`) with the permissions/repositories it needs. +# +# Installation tokens (ghs_*) are exempt from the enterprise classic-PAT +# lifetime policy, which is why this replaces the long-lived PAT. + +[CmdletBinding()] +param( + # Name of the Key Vault that holds the GitHub App's RSA signing key. + [Parameter(Mandatory = $true)] + [string] $KeyVaultName, + + # Name of the RSA key inside the Key Vault (the App's private key). + [Parameter(Mandatory = $true)] + [string] $KeyName, + + # The GitHub App's Client ID (the value to put in the `iss` JWT claim). + [Parameter(Mandatory = $true)] + [string] $AppClientId, + + # Login of the organization or user account whose installation we should + # mint the token for (e.g. `dotnet`, `microsoft`). + [Parameter(Mandatory = $true)] + [string] $InstallationOwner, + + # Optional Azure DevOps pipeline variable name to set with the installation + # token (marked as a secret). When not specified, the token is written to + # stdout instead. + [Parameter(Mandatory = $false)] + [string] $OutputVariableName +) + +$ErrorActionPreference = 'Stop' +$PSNativeCommandUseErrorActionPreference = $true + +. $PSScriptRoot\pipeline-logging-functions.ps1 + +function ConvertTo-Base64Url([byte[]] $bytes) { + return [Convert]::ToBase64String($bytes).TrimEnd('=').Replace('+', '-').Replace('/', '_') +} + +# Build JWT header and payload. Use [ordered] hashtables so JSON +# serialization is deterministic. +$jwtHeader = [ordered]@{ + alg = 'RS256' + typ = 'JWT' +} +$now = [System.DateTimeOffset]::UtcNow +$jwtPayload = [ordered]@{ + iat = $now.AddMinutes(-1).ToUnixTimeSeconds() + exp = $now.AddMinutes(5).ToUnixTimeSeconds() + iss = $AppClientId +} + +$headerEncoded = ConvertTo-Base64Url ([System.Text.Encoding]::UTF8.GetBytes(($jwtHeader | ConvertTo-Json -Compress))) +$payloadEncoded = ConvertTo-Base64Url ([System.Text.Encoding]::UTF8.GetBytes(($jwtPayload | ConvertTo-Json -Compress))) +$signingInput = "$headerEncoded.$payloadEncoded" + +# Key Vault `sign` expects the *digest* (base64), not the raw bytes. +$sha256 = [System.Security.Cryptography.SHA256]::Create() +$digestBytes = $sha256.ComputeHash([System.Text.Encoding]::UTF8.GetBytes($signingInput)) +$digestBase64 = [Convert]::ToBase64String($digestBytes) + +Write-Host "Signing JWT with key '$KeyName' in vault '$KeyVaultName'..." +try { + $signatureUrl = az keyvault key sign ` + --vault-name $KeyVaultName ` + --name $KeyName ` + --algorithm RS256 ` + --digest $digestBase64 ` + --query value ` + --output tsv ` + --only-show-errors +} +catch { + Write-PipelineTelemetryError -Category 'Build' -Message "Failed to sign the JWT via Key Vault (key '$KeyName', vault '$KeyVaultName'): $_. Verify the service connection identity has the 'Key Vault Crypto User' role (Sign action) on the key." + exit 1 +} +if ($LASTEXITCODE -ne 0 -or [string]::IsNullOrWhiteSpace($signatureUrl)) { + Write-PipelineTelemetryError -Category 'Build' -Message "'az keyvault key sign' exited with code $LASTEXITCODE for key '$KeyName' in vault '$KeyVaultName'. Verify the service connection identity has the 'Key Vault Crypto User' role (Sign action) on the key." + exit 1 +} +$jwt = "$signingInput.$($signatureUrl.Trim())" + +$headers = @{ + Authorization = "Bearer $jwt" + 'X-GitHub-Api-Version' = '2022-11-28' + Accept = 'application/vnd.github+json' + 'User-Agent' = 'dotnet-arcade-onelocbuild' +} + +Write-Host "Looking up installation for '$InstallationOwner'..." +try { + $installations = @() + $page = 1 + do { + # Assign the response before wrapping it in @(). PowerShell otherwise + # preserves a top-level JSON array as one nested pipeline object. + $pageResponse = Invoke-RestMethod ` + -Uri "https://api.github.com/app/installations?per_page=100&page=$page" ` + -Headers $headers ` + -Method Get + $pageInstallations = @($pageResponse) + $installations += $pageInstallations + $page++ + } while ($pageInstallations.Count -eq 100) +} +catch { + Write-PipelineTelemetryError -Category 'Build' -Message "Failed to list GitHub App installations: $_. The signed JWT may be invalid or the App's Client ID ('$AppClientId') may be incorrect." + exit 1 +} +$matchingInstallations = @($installations | Where-Object { $_.account.login -ieq $InstallationOwner }) +if ($matchingInstallations.Count -eq 0) { + $found = ($installations | ForEach-Object { $_.account.login }) -join ', ' + Write-PipelineTelemetryError -Category 'Build' -Message "No installation found for '$InstallationOwner'. App is installed on: $found" + exit 1 +} +if ($matchingInstallations.Count -ne 1) { + $matchingIds = ($matchingInstallations | ForEach-Object { $_.id }) -join ', ' + Write-PipelineTelemetryError -Category 'Build' -Message "Found multiple installations for '$InstallationOwner': $matchingIds" + exit 1 +} +$installation = $matchingInstallations[0] +Write-Host "Using installation $($installation.id) for '$($installation.account.login)'." + +try { + $tokenResponse = Invoke-RestMethod ` + -Uri "https://api.github.com/app/installations/$($installation.id)/access_tokens" ` + -Headers $headers ` + -Method Post ` + -ContentType 'application/json' +} +catch { + Write-PipelineTelemetryError -Category 'Build' -Message "Failed to mint an installation access token for '$InstallationOwner' (installation $($installation.id)): $_" + exit 1 +} + +Write-Host "Got installation token for '$InstallationOwner' (expires $($tokenResponse.expires_at))." +if ($OutputVariableName) { + Write-Host "Setting pipeline variable '$OutputVariableName'." + Write-Host "##vso[task.setvariable variable=$OutputVariableName;issecret=true]$($tokenResponse.token)" +} +else { + Write-Host $tokenResponse.token -ForegroundColor Green +} diff --git a/eng/common/core-templates/steps/get-github-app-token.yml b/eng/common/core-templates/steps/get-github-app-token.yml new file mode 100644 index 00000000000..6d42a48d3c3 --- /dev/null +++ b/eng/common/core-templates/steps/get-github-app-token.yml @@ -0,0 +1,79 @@ +# Mints a short-lived GitHub App installation access token by signing a JWT +# with a private key stored in Azure Key Vault (RSA, RS256). The JWT is +# exchanged with the GitHub API for a token scoped to a single installation. +# +# Requirements (per GitHub App you want to authenticate as): +# - A GitHub App with its private key uploaded into Key Vault as an RSA key +# (PEM converted to a key, NOT stored as a secret). +# - The Azure service connection passed via `azureSubscription` must be +# granted the `Key Vault Crypto User` role (or at minimum `Sign` action) +# on that key. +# - The App must be installed on the target organization/account +# (`installationOwner`) with the permissions/repositories you need. +# +# Output: a secret pipeline variable named ${{ parameters.outputVariableName }} +# containing the installation access token. Token lifetime is ~1 hour and is +# automatically scrubbed from logs. Installation tokens are exempt from the +# enterprise classic-PAT lifetime policy. + +parameters: +# Azure DevOps service connection (federated) that can call +# `az keyvault key sign` on the App's signing key. +- name: azureSubscription + type: string + +# Name of the Key Vault that holds the GitHub App's RSA signing key. +- name: keyVaultName + type: string + +# Name of the RSA key inside the Key Vault (the App's private key). +- name: keyName + type: string + +# The GitHub App's Client ID (the value to put in the `iss` JWT claim). +# Prefer this over the numeric App ID; GitHub accepts either, but Client ID +# is the documented form going forward. +- name: appClientId + type: string + +# Login of the organization or user account whose installation we should +# mint the token for (e.g. `dotnet`, `microsoft`). +- name: installationOwner + type: string + +# Name of the pipeline variable that will receive the installation token. +- name: outputVariableName + type: string + +- name: is1ESPipeline + type: boolean + +- name: stepName + type: string + default: getGitHubAppInstallationToken + +- name: condition + type: string + default: '' + +- name: displayName + type: string + default: Get GitHub App installation token + +steps: +- task: AzureCLI@2 + displayName: ${{ parameters.displayName }} + name: ${{ parameters.stepName }} + ${{ if ne(parameters.condition, '') }}: + condition: ${{ parameters.condition }} + inputs: + azureSubscription: ${{ parameters.azureSubscription }} + scriptType: pscore + scriptLocation: inlineScript + inlineScript: | + & "$(System.DefaultWorkingDirectory)/eng/common/Get-GitHubAppToken.ps1" ` + -KeyVaultName '${{ parameters.keyVaultName }}' ` + -KeyName '${{ parameters.keyName }}' ` + -AppClientId '${{ parameters.appClientId }}' ` + -InstallationOwner '${{ parameters.installationOwner }}' ` + -OutputVariableName '${{ parameters.outputVariableName }}' diff --git a/eng/common/templates-official/job/publish-build-assets.yml b/eng/common/templates-official/job/publish-build-assets.yml index 67a0de433a3..4842d4c723e 100644 --- a/eng/common/templates-official/job/publish-build-assets.yml +++ b/eng/common/templates-official/job/publish-build-assets.yml @@ -48,8 +48,6 @@ jobs: variables: - template: /eng/common/templates-official/variables/pool-providers.yml - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - group: Publish-Build-Assets - - group: AzureDevOps-Artifact-Feeds-Pats - name: runCodesignValidationInjection value: false - ${{ if eq(parameters.publishAssetsImmediately, 'true') }}: diff --git a/eng/common/templates-official/jobs/codeql-build.yml b/eng/common/templates-official/jobs/codeql-build.yml index f6476912a86..b62247fe703 100644 --- a/eng/common/templates-official/jobs/codeql-build.yml +++ b/eng/common/templates-official/jobs/codeql-build.yml @@ -17,7 +17,6 @@ jobs: enableTelemetry: true variables: - - group: Publish-Build-Assets # The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in # sync with the packages.config file. - name: DefaultGuardianVersion diff --git a/eng/common/templates-official/post-build/common-variables.yml b/eng/common/templates-official/post-build/common-variables.yml index 173914f2364..c4b8507bb45 100644 --- a/eng/common/templates-official/post-build/common-variables.yml +++ b/eng/common/templates-official/post-build/common-variables.yml @@ -1,6 +1,4 @@ variables: - - group: Publish-Build-Assets - # Whether the build is internal or not - name: IsInternalBuild value: ${{ and(ne(variables['System.TeamProject'], 'public'), contains(variables['Build.SourceBranch'], 'internal')) }} diff --git a/eng/common/templates-official/steps/get-github-app-token.yml b/eng/common/templates-official/steps/get-github-app-token.yml new file mode 100644 index 00000000000..c89f3641a4d --- /dev/null +++ b/eng/common/templates-official/steps/get-github-app-token.yml @@ -0,0 +1,7 @@ +steps: +- template: /eng/common/core-templates/steps/get-github-app-token.yml + parameters: + is1ESPipeline: true + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates/job/execute-sdl.yml b/eng/common/templates/job/execute-sdl.yml index 28ccaaa4820..89f1b31bc55 100644 --- a/eng/common/templates/job/execute-sdl.yml +++ b/eng/common/templates/job/execute-sdl.yml @@ -36,7 +36,6 @@ jobs: displayName: Run SDL tool condition: and(succeededOrFailed(), eq( ${{ parameters.enable }}, 'true')) variables: - - group: DotNet-VSTS-Bot - name: AzDOProjectName value: ${{ parameters.AzDOProjectName }} - name: AzDOPipelineId diff --git a/eng/common/templates/job/publish-build-assets.yml b/eng/common/templates/job/publish-build-assets.yml index 9234ccab1a4..42c154b1304 100644 --- a/eng/common/templates/job/publish-build-assets.yml +++ b/eng/common/templates/job/publish-build-assets.yml @@ -48,8 +48,6 @@ jobs: variables: - template: /eng/common/templates/variables/pool-providers.yml - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - group: Publish-Build-Assets - - group: AzureDevOps-Artifact-Feeds-Pats - name: runCodesignValidationInjection value: false - ${{ if eq(parameters.publishAssetsImmediately, 'true') }}: diff --git a/eng/common/templates/jobs/codeql-build.yml b/eng/common/templates/jobs/codeql-build.yml index e8b43e3b4cb..a89c1f02682 100644 --- a/eng/common/templates/jobs/codeql-build.yml +++ b/eng/common/templates/jobs/codeql-build.yml @@ -17,7 +17,6 @@ jobs: enableTelemetry: true variables: - - group: Publish-Build-Assets # The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in # sync with the packages.config file. - name: DefaultGuardianVersion diff --git a/eng/common/templates/post-build/common-variables.yml b/eng/common/templates/post-build/common-variables.yml index 173914f2364..c4b8507bb45 100644 --- a/eng/common/templates/post-build/common-variables.yml +++ b/eng/common/templates/post-build/common-variables.yml @@ -1,6 +1,4 @@ variables: - - group: Publish-Build-Assets - # Whether the build is internal or not - name: IsInternalBuild value: ${{ and(ne(variables['System.TeamProject'], 'public'), contains(variables['Build.SourceBranch'], 'internal')) }} diff --git a/eng/common/templates/steps/get-github-app-token.yml b/eng/common/templates/steps/get-github-app-token.yml new file mode 100644 index 00000000000..79e182c6416 --- /dev/null +++ b/eng/common/templates/steps/get-github-app-token.yml @@ -0,0 +1,7 @@ +steps: +- template: /eng/common/core-templates/steps/get-github-app-token.yml + parameters: + is1ESPipeline: false + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/global.json b/global.json index 2b87ef8dcc1..d73d4ce63f9 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "8.0.126" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.26404.4" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.26411.4" } }