From d3a262876646fb9893576ae010c611f2a67076bd Mon Sep 17 00:00:00 2001 From: dotnet-docker-bot <60522487+dotnet-docker-bot@users.noreply.github.com> Date: Fri, 19 Jun 2026 09:24:42 -0700 Subject: [PATCH] Update common Docker engineering infrastructure with latest --- eng/docker-tools/CHANGELOG.md | 30 +++++++++++++ eng/docker-tools/DEV-GUIDE.md | 25 ++++++++++- eng/docker-tools/templates/jobs/publish.yml | 45 +++++++++++++------ .../stages/dotnet/build-test-publish-repo.yml | 5 +++ .../templates/stages/dotnet/publish.yml | 4 ++ eng/docker-tools/templates/stages/publish.yml | 5 +++ .../templates/variables/docker-images.yml | 2 +- 7 files changed, 100 insertions(+), 16 deletions(-) diff --git a/eng/docker-tools/CHANGELOG.md b/eng/docker-tools/CHANGELOG.md index 99dbceb697..0936f6feb6 100644 --- a/eng/docker-tools/CHANGELOG.md +++ b/eng/docker-tools/CHANGELOG.md @@ -4,6 +4,36 @@ All breaking changes and new features in `eng/docker-tools` will be documented i --- +## 2026-06-11: Configurable per-registry referrer-lookup rate limit + +- Issue: [#2141](https://github.com/dotnet/docker-tools/issues/2141) + +ImageBuilder now rate limits only OCI referrer lookups (the ACR +`/v2//referrers/` endpoint), instead of throttling and bounding the concurrency of all +ACR operations. All other ACR operations rely on the standard HTTP retry behavior, which retries on +HTTP 429 and respects `Retry-After` headers. This significantly speeds up image copying during +publishing. + +The referrer-lookup limit defaults to 250 requests per 60-second window and can be configured per +registry via an optional `referrerRequestsPerMinute` field on each `RegistryAuthentication` entry in +`publishConfig`: + +```yaml +RegistryAuthentication: + - server: $(acr.server) + resourceGroup: $(acr.resourceGroup) + subscription: $(acr.subscription) + referrerRequestsPerMinute: 1000 + serviceConnection: + name: $(publish.serviceConnectionName) + ... +``` + +This field is optional; when omitted, the 250/60s default is used. No action is required for +existing configurations. + +--- + ## 2026-04-02: Extra Docker build options can be passed through ImageBuilder - Pull request: [#2063](https://github.com/dotnet/docker-tools/pull/2063) diff --git a/eng/docker-tools/DEV-GUIDE.md b/eng/docker-tools/DEV-GUIDE.md index 28ddc471bc..d47fb3ebda 100644 --- a/eng/docker-tools/DEV-GUIDE.md +++ b/eng/docker-tools/DEV-GUIDE.md @@ -143,6 +143,7 @@ Build Stage ▼ Post_Build Stage ├── Merge image info files + ├── Create multi-arch manifests └── Consolidate SBOMs │ ▼ @@ -191,13 +192,35 @@ Common patterns: - `"publish"` - Publish only (when re-running a failed publish from a previous build) - `"build,test,sign,publish"` - Full pipeline -**Note:** The `Post_Build` stage is implicitly included whenever `build` is in the stages list. You don't need to specify it separately—it automatically runs after Build to merge image info files and consolidate SBOMs. +**Note:** The `Post_Build` stage is implicitly included whenever `build` is in the stages list. You don't need to specify it separately—it automatically runs after Build to merge image info files, create and validate multi-arch manifests, and consolidate SBOMs. The stages variable is useful for: - Re-running just the publish stage after fixing a transient failure - Skipping tests during initial development - Running isolated stages for debugging +### Decoupling build OS from the base image OS + +By default, a platform's `osVersion` represents the base image OS version, but also determines what +build leg an image is built in. This can cause problems when build image and base image don't match +up. For example, building a .NET app on Windows Server 2025 and copying the artifacts into a +Windows Server 2019 base image won't work, because the build matrix generation will attempt to +build the image on the Server 2019 build leg (which can't run Server 2025 images). + +To fix this, set the optional `buildOsVersion` field in order to override only the OS used in the +build matrix generation. Here is an example of building a Windows Server 2019 image using Windows +Server 2025: + +```jsonc +{ + // ... + "os": "windows", + "osVersion": "windowsservercore-ltsc2019", + "buildOsVersion": "windowsservercore-ltsc2025" + // ... +} +``` + ### Image Info Files: The Build's Memory Image info files (defined by [`ImageArtifactDetails`](https://github.com/dotnet/docker-tools/blob/main/src/ImageBuilder/Models/Image/ImageArtifactDetails.cs)) are the mechanism that tracks what was built: diff --git a/eng/docker-tools/templates/jobs/publish.yml b/eng/docker-tools/templates/jobs/publish.yml index 5839be2d76..3adc410799 100644 --- a/eng/docker-tools/templates/jobs/publish.yml +++ b/eng/docker-tools/templates/jobs/publish.yml @@ -15,6 +15,11 @@ parameters: # Service connections not in publishConfig.RegistryAuthentication that need OIDC # token access during publish (e.g., kusto, marStatus). Shape: [{ name: string }] additionalServiceConnections: [] + # When true, the merged image info is published as an OCI artifact to the publish + # registry instead of being committed to the GitHub versions repo. Note that the + # `publishImageInfo` variable must also be set to 'true' for either form of image + # info publishing to run. + useOciArtifactForImageInfo: false jobs: - job: Publish @@ -217,20 +222,32 @@ jobs: acr: ${{ parameters.publishConfig.PublishRegistry }} dataFile: $(artifactsPath)/eol-annotation-data/eol-annotation-data.json - - script: > - $(runImageBuilderCmd) publishImageInfo - '$(imageInfoContainerDir)/full-image-info-new.json' - '$(gitHubVersionsRepoInfo.userName)' - '$(gitHubVersionsRepoInfo.email)' - $(gitHubVersionsRepoInfo.authArgs) - --git-owner '$(gitHubVersionsRepoInfo.org)' - --git-repo '$(gitHubVersionsRepoInfo.repo)' - --git-branch '$(gitHubVersionsRepoInfo.branch)' - --git-path '$(gitHubImageInfoVersionsPath)' - $(dryRunArg) - $(imageBuilder.commonCmdArgs) - condition: and(succeeded(), eq(variables['publishImageInfo'], 'true')) - displayName: Publish Image Info + - ${{ if eq(parameters.useOciArtifactForImageInfo, true) }}: + - script: > + $(runAuthedImageBuilderCmd) publishImageInfoArtifact + '$(imageInfoContainerDir)/full-image-info-new.json' + $(dryRunArg) + $(imageBuilder.commonCmdArgs) + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + SYSTEM_OIDCREQUESTURI: $(System.OidcRequestUri) + condition: and(succeeded(), eq(variables['publishImageInfo'], 'true')) + displayName: Publish Image Info + - ${{ else }}: + - script: > + $(runImageBuilderCmd) publishImageInfo + '$(imageInfoContainerDir)/full-image-info-new.json' + '$(gitHubVersionsRepoInfo.userName)' + '$(gitHubVersionsRepoInfo.email)' + $(gitHubVersionsRepoInfo.authArgs) + --git-owner '$(gitHubVersionsRepoInfo.org)' + --git-repo '$(gitHubVersionsRepoInfo.repo)' + --git-branch '$(gitHubVersionsRepoInfo.branch)' + --git-path '$(gitHubImageInfoVersionsPath)' + $(dryRunArg) + $(imageBuilder.commonCmdArgs) + condition: and(succeeded(), eq(variables['publishImageInfo'], 'true')) + displayName: Publish Image Info # Task displayNames names are hardcoded to reference the task prefix used by 1ES official # pipelines in eng/docker-tools/templates/1es-official.yml. diff --git a/eng/docker-tools/templates/stages/dotnet/build-test-publish-repo.yml b/eng/docker-tools/templates/stages/dotnet/build-test-publish-repo.yml index c590a52874..eaf153c2c1 100644 --- a/eng/docker-tools/templates/stages/dotnet/build-test-publish-repo.yml +++ b/eng/docker-tools/templates/stages/dotnet/build-test-publish-repo.yml @@ -32,6 +32,10 @@ parameters: # Publish parameters customPublishInitSteps: [] + # When true, the merged image info is published as an OCI artifact to the publish + # registry instead of being committed to the GitHub versions repo. + useOciArtifactForImageInfo: false + # Additional service connections not in publishConfig.RegistryAuthentication # that need OIDC token access (e.g., kusto, marStatus). Shape: [{ name: string }] additionalServiceConnections: [] @@ -82,3 +86,4 @@ stages: additionalServiceConnections: ${{ parameters.additionalServiceConnections }} sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} versionsRepoRef: ${{ parameters.versionsRepoRef }} + useOciArtifactForImageInfo: ${{ parameters.useOciArtifactForImageInfo }} diff --git a/eng/docker-tools/templates/stages/dotnet/publish.yml b/eng/docker-tools/templates/stages/dotnet/publish.yml index 5cac237b95..7bf845ee68 100644 --- a/eng/docker-tools/templates/stages/dotnet/publish.yml +++ b/eng/docker-tools/templates/stages/dotnet/publish.yml @@ -16,6 +16,9 @@ parameters: # Service connections not in publishConfig.RegistryAuthentication that need OIDC # token access during publish (e.g., kusto, marStatus). Shape: [{ name: string }] additionalServiceConnections: [] + # When true, the merged image info is published as an OCI artifact to the publish + # registry instead of being committed to the GitHub versions repo. + useOciArtifactForImageInfo: false stages: - template: /eng/docker-tools/templates/stages/publish.yml@self @@ -30,6 +33,7 @@ stages: sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} versionsRepoRef: ${{ parameters.versionsRepoRef }} overrideImageInfoCommit: ${{ parameters.overrideImageInfoCommit }} + useOciArtifactForImageInfo: ${{ parameters.useOciArtifactForImageInfo }} customPublishInitSteps: - pwsh: | diff --git a/eng/docker-tools/templates/stages/publish.yml b/eng/docker-tools/templates/stages/publish.yml index 2c17283eca..e6575e4f16 100644 --- a/eng/docker-tools/templates/stages/publish.yml +++ b/eng/docker-tools/templates/stages/publish.yml @@ -29,6 +29,10 @@ parameters: # token access during publish (e.g., kusto, marStatus). Shape: [{ name: string }] additionalServiceConnections: [] + # When true, the merged image info is published as an OCI artifact to the publish + # registry instead of being committed to the GitHub versions repo. + useOciArtifactForImageInfo: false + ################################################################################ # Publish Images ################################################################################ @@ -86,3 +90,4 @@ stages: versionsRepoPath: ${{ parameters.versionsRepoPath }} overrideImageInfoCommit: ${{ parameters.overrideImageInfoCommit }} additionalServiceConnections: ${{ parameters.additionalServiceConnections }} + useOciArtifactForImageInfo: ${{ parameters.useOciArtifactForImageInfo }} diff --git a/eng/docker-tools/templates/variables/docker-images.yml b/eng/docker-tools/templates/variables/docker-images.yml index ac60867f30..4a68f55ad4 100644 --- a/eng/docker-tools/templates/variables/docker-images.yml +++ b/eng/docker-tools/templates/variables/docker-images.yml @@ -1,5 +1,5 @@ variables: - imageNames.imageBuilderName: mcr.microsoft.com/dotnet-buildtools/image-builder:2996325 + imageNames.imageBuilderName: mcr.microsoft.com/dotnet-buildtools/image-builder:3002630 imageNames.imageBuilder: $(imageNames.imageBuilderName) imageNames.imageBuilder.withrepo: imagebuilder-withrepo:$(Build.BuildId)-$(System.JobId) imageNames.testRunner: mcr.microsoft.com/dotnet-buildtools/prereqs:azurelinux3.0-docker-testrunner