From f0b0b7a66eb2b1d6546d26a6bd2cb5c5560802aa Mon Sep 17 00:00:00 2001 From: Louis Parkin Date: Fri, 21 Aug 2026 15:46:52 +0200 Subject: [PATCH 1/2] STAC-25584: restore the omnibus git and bazel caches in the DEB build Only the Go module cache survived the port from GitLab, re-implemented as the godeps-cache image (STAC-25429). The omnibus git cache and the bazel caches were dropped, so every DEB build recompiles from source: summing the omnibus Builder/NetFetcher timings in a green run, roughly 31 of the 49 minutes is compile/fetch, and bazelisk rebuilds openssl (140s) and zlib (30s) every time. That is paid twice per nightly across two architectures on self-hosted runners. OMNIBUS_GIT_CACHE_DIR was not set at all, so tasks/omnibus.py disabled the git cache entirely. Setting it cannot reach the S3 path that upstream assumes: use_remote_cache additionally requires CI_JOB_NAME_SLUG and S3_OMNIBUS_GIT_CACHE_BUCKET, and neither exists on GitHub, so the local cache is used and the KeyError documented in UPSTREAM_MERGE.md stays unreachable. The --install-directory /opt/stackstate-agent override is what keeps the cache path branded; without it the post-build purge would git -C against a nonexistent /opt/datadog-agent. Cache writes are deliberately bounded rather than a one-to-one port of the GitLab layers. The repository cache is capped at 10 GB with LRU eviction, and the omnibus git cache is the highest value per byte, so: - keys are content-addressed (omnibus definitions, release.json; bazel version and lockfile) with no run id, and save is gated on a non-exact restore, so an unchanged build reuses the entry instead of writing a fresh multi-GB copy; - saves are skipped on pull_request, so PRs read the cache without competing for the cap or paying the upload; - only bazelisk, bazel/install and the repository cache are persisted, not the whole output_user_root. XDG_CACHE_HOME now derives from BAZEL_CACHE_ROOT so the build and the cache steps cannot drift apart. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/build-deb.yml | 45 +++++++++++++++++++++++++++++++-- .gitignore | 3 +++ 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-deb.yml b/.github/workflows/build-deb.yml index 5ca7bd04e7d2..2efdad57ff46 100644 --- a/.github/workflows/build-deb.yml +++ b/.github/workflows/build-deb.yml @@ -72,6 +72,8 @@ jobs: password: ${{ secrets.REGISTRY_PASSWORD }} env: OMNIBUS_BASE_DIR: /omnibus + OMNIBUS_GIT_CACHE_DIR: ${{ github.workspace }}/.omnibus-git-cache + BAZEL_CACHE_ROOT: ${{ github.workspace }}/.cache steps: - name: Check out repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -79,6 +81,27 @@ jobs: persist-credentials: false fetch-depth: 0 + - name: Restore omnibus git cache (${{ matrix.arch }}) + id: omnibus-git-cache + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ env.OMNIBUS_GIT_CACHE_DIR }} + key: omnibus-git-${{ matrix.arch }}-${{ hashFiles('omnibus/**', 'release.json') }} + restore-keys: | + omnibus-git-${{ matrix.arch }}- + + - name: Restore bazel caches (${{ matrix.arch }}) + id: bazel-cache + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + ${{ env.BAZEL_CACHE_ROOT }}/bazelisk + ${{ env.BAZEL_CACHE_ROOT }}/bazel/install + ${{ env.BAZEL_CACHE_ROOT }}/bazel-repo + key: bazel-${{ matrix.arch }}-${{ hashFiles('.bazelversion', 'MODULE.bazel.lock', 'omnibus/config/software/datadog-agent-dependencies.rb') }} + restore-keys: | + bazel-${{ matrix.arch }}- + - name: Build DEB package with omnibus env: GITLAB_PACKAGE_REGISTRY_PYPI_SIMPLE_URL: ${{ vars.GITLAB_PACKAGE_REGISTRY_PYPI_SIMPLE_URL }} @@ -104,8 +127,9 @@ jobs: # DD 7.78 builds some dependencies through bazelisk from # omnibus/config/software/datadog-agent-dependencies.rb. bazelisk aborts # with "XDG_CACHE_HOME () must denote a directory in CI!" unless these - # point at real directories. - export XDG_CACHE_HOME="${GITHUB_WORKSPACE}/.cache" + # point at real directories. BAZEL_CACHE_ROOT is what the cache steps + # restore and save, so the two must stay in agreement. + export XDG_CACHE_HOME="${BAZEL_CACHE_ROOT}" export BAZELISK_HOME="${XDG_CACHE_HOME}/bazelisk" mkdir -p "${XDG_CACHE_HOME}/bazel" "${XDG_CACHE_HOME}/bazel-repo" "${BAZELISK_HOME}" # .bazelrc:75 `try-import %workspace%/user.bazelrc` is the upstream-blessed @@ -158,6 +182,23 @@ jobs: --skip-sign \ --install-directory /opt/stackstate-agent + - name: Save omnibus git cache (${{ matrix.arch }}) + if: github.event_name != 'pull_request' && steps.omnibus-git-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ env.OMNIBUS_GIT_CACHE_DIR }} + key: omnibus-git-${{ matrix.arch }}-${{ hashFiles('omnibus/**', 'release.json') }} + + - name: Save bazel caches (${{ matrix.arch }}) + if: github.event_name != 'pull_request' && steps.bazel-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + ${{ env.BAZEL_CACHE_ROOT }}/bazelisk + ${{ env.BAZEL_CACHE_ROOT }}/bazel/install + ${{ env.BAZEL_CACHE_ROOT }}/bazel-repo + key: bazel-${{ matrix.arch }}-${{ hashFiles('.bazelversion', 'MODULE.bazel.lock', 'omnibus/config/software/datadog-agent-dependencies.rb') }} + - name: Collect package outputs run: | set -eo pipefail diff --git a/.gitignore b/.gitignore index b283757f4060..f19f48b9b35e 100644 --- a/.gitignore +++ b/.gitignore @@ -54,6 +54,9 @@ omnibus/omnibus/ # User specific bazelrc file /user.bazelrc +# Local omnibus git cache (OMNIBUS_GIT_CACHE_DIR), persisted by CI between builds +/.omnibus-git-cache/ + # file generated by Cluster Agent image build Dockerfiles/cluster-agent/nosys-seccomp From 7a491ac0eb67de1a0451e7b31b6a76383e162dea Mon Sep 17 00:00:00 2001 From: Louis Parkin Date: Fri, 21 Aug 2026 16:33:19 +0200 Subject: [PATCH 2/2] STAC-25584: give the DEB build a git identity for the omnibus cache Setting OMNIBUS_GIT_CACHE_DIR makes omnibus's git cache reachable for the first time on GitHub Actions, and its incremental cache step commits each built software definition into the cache repository. The build container runs as root with no user.name/user.email, so that commit aborted the build with "unable to auto-detect email address". The identity matches the one the repo already uses for bot commits in tasks/libs/common/utils.py. Passing it through GIT_AUTHOR_*/GIT_COMMITTER_* rather than `git config --global` keeps it scoped to the job and avoids mutating the container's global git state. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/build-deb.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/build-deb.yml b/.github/workflows/build-deb.yml index 2efdad57ff46..04bb6007a6ca 100644 --- a/.github/workflows/build-deb.yml +++ b/.github/workflows/build-deb.yml @@ -74,6 +74,10 @@ jobs: OMNIBUS_BASE_DIR: /omnibus OMNIBUS_GIT_CACHE_DIR: ${{ github.workspace }}/.omnibus-git-cache BAZEL_CACHE_ROOT: ${{ github.workspace }}/.cache + GIT_AUTHOR_NAME: github-actions[bot] + GIT_AUTHOR_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_COMMITTER_NAME: github-actions[bot] + GIT_COMMITTER_EMAIL: github-actions[bot]@users.noreply.github.com steps: - name: Check out repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2