From 1361df985fd085d5271b79ce53cffa4f23a22680 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Thu, 2 Jul 2026 15:14:28 +0200 Subject: [PATCH 1/9] SRE-3874 ci: optimized docker images size Ensure optimized docker images size in CI Signed-off-by: Tomasz Gromadzki Cancel-prev-build: false Skip-python-bandit: true Skip-unit-test: true Skip-unit-test-memcheck: true Skip-test-el-9-rpms: false Skip-test-leap-15-rpms: true Skip-func-hw-test: true Skip-build-el8-gcc: true Skip-build-leap15-gcc: true Skip-unit-tests:true Skip-NLT: true Skip-fault-injection-test: true Skip-func-vm-all: true Signed-off-by: Tomasz Gromadzki --- Jenkinsfile | 6 ++-- utils/docker/Dockerfile.el.9 | 56 ++++++++++++++++++++++++------------ utils/scripts/install-el9.sh | 16 +++++++---- 3 files changed, 53 insertions(+), 25 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index dd086e88a0e..fbb9ce30aa4 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -887,7 +887,7 @@ pipeline { agent { dockerfile { filename 'utils/docker/Dockerfile.el.9' - label 'docker_runner' + label 'fox-119_docker_1' additionalBuildArgs dockerBuildArgs(repo_type: 'stable', deps_build: false, parallel_build: true) + @@ -896,7 +896,9 @@ pipeline { ' --build-arg DAOS_KEEP_SRC=yes ' + ' --build-arg REPOS="' + prRepos() + '"' + ' --build-arg POINT_RELEASE=.7' + - " --build-arg PYTHON_VERSION=${env.PYTHON_VERSION}" + " --build-arg PYTHON_VERSION=${env.PYTHON_VERSION}" + + ' --target build-only' + + ' --build-arg INSTALL_ONLY_ESSENTIAL=true' } } steps { diff --git a/utils/docker/Dockerfile.el.9 b/utils/docker/Dockerfile.el.9 index ac1a99798fc..27fa6c27f8a 100644 --- a/utils/docker/Dockerfile.el.9 +++ b/utils/docker/Dockerfile.el.9 @@ -32,21 +32,24 @@ ENV NO_PROXY=${DAOS_NO_PROXY} RUN echo "no_proxy=${DAOS_NO_PROXY}" >> /etc/environment && \ echo "NO_PROXY=${DAOS_NO_PROXY}" >> /etc/environment -# script to install OS updates basic tools and daos dependencies -COPY ./utils/scripts/install-el9.sh /tmp/install.sh -# script to setup local repo if available +# Script to setup local repo if available COPY ./utils/scripts/helpers/repo-helper-el9.sh /tmp/repo-helper.sh -RUN chmod +x /tmp/repo-helper.sh /tmp/install.sh && \ - /tmp/repo-helper.sh && \ +# Script to install basic tools and daos dependencies +COPY ./utils/scripts/install-el9.sh /tmp/install.sh + +RUN chmod +x /tmp/repo-helper.sh && \ + /tmp/repo-helper.sh && \ rm -f /tmp/repo-helper.sh -FROM basic +FROM basic AS build-only +ARG INSTALL_ONLY_ESSENTIAL=false + # Install OS updates and package. Include basic tools and daos dependencies -RUN dnf upgrade && \ - /tmp/install.sh && \ - dnf clean all && \ - rm -f /tmp/install.sh +RUN dnf upgrade && \ + chmod +x /tmp/install.sh && \ + INSTALL_ONLY_ESSENTIAL="$INSTALL_ONLY_ESSENTIAL" /tmp/install.sh && \ + dnf clean all # Add DAOS users ARG UID=1000 @@ -54,14 +57,12 @@ COPY ./utils/scripts/helpers/daos-server-user-setup.sh \ /tmp/daos-server-user-setup.sh RUN set -e; \ chmod +x /tmp/daos-server-user-setup.sh && \ - /tmp/daos-server-user-setup.sh -RUN useradd --no-log-init --user-group --create-home --shell /bin/bash daos_agent -RUN echo "daos_agent:daos_agent" | chpasswd + /tmp/daos-server-user-setup.sh && \ + rm -f /tmp/daos-server-user-setup.sh # Create directory for DAOS backend storage -RUN mkdir -p /opt/daos /mnt/daos /var/run/daos_server /var/run/daos_agent /home/daos/pre /home/daos/daos && \ - chown -R daos_server.daos_server /opt/daos /mnt/daos /var/run/daos_server /home/daos && \ - chown daos_agent.daos_agent /var/run/daos_agent +RUN mkdir -p /opt/daos /home/daos/pre && \ + chown -R daos_server.daos_server /opt/daos /home/daos USER daos_server:daos_server @@ -72,10 +73,10 @@ ENV PATH=/home/daos/venv/bin:$PATH ENV VIRTUAL_ENV=/home/daos/venv/ # Install latest versions of python tools. -COPY requirements-build.txt requirements-utest.txt ./ +COPY requirements-build.txt ./ RUN . /home/daos/venv/bin/activate && \ pip --no-cache-dir install --upgrade pip && \ - pip --no-cache-dir install -r requirements-build.txt -r requirements-utest.txt + pip --no-cache-dir install -r requirements-build.txt WORKDIR /home/daos/pre RUN mkdir -p /home/daos/pre/site_scons/prereq_tools /home/daos/pre/site_scons/components @@ -87,6 +88,25 @@ COPY --chown=daos_server:daos_server utils/build.config utils/ COPY --chown=daos_server:daos_server utils/scripts/copy_files.sh utils/scripts/copy_files.sh COPY --chown=daos_server:daos_server utils/scripts/create_spdk_pkgconfig.sh utils/scripts/create_spdk_pkgconfig.sh +FROM build-only AS final +USER root:root +RUN /tmp/install.sh && \ + dnf clean all && \ + rm -f /tmp/install.sh + +# Add DAOS users +RUN useradd --no-log-init --user-group --create-home --shell /bin/bash daos_agent +RUN echo "daos_agent:daos_agent" | chpasswd + +# Create directory for DAOS backend storage +RUN mkdir -p /mnt/daos /var/run/daos_server /var/run/daos_agent /home/daos/daos && \ + chown -R daos_server.daos_server /mnt/daos /var/run/daos_server /home/daos && \ + chown daos_agent.daos_agent /var/run/daos_agent + +USER daos_server:daos_server +COPY requirements-utest.txt ./ +RUN pip --no-cache-dir install -r requirements-utest.txt + # Control what to build. By default Dockerfiles build everything to allow for # ease-of-use for users, however in CI everything is turned off and then # selectively enabled. Turning off any step disables all future steps. diff --git a/utils/scripts/install-el9.sh b/utils/scripts/install-el9.sh index 8f32803071c..d67a99149bd 100755 --- a/utils/scripts/install-el9.sh +++ b/utils/scripts/install-el9.sh @@ -20,8 +20,6 @@ dnf --nodocs install ${dnf_install_args} \ boost-python3-devel \ bzip2 \ capstone-devel \ - clang \ - clang-tools-extra \ cmake \ createrepo \ CUnit-devel \ @@ -37,13 +35,10 @@ dnf --nodocs install ${dnf_install_args} \ git \ glibc-langpack-en \ golang \ - gperftools-devel \ - graphviz \ help2man \ hdf5-devel \ hwloc-devel \ ipmctl \ - java-1.8.0-openjdk \ json-c-devel \ libaio-devel \ libasan \ @@ -86,6 +81,17 @@ dnf --nodocs install ${dnf_install_args} \ ncurses-devel \ yasm +if [[ "${INSTALL_ONLY_ESSENTIAL:-}" != "true" ]]; then + # Optional packages for full-featured images; can be skipped in essential-only mode. + # shellcheck disable=SC2086 + dnf --nodocs install ${dnf_install_args} \ + clang \ + clang-tools-extra \ + gperftools-devel \ + graphviz \ + java-1.8.0-openjdk +fi + if [[ -z "${NO_OPENMPI_DEVEL+set}" ]]; then # shellcheck disable=SC2086 dnf --nodocs install ${dnf_install_args} \ From 518d709ae15b68e5fe300d11d71468f5e4290207 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Thu, 2 Jul 2026 17:10:25 +0200 Subject: [PATCH 2/9] First test on regular docker node Signed-off-by: Tomasz Gromadzki Priority: 2 Cancel-prev-build: false Skip-python-bandit: true Skip-unit-test: true Skip-unit-test-memcheck: true Skip-test-el-9-rpms: false Skip-test-leap-15-rpms: true Skip-func-hw-test: true Skip-build-el8-gcc: true Skip-build-leap15-gcc: true Skip-unit-tests:true Skip-NLT: true Skip-fault-injection-test: true Skip-func-vm-all: true --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index fbb9ce30aa4..bfd6d94541c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -887,7 +887,7 @@ pipeline { agent { dockerfile { filename 'utils/docker/Dockerfile.el.9' - label 'fox-119_docker_1' + label 'docker_runner' additionalBuildArgs dockerBuildArgs(repo_type: 'stable', deps_build: false, parallel_build: true) + From 61565fe0797630991e7b5b70d627024337a92b48 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Fri, 3 Jul 2026 14:51:26 +0200 Subject: [PATCH 3/9] Final solution for EL 9 Signed-off-by: Tomasz Gromadzki Priority: 2 Cancel-prev-build: false Skip-python-bandit: true Skip-unit-test: true Skip-unit-test-memcheck: true Skip-test-el-9-rpms: false Skip-test-leap-15-rpms: true Skip-func-hw-test: true Skip-build-el8-gcc: true Skip-build-leap15-gcc: false Skip-unit-tests:true Skip-NLT: true Skip-fault-injection-test: true Skip-func-vm: true --- Jenkinsfile | 4 +- ci/rpm/build_daos_rpms_in_docker.sh | 136 +++++++++++++++++++++++++++ ci/rpm/build_deps.sh | 1 - utils/docker/Dockerfile.el.9 | 137 +++++++++++++++------------- utils/scripts/install-el9.sh | 2 +- 5 files changed, 210 insertions(+), 70 deletions(-) create mode 100755 ci/rpm/build_daos_rpms_in_docker.sh diff --git a/Jenkinsfile b/Jenkinsfile index bfd6d94541c..444455b9867 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -897,9 +897,7 @@ pipeline { ' --build-arg REPOS="' + prRepos() + '"' + ' --build-arg POINT_RELEASE=.7' + " --build-arg PYTHON_VERSION=${env.PYTHON_VERSION}" + - ' --target build-only' + - ' --build-arg INSTALL_ONLY_ESSENTIAL=true' - } + ' --target build-ci' } steps { script { diff --git a/ci/rpm/build_daos_rpms_in_docker.sh b/ci/rpm/build_daos_rpms_in_docker.sh new file mode 100755 index 00000000000..2eefcc21318 --- /dev/null +++ b/ci/rpm/build_daos_rpms_in_docker.sh @@ -0,0 +1,136 @@ +#!/bin/bash +set -euo pipefail + +# Build DAOS RPMs inside a disposable container and copy resulting RPMs back +# to the host before container teardown. +# +# Usage: +# ./ci/rpm/build_daos_rpms_in_docker.sh +# +# Arguments: +# Image flavor tag suffix (el9, el8, leap15) +# +# Requirements: +# - docker command must be available on host +# - host must contain a built image for the specified target, e.g. daos/el9:build-ci +# - host must contain /home/daos directory with write permissions for the current user +# +# Environment variables: +# JOBS SCons parallelism for main build (default: 88) +# KEEP_CONTAINER If true, don't remove container on exit (default: false) +# HOST_RPM_DIR Host directory for collected RPMs +# (default: /artifacts/) + +usage() { + cat <<'EOF' +Usage: build_daos_rpms_in_docker.sh + +Example: + ./ci/rpm/build_daos_rpms_in_docker.sh el9 + ./ci/rpm/build_daos_rpms_in_docker.sh leap15 +EOF +} + +if [[ $# -lt 1 || $# -gt 1 ]]; then + usage >&2 + exit 1 +fi + +TARGET="$1" +JOBS="${JOBS:-88}" +KEEP_CONTAINER="${KEEP_CONTAINER:-false}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." >/dev/null 2>&1 && pwd)" +WORKDIR_HOST="${REPO_ROOT}" +WORKDIR_CONT="/workdir" +HOST_RPM_DIR_DEFAULT="${REPO_ROOT}/artifacts/${TARGET}" +HOST_RPM_DIR="${HOST_RPM_DIR:-$HOST_RPM_DIR_DEFAULT}" + +command -v docker >/dev/null 2>&1 || { + echo "docker command not found" >&2 + exit 1 +} + +case "$TARGET" in + el9|leap15) ;; + *) + echo "Invalid target: $TARGET (allowed: el9, leap15)" >&2 + exit 1 + ;; +esac + +CONTAINER_NAME="daos/${TARGET}:build-ci" +STAGE_NAME_VALUE="$TARGET" + +docker image inspect "${CONTAINER_NAME}" >/dev/null 2>&1 || { + echo "Image not found: ${CONTAINER_NAME}" >&2 + echo "Build it first, e.g. docker build --target build-ci -t ${CONTAINER_NAME} ..." >&2 + exit 1 +} + +for required in \ + "${REPO_ROOT}/ci/rpm/build_deps.sh" \ + "${REPO_ROOT}/ci/rpm/gen_rpms.sh"\ + "${REPO_ROOT}/ci/parse_ci_envs.sh"; do + [[ -f "$required" ]] || { + echo "Missing required file: $required" >&2 + exit 1 + } +done + +echo "Starting container from image: ${CONTAINER_NAME}" +CONTAINER="$( + docker run \ + --userns=keep-id \ + -t \ + -d \ + -u "1101:1101" \ + -w "${WORKDIR_CONT}" \ + -v "${WORKDIR_HOST}:${WORKDIR_CONT}:rw,z" \ + "${CONTAINER_NAME}" \ + cat +)" + +echo "Container ID: ${CONTAINER}" + +cleanup() { + if [[ "${KEEP_CONTAINER}" == "true" ]]; then + echo "Keeping container (KEEP_CONTAINER=true): ${CONTAINER}" + return + fi + echo "Stopping/removing container: ${CONTAINER}" + docker rm -f "${CONTAINER}" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +echo "Container top:" +docker top "${CONTAINER}" -eo pid,comm + +echo "Build deps:" +docker exec -i --user daos_server "${CONTAINER}" bash -lc './ci/rpm/build_deps.sh' + +echo "Clean scons:" +docker exec -i --user daos_server "${CONTAINER}" bash -lc '/home/daos/venv/bin/scons -c' + +echo "Remove old build artifacts:" +docker exec -i --user daos_server "${CONTAINER}" bash -lc \ + 'rm -rf _build.external install build daos_m.conf daos.conf iof.conf cart-Linux.conf .sconsign.dblite .sconsign-Linux.dblite .sconf-temp .sconf-temp-Linux' + +echo "Build DAOS:" +docker exec -i --user daos_server "${CONTAINER}" bash -lc \ + "/home/daos/venv/bin/scons --config=force -j ${JOBS} \ + --build-deps=no install USE_INSTALLED=all COMPILER=gcc \ + BUILD_TYPE=dev PREFIX=/opt/daos TARGET_TYPE=release" + +echo "Generate RPMs with STAGE_NAME=${STAGE_NAME_VALUE}:" +docker exec -i --user daos_server "${CONTAINER}" bash -lc \ + "STAGE_NAME=${STAGE_NAME_VALUE} ./ci/rpm/gen_rpms.sh ${TARGET} false" + +echo "Collect RPM artifacts to host: ${HOST_RPM_DIR}" +mkdir -p "${HOST_RPM_DIR}" +docker cp "${CONTAINER}:/home/daos/rpms/." "${HOST_RPM_DIR}" >/dev/null + +echo "RPM artifacts copied to: ${HOST_RPM_DIR}" + +echo "Done." \ No newline at end of file diff --git a/ci/rpm/build_deps.sh b/ci/rpm/build_deps.sh index f989a96332a..06638601d0b 100755 --- a/ci/rpm/build_deps.sh +++ b/ci/rpm/build_deps.sh @@ -1,3 +1,2 @@ #!/bin/bash -cd /home/daos/pre || exit 1 scons install --build-deps=only USE_INSTALLED=all PREFIX=/opt/daos TARGET_TYPE=release -j 32 diff --git a/utils/docker/Dockerfile.el.9 b/utils/docker/Dockerfile.el.9 index 27fa6c27f8a..b40d9c5e4da 100644 --- a/utils/docker/Dockerfile.el.9 +++ b/utils/docker/Dockerfile.el.9 @@ -35,20 +35,22 @@ RUN echo "no_proxy=${DAOS_NO_PROXY}" >> /etc/environment && \ # Script to setup local repo if available COPY ./utils/scripts/helpers/repo-helper-el9.sh /tmp/repo-helper.sh -# Script to install basic tools and daos dependencies -COPY ./utils/scripts/install-el9.sh /tmp/install.sh - RUN chmod +x /tmp/repo-helper.sh && \ /tmp/repo-helper.sh && \ rm -f /tmp/repo-helper.sh -FROM basic AS build-only -ARG INSTALL_ONLY_ESSENTIAL=false +# Install OS updates. +RUN dnf upgrade && \ + dnf clean all -# Install OS updates and package. Include basic tools and daos dependencies -RUN dnf upgrade && \ - chmod +x /tmp/install.sh && \ - INSTALL_ONLY_ESSENTIAL="$INSTALL_ONLY_ESSENTIAL" /tmp/install.sh && \ +FROM basic AS build-ci +# Script to install basic tools and daos dependencies +COPY ./utils/scripts/install-el9.sh /tmp/install.sh + +# Install basic tools and daos dependencies +RUN dnf upgrade && \ + chmod +x /tmp/install.sh && \ + INSTALL_BULD_ESSENTIAL_ONLY="true" /tmp/install.sh && \ dnf clean all # Add DAOS users @@ -57,11 +59,11 @@ COPY ./utils/scripts/helpers/daos-server-user-setup.sh \ /tmp/daos-server-user-setup.sh RUN set -e; \ chmod +x /tmp/daos-server-user-setup.sh && \ - /tmp/daos-server-user-setup.sh && \ + /tmp/daos-server-user-setup.sh && \ rm -f /tmp/daos-server-user-setup.sh # Create directory for DAOS backend storage -RUN mkdir -p /opt/daos /home/daos/pre && \ +RUN mkdir -p /opt/daos /home/daos && \ chown -R daos_server.daos_server /opt/daos /home/daos USER daos_server:daos_server @@ -74,12 +76,23 @@ ENV VIRTUAL_ENV=/home/daos/venv/ # Install latest versions of python tools. COPY requirements-build.txt ./ -RUN . /home/daos/venv/bin/activate && \ +RUN . /home/daos/venv/bin/activate && \ pip --no-cache-dir install --upgrade pip && \ pip --no-cache-dir install -r requirements-build.txt -WORKDIR /home/daos/pre +FROM build-ci AS build-local +# Control what to build. By default Dockerfiles build everything to allow for +# ease-of-use for users, however in CI everything is turned off and then +# selectively enabled. Turning off any step disables all future steps. +ARG DAOS_DEPS_BUILD=yes +ARG DAOS_KEEP_BUILD=no +ARG DAOS_TARGET_TYPE=release +ARG DAOS_PACKAGES_BUILD=yes + +# Create directory for DAOS backend build storage RUN mkdir -p /home/daos/pre/site_scons/prereq_tools /home/daos/pre/site_scons/components + +WORKDIR /home/daos/pre COPY --chown=daos_server:daos_server SConstruct . COPY --chown=daos_server:daos_server deps deps COPY --chown=daos_server:daos_server site_scons/prereq_tools site_scons/prereq_tools @@ -88,33 +101,6 @@ COPY --chown=daos_server:daos_server utils/build.config utils/ COPY --chown=daos_server:daos_server utils/scripts/copy_files.sh utils/scripts/copy_files.sh COPY --chown=daos_server:daos_server utils/scripts/create_spdk_pkgconfig.sh utils/scripts/create_spdk_pkgconfig.sh -FROM build-only AS final -USER root:root -RUN /tmp/install.sh && \ - dnf clean all && \ - rm -f /tmp/install.sh - -# Add DAOS users -RUN useradd --no-log-init --user-group --create-home --shell /bin/bash daos_agent -RUN echo "daos_agent:daos_agent" | chpasswd - -# Create directory for DAOS backend storage -RUN mkdir -p /mnt/daos /var/run/daos_server /var/run/daos_agent /home/daos/daos && \ - chown -R daos_server.daos_server /mnt/daos /var/run/daos_server /home/daos && \ - chown daos_agent.daos_agent /var/run/daos_agent - -USER daos_server:daos_server -COPY requirements-utest.txt ./ -RUN pip --no-cache-dir install -r requirements-utest.txt - -# Control what to build. By default Dockerfiles build everything to allow for -# ease-of-use for users, however in CI everything is turned off and then -# selectively enabled. Turning off any step disables all future steps. -ARG DAOS_DEPS_BUILD=yes -ARG DAOS_KEEP_BUILD=no -ARG DAOS_TARGET_TYPE=release -ARG DAOS_PACKAGES_BUILD=yes - # Now do an update to ensure software is up to date for the deps build. If the # src hasn't changed then this won't do anything, but if it has then we want to # ensure that latest dependencies are used. @@ -123,10 +109,10 @@ RUN [ "$DAOS_DEPS_BUILD" != "yes" ] || { dnf upgrade --exclude=spdk,spdk-devel,dpdk-devel,dpdk,mercury-devel,mercury && \ dnf clean all; \ } -USER daos_server:daos_server +# Build third party +USER daos_server:daos_server ARG DEPS_JOBS=1 - RUN [ "$DAOS_DEPS_BUILD" != "yes" ] || { \ scons --build-deps=only --jobs $DEPS_JOBS PREFIX=/opt/daos \ TARGET_TYPE=$DAOS_TARGET_TYPE && \ @@ -138,25 +124,26 @@ COPY --chown=daos_server:daos_server utils/sl utils/sl # Build third party RPMs RUN [ "$DAOS_PACKAGES_BUILD" != "yes" ] || [ "$DAOS_DEPS_BUILD" != "yes" ] || { \ - export DISTRO="el9" && \ - utils/rpms/build_packages.sh deps && \ - mkdir -p /home/daos/rpms && \ - mv *.rpm /home/daos/rpms; \ + export DISTRO="el9" && \ + utils/rpms/build_packages.sh deps && \ + mkdir -p /home/daos/rpms/deps && \ + mv *.rpm /home/daos/rpms/deps; \ } -USER root:root +USER root:root # force an upgrade to get any newly built RPMs, but only if CB1 is set. ARG CB1 -RUN [ -z "$CB1" ] || { \ +RUN [ -z "$CB1" ] || { \ dnf upgrade --exclude=spdk,spdk-devel,dpdk-devel,dpdk,mercury-devel,mercury && \ dnf clean all; \ } -USER daos_server:daos_server +USER daos_server:daos_server # Set a label. This is useful for searching for DAOS images, but is also used # in github-actions to prune elements of the dockerfile below this point. LABEL DAOS=true +RUN mkdir -p /home/daos/daos WORKDIR /home/daos/daos/ COPY --chown=daos_server:daos_server VERSION LICENSE ftest.sh SConstruct requirements-ftest.txt .clang-format ./ COPY --chown=daos_server:daos_server site_scons site_scons @@ -176,7 +163,7 @@ ARG DAOS_BUILD=$DAOS_DEPS_BUILD RUN [ "$DAOS_BUILD" != "yes" ] || { \ scons --jobs $JOBS install PREFIX=/opt/daos COMPILER=$COMPILER \ FIRMWARE_MGMT=1 BUILD_TYPE=$DAOS_BUILD_TYPE \ - TARGET_TYPE=$DAOS_TARGET_TYPE && \ + TARGET_TYPE=$DAOS_TARGET_TYPE && \ ([ "$DAOS_KEEP_BUILD" != "no" ] || /bin/rm -rf build) && \ go clean -cache && \ cp -r utils/config/examples /opt/daos; \ @@ -186,22 +173,17 @@ COPY --chown=daos_server:daos_server utils utils # Build DAOS RPMs RUN [ "$DAOS_PACKAGES_BUILD" != "yes" ] || [ "$DAOS_BUILD" != "yes" ] || { \ - export DISTRO="el9" && \ - utils/rpms/build_packages.sh daos && \ - mkdir -p /home/daos/rpms && \ - cp *.rpm /home/daos/rpms; \ + export DISTRO="el9" && \ + utils/rpms/build_packages.sh daos && \ + mkdir -p /home/daos/rpms/daos && \ + mv *.rpm /home/daos/rpms/daos; \ } -# Set environment variables -ENV PATH=/opt/daos/bin:$PATH -ENV FI_SOCKETS_MAX_CONN_RETRY=1 - -# Build java and hadoop bindings -WORKDIR /home/daos/daos/src/client/java - -ARG DAOS_JAVA_BUILD=$DAOS_BUILD - # Disable Java build for now since it fails +# Build java and hadoop bindings +#WORKDIR /home/daos/daos/src/client/java +# +#ARG DAOS_JAVA_BUILD=$DAOS_BUILD #RUN [ "$DAOS_JAVA_BUILD" != "yes" ] || { \ # mkdir /home/daos/.m2 && \ # cp /home/daos/daos/utils/scripts/helpers/maven-settings.xml.in /home/daos/.m2/settings.xml && \ @@ -209,8 +191,33 @@ ARG DAOS_JAVA_BUILD=$DAOS_BUILD # -B -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \ # -DskipITs -Dgpg.skip -Ddaos.install.path=/opt/daos; \ # } -WORKDIR /home/daos +#WORKDIR /home/daos -ARG DAOS_KEEP_SRC=no # Remove local copy +ARG DAOS_KEEP_SRC=no RUN [ "$DAOS_KEEP_SRC" != "no" ] || rm -rf /home/daos/daos /home/daos/pre + +FROM build-local AS full +# Install remaining tools and dependencies needed for testing. +# This is done in a separate stage to allow for a smaller image for building only. +USER root:root +RUN /tmp/install.sh && \ + dnf clean all && \ + rm -f /tmp/install.sh + +# Add DAOS agent users +RUN useradd --no-log-init --user-group --create-home --shell /bin/bash daos_agent +RUN echo "daos_agent:daos_agent" | chpasswd + +# Create directory for DAOS runetime +RUN mkdir -p /mnt/daos /var/run/daos_server /var/run/daos_agent && \ + chown -R daos_server.daos_server /mnt/daos /var/run/daos_server && \ + chown -R daos_agent.daos_agent /var/run/daos_agent + +USER daos_server:daos_server +COPY requirements-utest.txt ./ +RUN pip --no-cache-dir install -r requirements-utest.txt + +# Set environment variables +ENV PATH=/opt/daos/bin:$PATH +ENV FI_SOCKETS_MAX_CONN_RETRY=1 diff --git a/utils/scripts/install-el9.sh b/utils/scripts/install-el9.sh index d67a99149bd..dc8786582eb 100755 --- a/utils/scripts/install-el9.sh +++ b/utils/scripts/install-el9.sh @@ -81,7 +81,7 @@ dnf --nodocs install ${dnf_install_args} \ ncurses-devel \ yasm -if [[ "${INSTALL_ONLY_ESSENTIAL:-}" != "true" ]]; then +if [[ "${INSTALL_BULD_ESSENTIAL_ONLY:-}" != "true" ]]; then # Optional packages for full-featured images; can be skipped in essential-only mode. # shellcheck disable=SC2086 dnf --nodocs install ${dnf_install_args} \ From 9fe742b71d532f1b3dc5d5f46ea2c55498406b04 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Fri, 3 Jul 2026 15:25:00 +0200 Subject: [PATCH 4/9] Fix github builds Signed-off-by: Tomasz Gromadzki Priority: 2 Cancel-prev-build: false Skip-python-bandit: true Skip-unit-test: true Skip-unit-test-memcheck: true Skip-test-el-9-rpms: false Skip-test-leap-15-rpms: true Skip-func-hw-test: true Skip-build-el8-gcc: true Skip-build-leap15-gcc: false Skip-unit-tests:true Skip-NLT: true Skip-fault-injection-test: true Skip-func-vm: true --- Jenkinsfile | 1 + utils/docker/Dockerfile.el.9 | 17 ++++++++--------- utils/scripts/install-el9.sh | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 444455b9867..ed4c4c4a5e7 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -898,6 +898,7 @@ pipeline { ' --build-arg POINT_RELEASE=.7' + " --build-arg PYTHON_VERSION=${env.PYTHON_VERSION}" + ' --target build-ci' + } } steps { script { diff --git a/utils/docker/Dockerfile.el.9 b/utils/docker/Dockerfile.el.9 index b40d9c5e4da..daf391d5708 100644 --- a/utils/docker/Dockerfile.el.9 +++ b/utils/docker/Dockerfile.el.9 @@ -50,7 +50,7 @@ COPY ./utils/scripts/install-el9.sh /tmp/install.sh # Install basic tools and daos dependencies RUN dnf upgrade && \ chmod +x /tmp/install.sh && \ - INSTALL_BULD_ESSENTIAL_ONLY="true" /tmp/install.sh && \ + INSTALL_BUILD_ONLY="true" /tmp/install.sh && \ dnf clean all # Add DAOS users @@ -81,6 +81,13 @@ RUN . /home/daos/venv/bin/activate && \ pip --no-cache-dir install -r requirements-build.txt FROM build-ci AS build-local +# Install remaining tools and dependencies needed for builds. +# This is done in a separate stage to allow for a smaller image for CI building only. +USER root:root +RUN /tmp/install.sh && \ + dnf clean all && \ + rm -f /tmp/install.sh + # Control what to build. By default Dockerfiles build everything to allow for # ease-of-use for users, however in CI everything is turned off and then # selectively enabled. Turning off any step disables all future steps. @@ -104,7 +111,6 @@ COPY --chown=daos_server:daos_server utils/scripts/create_spdk_pkgconfig.sh util # Now do an update to ensure software is up to date for the deps build. If the # src hasn't changed then this won't do anything, but if it has then we want to # ensure that latest dependencies are used. -USER root:root RUN [ "$DAOS_DEPS_BUILD" != "yes" ] || { \ dnf upgrade --exclude=spdk,spdk-devel,dpdk-devel,dpdk,mercury-devel,mercury && \ dnf clean all; \ @@ -198,13 +204,6 @@ ARG DAOS_KEEP_SRC=no RUN [ "$DAOS_KEEP_SRC" != "no" ] || rm -rf /home/daos/daos /home/daos/pre FROM build-local AS full -# Install remaining tools and dependencies needed for testing. -# This is done in a separate stage to allow for a smaller image for building only. -USER root:root -RUN /tmp/install.sh && \ - dnf clean all && \ - rm -f /tmp/install.sh - # Add DAOS agent users RUN useradd --no-log-init --user-group --create-home --shell /bin/bash daos_agent RUN echo "daos_agent:daos_agent" | chpasswd diff --git a/utils/scripts/install-el9.sh b/utils/scripts/install-el9.sh index dc8786582eb..78cc705b458 100755 --- a/utils/scripts/install-el9.sh +++ b/utils/scripts/install-el9.sh @@ -81,7 +81,7 @@ dnf --nodocs install ${dnf_install_args} \ ncurses-devel \ yasm -if [[ "${INSTALL_BULD_ESSENTIAL_ONLY:-}" != "true" ]]; then +if [[ "${INSTALL_BUILD_ESSENTIAL_ONLY:-}" != "true" ]]; then # Optional packages for full-featured images; can be skipped in essential-only mode. # shellcheck disable=SC2086 dnf --nodocs install ${dnf_install_args} \ From e1214efd66aff0b331d3d7f63c3b26e4be1febc8 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Fri, 3 Jul 2026 16:27:02 +0200 Subject: [PATCH 5/9] Typo fix Signed-off-by: Tomasz Gromadzki --- utils/docker/Dockerfile.el.9 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/docker/Dockerfile.el.9 b/utils/docker/Dockerfile.el.9 index daf391d5708..ed83fd1cef3 100644 --- a/utils/docker/Dockerfile.el.9 +++ b/utils/docker/Dockerfile.el.9 @@ -208,7 +208,7 @@ FROM build-local AS full RUN useradd --no-log-init --user-group --create-home --shell /bin/bash daos_agent RUN echo "daos_agent:daos_agent" | chpasswd -# Create directory for DAOS runetime +# Create directory for DAOS runtime RUN mkdir -p /mnt/daos /var/run/daos_server /var/run/daos_agent && \ chown -R daos_server.daos_server /mnt/daos /var/run/daos_server && \ chown -R daos_agent.daos_agent /var/run/daos_agent From 4c10c5d1cf139196948f6c1c205c470479cae13f Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Fri, 3 Jul 2026 16:27:55 +0200 Subject: [PATCH 6/9] Test with new pmdk rpm Signed-off-by: Tomasz Gromadzki Priority: 2 Cancel-prev-build: false Skip-python-bandit: true Skip-unit-test: true Skip-unit-test-memcheck: true Skip-test-el-9-rpms: false Skip-test-leap-15-rpms: true Skip-func-hw-test: true Skip-build-el8-gcc: true Skip-build-leap15-gcc: false Skip-unit-tests:true Skip-NLT: true Skip-fault-injection-test: true Skip-func-vm: true --- utils/rpms/package_info.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/rpms/package_info.sh b/utils/rpms/package_info.sh index 382fbb39aa9..d289f39ce29 100644 --- a/utils/rpms/package_info.sh +++ b/utils/rpms/package_info.sh @@ -51,7 +51,7 @@ export argobots_version="1.2" export argobots_release="4${distro_name}" export argobots_full="${argobots_version}-${argobots_release}" export pmdk_version="2.1.3" -export pmdk_release="2${distro_name}" +export pmdk_release="5${distro_name}" export pmdk_full="${pmdk_version}-${pmdk_release}" export isal_version="2.31.1" export isal_release="8${distro_name}" From 7f8e2a934cacd19cd1073ecdc0e5fa5e8babef09 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Fri, 3 Jul 2026 16:35:53 +0200 Subject: [PATCH 7/9] Fix access right of /home/daos Signed-off-by: Tomasz Gromadzki Priority: 2 Cancel-prev-build: false Skip-python-bandit: true Skip-unit-test: true Skip-unit-test-memcheck: true Skip-test-el-9-rpms: false Skip-test-leap-15-rpms: true Skip-func-hw-test: true Skip-build-el8-gcc: true Skip-build-leap15-gcc: false Skip-unit-tests:true Skip-NLT: true Skip-fault-injection-test: true Skip-func-vm: true Signed-off-by: Tomasz Gromadzki --- utils/docker/Dockerfile.el.9 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/utils/docker/Dockerfile.el.9 b/utils/docker/Dockerfile.el.9 index ed83fd1cef3..8a245043243 100644 --- a/utils/docker/Dockerfile.el.9 +++ b/utils/docker/Dockerfile.el.9 @@ -108,6 +108,8 @@ COPY --chown=daos_server:daos_server utils/build.config utils/ COPY --chown=daos_server:daos_server utils/scripts/copy_files.sh utils/scripts/copy_files.sh COPY --chown=daos_server:daos_server utils/scripts/create_spdk_pkgconfig.sh utils/scripts/create_spdk_pkgconfig.sh +RUN chown -R daos_server.daos_server /home/daos + # Now do an update to ensure software is up to date for the deps build. If the # src hasn't changed then this won't do anything, but if it has then we want to # ensure that latest dependencies are used. From c931535cb200116bb02ccf8511def1171264d99b Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Fri, 3 Jul 2026 21:05:52 +0200 Subject: [PATCH 8/9] Fix access Signed-off-by: Tomasz Gromadzki Priority: 2 Cancel-prev-build: false Skip-python-bandit: true Skip-unit-test: true Skip-unit-test-memcheck: true Skip-test-el-9-rpms: false Skip-test-leap-15-rpms: true Skip-func-hw-test: true Skip-build-el8-gcc: true Skip-build-leap15-gcc: false Skip-unit-tests:true Skip-NLT: true Skip-fault-injection-test: true Skip-func-vm: true Skip-func: true Skip-test: true Signed-off-by: Tomasz Gromadzki --- utils/docker/Dockerfile.el.9 | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/docker/Dockerfile.el.9 b/utils/docker/Dockerfile.el.9 index 8a245043243..aa115f4741b 100644 --- a/utils/docker/Dockerfile.el.9 +++ b/utils/docker/Dockerfile.el.9 @@ -206,6 +206,7 @@ ARG DAOS_KEEP_SRC=no RUN [ "$DAOS_KEEP_SRC" != "no" ] || rm -rf /home/daos/daos /home/daos/pre FROM build-local AS full +USER root:root # Add DAOS agent users RUN useradd --no-log-init --user-group --create-home --shell /bin/bash daos_agent RUN echo "daos_agent:daos_agent" | chpasswd From ffe61fec9676b361f41b2b9569128b48d3b6ec97 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Sun, 5 Jul 2026 17:04:39 +0200 Subject: [PATCH 9/9] Use proper user in runtime Signed-off-by: Tomasz Gromadzki Priority: 2 Cancel-prev-build: false Skip-python-bandit: true Skip-unit-test: true Skip-unit-test-memcheck: true Skip-test-el-9-rpms: false Skip-test-leap-15-rpms: true Skip-func-hw-test: true Skip-build-el8-gcc: true Skip-build-leap15-gcc: false Skip-unit-tests:true Skip-NLT: true Skip-fault-injection-test: true Skip-func-vm: true Skip-func: true Skip-test: true Signed-off-by: Tomasz Gromadzki --- .github/workflows/landing-builds.yml | 2 +- ci/rpm/build_daos_rpms_in_docker.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/landing-builds.yml b/.github/workflows/landing-builds.yml index b3265275771..95cd56a0447 100644 --- a/.github/workflows/landing-builds.yml +++ b/.github/workflows/landing-builds.yml @@ -345,7 +345,7 @@ jobs: --build-arg DAOS_BUILD_TYPE=dev --build-arg COMPILER=gcc - name: Run NLT - run: docker run --mount type=tmpfs,destination=/mnt/daos_0,tmpfs-mode=1777 --user root:root + run: docker run --mount type=tmpfs,destination=/mnt/daos_0,tmpfs-mode=1777 --user daos_server:daos_server build-image ./daos/utils/node_local_test.py --no-root --memcheck no --test cont_copy --system-ram-reserved 1 diff --git a/ci/rpm/build_daos_rpms_in_docker.sh b/ci/rpm/build_daos_rpms_in_docker.sh index 2eefcc21318..61f028c46fd 100755 --- a/ci/rpm/build_daos_rpms_in_docker.sh +++ b/ci/rpm/build_daos_rpms_in_docker.sh @@ -5,10 +5,10 @@ set -euo pipefail # to the host before container teardown. # # Usage: -# ./ci/rpm/build_daos_rpms_in_docker.sh +# ./ci/rpm/build_daos_rpms_in_docker.sh # # Arguments: -# Image flavor tag suffix (el9, el8, leap15) +# Image flavor tag suffix (el9, leap15) # # Requirements: # - docker command must be available on host