SRE-3874 ci: optimized docker images size

.github/workflows/landing-builds.yml

@@ -345,7 +345,7 @@ jobs:
                             --build-arg DAOS_BUILD_TYPE=dev
                             --build-arg COMPILER=gcc
       - name: Run NLT
-        run: docker run --mount type=tmpfs,destination=/mnt/daos_0,tmpfs-mode=1777 --user root:root
+        run: docker run --mount type=tmpfs,destination=/mnt/daos_0,tmpfs-mode=1777 --user daos_server:daos_server
                  build-image ./daos/utils/node_local_test.py --no-root
                  --memcheck no --test cont_copy --system-ram-reserved 1
 

Jenkinsfile

@@ -896,7 +896,8 @@ pipeline {
                                                 ' --build-arg DAOS_KEEP_SRC=yes ' +
                                                 ' --build-arg REPOS="' + prRepos() + '"' +
                                                 ' --build-arg POINT_RELEASE=.7' +
-                                                " --build-arg PYTHON_VERSION=${env.PYTHON_VERSION}"
+                                                " --build-arg PYTHON_VERSION=${env.PYTHON_VERSION}" +
+                                                ' --target build-ci'
                         }
                     }
                     steps {

ci/rpm/build_daos_rpms_in_docker.sh

@@ -0,0 +1,136 @@
+#!/bin/bash
+set -euo pipefail
+
+# Build DAOS RPMs inside a disposable container and copy resulting RPMs back
+# to the host before container teardown.
+#
+# Usage:
+#   ./ci/rpm/build_daos_rpms_in_docker.sh <el9|leap15>
+#
+# Arguments:
+#   <target>   Image flavor tag suffix (el9, leap15)
+#
+# Requirements:
+#   - docker command must be available on host
+#   - host must contain a built image for the specified target, e.g. daos/el9:build-ci
+#   - host must contain /home/daos directory with write permissions for the current user
+#
+# Environment variables:
+#   JOBS             SCons parallelism for main build (default: 88)
+#   KEEP_CONTAINER   If true, don't remove container on exit (default: false)
+#   HOST_RPM_DIR     Host directory for collected RPMs
+#                   (default: <repo>/artifacts/<target>)
+
+usage() {
+  cat <<'EOF'
+Usage: build_daos_rpms_in_docker.sh <el9|leap15>
+
+Example:
+  ./ci/rpm/build_daos_rpms_in_docker.sh el9
+  ./ci/rpm/build_daos_rpms_in_docker.sh leap15
+EOF
+}
+
+if [[ $# -lt 1 || $# -gt 1 ]]; then
+  usage >&2
+  exit 1
+fi
+
+TARGET="$1"
+JOBS="${JOBS:-88}"
+KEEP_CONTAINER="${KEEP_CONTAINER:-false}"
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." >/dev/null 2>&1 && pwd)"
+WORKDIR_HOST="${REPO_ROOT}"
+WORKDIR_CONT="/workdir"
+HOST_RPM_DIR_DEFAULT="${REPO_ROOT}/artifacts/${TARGET}"
+HOST_RPM_DIR="${HOST_RPM_DIR:-$HOST_RPM_DIR_DEFAULT}"
+
+command -v docker >/dev/null 2>&1 || {
+  echo "docker command not found" >&2
+  exit 1
+}
+
+case "$TARGET" in
+  el9|leap15) ;;
+  *)
+    echo "Invalid target: $TARGET (allowed: el9, leap15)" >&2
+    exit 1
+    ;;
+esac
+
+CONTAINER_NAME="daos/${TARGET}:build-ci"
+STAGE_NAME_VALUE="$TARGET"
+
+docker image inspect "${CONTAINER_NAME}" >/dev/null 2>&1 || {
+  echo "Image not found: ${CONTAINER_NAME}" >&2
+  echo "Build it first, e.g. docker build --target build-ci -t ${CONTAINER_NAME} ..." >&2
+  exit 1
+}
+
+for required in \
+  "${REPO_ROOT}/ci/rpm/build_deps.sh" \
+  "${REPO_ROOT}/ci/rpm/gen_rpms.sh"\
+  "${REPO_ROOT}/ci/parse_ci_envs.sh"; do
+  [[ -f "$required" ]] || {
+    echo "Missing required file: $required" >&2
+    exit 1
+  }
+done
+
+echo "Starting container from image: ${CONTAINER_NAME}"
+CONTAINER="$(
+  docker run \
+    --userns=keep-id \
+    -t \
+    -d \
+    -u "1101:1101" \
+    -w "${WORKDIR_CONT}" \
+    -v "${WORKDIR_HOST}:${WORKDIR_CONT}:rw,z" \
+    "${CONTAINER_NAME}" \
+    cat
+)"
+
+echo "Container ID: ${CONTAINER}"
+
+cleanup() {
+  if [[ "${KEEP_CONTAINER}" == "true" ]]; then
+    echo "Keeping container (KEEP_CONTAINER=true): ${CONTAINER}"
+    return
+  fi
+  echo "Stopping/removing container: ${CONTAINER}"
+  docker rm -f "${CONTAINER}" >/dev/null 2>&1 || true
+}
+trap cleanup EXIT
+
+echo "Container top:"
+docker top "${CONTAINER}" -eo pid,comm
+
+echo "Build deps:"
+docker exec -i --user daos_server "${CONTAINER}" bash -lc './ci/rpm/build_deps.sh'
+
+echo "Clean scons:"
+docker exec -i --user daos_server "${CONTAINER}" bash -lc '/home/daos/venv/bin/scons -c'
+
+echo "Remove old build artifacts:"
+docker exec -i --user daos_server "${CONTAINER}" bash -lc \
+  'rm -rf _build.external install build daos_m.conf daos.conf iof.conf cart-Linux.conf .sconsign.dblite .sconsign-Linux.dblite .sconf-temp .sconf-temp-Linux'
+
+echo "Build DAOS:"
+docker exec -i --user daos_server "${CONTAINER}" bash -lc \
+       "/home/daos/venv/bin/scons --config=force -j ${JOBS} \
+       --build-deps=no install USE_INSTALLED=all COMPILER=gcc \
+       BUILD_TYPE=dev PREFIX=/opt/daos TARGET_TYPE=release"
+
+echo "Generate RPMs with STAGE_NAME=${STAGE_NAME_VALUE}:"
+docker exec -i --user daos_server "${CONTAINER}" bash -lc \
+       "STAGE_NAME=${STAGE_NAME_VALUE} ./ci/rpm/gen_rpms.sh ${TARGET} false"
+
+echo "Collect RPM artifacts to host: ${HOST_RPM_DIR}"
+mkdir -p "${HOST_RPM_DIR}"
+docker cp "${CONTAINER}:/home/daos/rpms/." "${HOST_RPM_DIR}" >/dev/null
+
+echo "RPM artifacts copied to: ${HOST_RPM_DIR}"
+
+echo "Done."

ci/rpm/build_deps.sh

@@ -1,3 +1,2 @@
 #!/bin/bash
-cd /home/daos/pre || exit 1
 scons install --build-deps=only USE_INSTALLED=all PREFIX=/opt/daos TARGET_TYPE=release -j 32

utils/docker/Dockerfile.el.9

@@ -32,36 +32,39 @@
 RUN echo "no_proxy=${DAOS_NO_PROXY}" >> /etc/environment && \
     echo "NO_PROXY=${DAOS_NO_PROXY}" >> /etc/environment
 
-# script to install OS updates basic tools and daos dependencies
-COPY ./utils/scripts/install-el9.sh /tmp/install.sh
-# script to setup local repo if available
+# Script to setup local repo if available
 COPY ./utils/scripts/helpers/repo-helper-el9.sh /tmp/repo-helper.sh
 
-RUN chmod +x /tmp/repo-helper.sh /tmp/install.sh && \
-    /tmp/repo-helper.sh &&                          \
+RUN chmod +x /tmp/repo-helper.sh && \
+    /tmp/repo-helper.sh &&          \
     rm -f /tmp/repo-helper.sh
 
-FROM basic
-# Install OS updates and package.  Include basic tools and daos dependencies
-RUN dnf upgrade &&        \
-    /tmp/install.sh &&    \
-    dnf clean all &&      \
-    rm -f /tmp/install.sh
+# Install OS updates.
+RUN dnf upgrade && \
+    dnf clean all
+
+FROM basic AS build-ci
+# Script to install basic tools and daos dependencies
+COPY ./utils/scripts/install-el9.sh /tmp/install.sh
+
+# Install basic tools and daos dependencies
+RUN dnf upgrade &&                                        \
+    chmod +x /tmp/install.sh &&                           \
+    INSTALL_BUILD_ONLY="true" /tmp/install.sh && \
+    dnf clean all
 
 # Add DAOS users
 ARG UID=1000
 COPY ./utils/scripts/helpers/daos-server-user-setup.sh \
      /tmp/daos-server-user-setup.sh
 RUN set -e;                                    \
     chmod +x /tmp/daos-server-user-setup.sh && \
-    /tmp/daos-server-user-setup.sh
-RUN useradd --no-log-init --user-group --create-home --shell /bin/bash daos_agent
-RUN echo "daos_agent:daos_agent" | chpasswd
+    /tmp/daos-server-user-setup.sh &&          \
+    rm -f /tmp/daos-server-user-setup.sh
 
 # Create directory for DAOS backend storage
-RUN mkdir -p /opt/daos /mnt/daos /var/run/daos_server /var/run/daos_agent /home/daos/pre /home/daos/daos &&   \
-    chown -R daos_server.daos_server /opt/daos /mnt/daos /var/run/daos_server /home/daos &&  \
-    chown daos_agent.daos_agent /var/run/daos_agent
+RUN mkdir -p /opt/daos /home/daos && \
+    chown -R daos_server.daos_server /opt/daos /home/daos
 
 USER daos_server:daos_server
 
@@ -72,13 +75,31 @@
 ENV VIRTUAL_ENV=/home/daos/venv/
 
 # Install latest versions of python tools.
-COPY requirements-build.txt requirements-utest.txt ./
-RUN . /home/daos/venv/bin/activate && \
+COPY requirements-build.txt ./
+RUN . /home/daos/venv/bin/activate &&           \
     pip --no-cache-dir install --upgrade pip && \
-    pip --no-cache-dir install -r requirements-build.txt -r requirements-utest.txt
+    pip --no-cache-dir install -r requirements-build.txt
 
-WORKDIR /home/daos/pre
+FROM build-ci AS build-local
+# Install remaining tools and dependencies needed for builds.
+# This is done in a separate stage to allow for a smaller image for CI building only.
+USER root:root
+RUN /tmp/install.sh && \
+    dnf clean all &&   \
+    rm -f /tmp/install.sh
+
+# Control what to build.  By default Dockerfiles build everything to allow for
+# ease-of-use for users, however in CI everything is turned off and then
+# selectively enabled.  Turning off any step disables all future steps.
+ARG DAOS_DEPS_BUILD=yes
+ARG DAOS_KEEP_BUILD=no
+ARG DAOS_TARGET_TYPE=release
+ARG DAOS_PACKAGES_BUILD=yes
+
+# Create directory for DAOS backend build storage
 RUN mkdir -p /home/daos/pre/site_scons/prereq_tools /home/daos/pre/site_scons/components
+
+WORKDIR /home/daos/pre
 COPY --chown=daos_server:daos_server SConstruct .
 COPY --chown=daos_server:daos_server deps deps
 COPY --chown=daos_server:daos_server site_scons/prereq_tools site_scons/prereq_tools
@@ -87,26 +108,19 @@
 COPY --chown=daos_server:daos_server utils/scripts/copy_files.sh utils/scripts/copy_files.sh
 COPY --chown=daos_server:daos_server utils/scripts/create_spdk_pkgconfig.sh utils/scripts/create_spdk_pkgconfig.sh
 
-# Control what to build.  By default Dockerfiles build everything to allow for
-# ease-of-use for users, however in CI everything is turned off and then
-# selectively enabled.  Turning off any step disables all future steps.
-ARG DAOS_DEPS_BUILD=yes
-ARG DAOS_KEEP_BUILD=no
-ARG DAOS_TARGET_TYPE=release
-ARG DAOS_PACKAGES_BUILD=yes
+RUN chown -R daos_server.daos_server /home/daos 
 
 # Now do an update to ensure software is up to date for the deps build.  If the
 # src hasn't changed then this won't do anything, but if it has then we want to
 # ensure that latest dependencies are used.
-USER root:root
 RUN [ "$DAOS_DEPS_BUILD" != "yes" ] || {                                               \
         dnf upgrade --exclude=spdk,spdk-devel,dpdk-devel,dpdk,mercury-devel,mercury && \
         dnf clean all;                                                                 \
     }
-USER daos_server:daos_server
 
+# Build third party
+USER daos_server:daos_server
 ARG DEPS_JOBS=1
-
 RUN [ "$DAOS_DEPS_BUILD" != "yes" ] || {                            \
         scons --build-deps=only --jobs $DEPS_JOBS PREFIX=/opt/daos  \
               TARGET_TYPE=$DAOS_TARGET_TYPE &&                      \
@@ -118,25 +132,26 @@
 
 # Build third party RPMs
 RUN [ "$DAOS_PACKAGES_BUILD" != "yes" ] || [ "$DAOS_DEPS_BUILD" != "yes" ] || { \
-	export DISTRO="el9" &&                                                  \
-	utils/rpms/build_packages.sh deps &&                                    \
-	mkdir -p /home/daos/rpms &&                                             \
-	mv *.rpm /home/daos/rpms;                                               \
+	export DISTRO="el9" &&                                                      \
+	utils/rpms/build_packages.sh deps &&                                        \
+	mkdir -p /home/daos/rpms/deps &&                                            \
+	mv *.rpm /home/daos/rpms/deps;                                              \
     }
-USER root:root
 
+USER root:root
 # force an upgrade to get any newly built RPMs, but only if CB1 is set.
 ARG CB1
-RUN [ -z "$CB1" ] || {                                                           \
+RUN [ -z "$CB1" ] || {                                                                 \
         dnf upgrade --exclude=spdk,spdk-devel,dpdk-devel,dpdk,mercury-devel,mercury && \
         dnf clean all;                                                                 \
     }
-USER daos_server:daos_server
 
+USER daos_server:daos_server
 # Set a label.  This is useful for searching for DAOS images, but is also used
 # in github-actions to prune elements of the dockerfile below this point.
 LABEL DAOS=true
 
+RUN mkdir -p /home/daos/daos
 WORKDIR /home/daos/daos/
 COPY --chown=daos_server:daos_server VERSION LICENSE ftest.sh SConstruct requirements-ftest.txt .clang-format ./
 COPY --chown=daos_server:daos_server site_scons site_scons
@@ -156,7 +171,7 @@
 RUN [ "$DAOS_BUILD" != "yes" ] || {                                        \
         scons --jobs $JOBS install PREFIX=/opt/daos COMPILER=$COMPILER     \
               FIRMWARE_MGMT=1 BUILD_TYPE=$DAOS_BUILD_TYPE                  \
-	      TARGET_TYPE=$DAOS_TARGET_TYPE &&                             \
+	      TARGET_TYPE=$DAOS_TARGET_TYPE &&                                 \
         ([ "$DAOS_KEEP_BUILD" != "no" ] || /bin/rm -rf build) &&           \
         go clean -cache &&                                                 \
         cp -r utils/config/examples /opt/daos;                             \
@@ -166,31 +181,45 @@
 
 # Build DAOS RPMs
 RUN [ "$DAOS_PACKAGES_BUILD" != "yes" ] || [ "$DAOS_BUILD" != "yes" ] || { \
-	export DISTRO="el9" &&                                             \
-	utils/rpms/build_packages.sh daos &&                               \
-	mkdir -p /home/daos/rpms &&                                        \
-	cp *.rpm /home/daos/rpms;                                          \
+	export DISTRO="el9" &&                                                 \
+	utils/rpms/build_packages.sh daos &&                                   \
+	mkdir -p /home/daos/rpms/daos &&                                       \
+	mv *.rpm /home/daos/rpms/daos;                                         \
     }
 
-# Set environment variables
-ENV PATH=/opt/daos/bin:$PATH
-ENV FI_SOCKETS_MAX_CONN_RETRY=1
-
-# Build java and hadoop bindings
-WORKDIR /home/daos/daos/src/client/java
-
-ARG DAOS_JAVA_BUILD=$DAOS_BUILD
-
 # Disable Java build for now since it fails
+# Build java and hadoop bindings
+#WORKDIR /home/daos/daos/src/client/java
+#
+#ARG DAOS_JAVA_BUILD=$DAOS_BUILD
 #RUN [ "$DAOS_JAVA_BUILD" != "yes" ] || {                                                      \
 #        mkdir /home/daos/.m2 &&                                                               \
 #        cp /home/daos/daos/utils/scripts/helpers/maven-settings.xml.in /home/daos/.m2/settings.xml &&      \
 #        mvn clean install -T 1C                                                               \
 #            -B -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn                                                   \
 #            -DskipITs -Dgpg.skip -Ddaos.install.path=/opt/daos;                               \
 #    }
-WORKDIR /home/daos
+#WORKDIR /home/daos
 
-ARG DAOS_KEEP_SRC=no
 # Remove local copy
+ARG DAOS_KEEP_SRC=no
 RUN [ "$DAOS_KEEP_SRC" != "no" ] || rm -rf /home/daos/daos /home/daos/pre
+
+FROM build-local AS full
+USER root:root
+# Add DAOS agent users
+RUN useradd --no-log-init --user-group --create-home --shell /bin/bash daos_agent
+RUN echo "daos_agent:daos_agent" | chpasswd
+
+# Create directory for DAOS runtime
+RUN mkdir -p /mnt/daos /var/run/daos_server /var/run/daos_agent &&     \
+    chown -R daos_server.daos_server /mnt/daos /var/run/daos_server && \
+    chown -R daos_agent.daos_agent /var/run/daos_agent
+
+USER daos_server:daos_server
+COPY requirements-utest.txt ./
+RUN pip --no-cache-dir install -r requirements-utest.txt
+
+# Set environment variables
+ENV PATH=/opt/daos/bin:$PATH
+ENV FI_SOCKETS_MAX_CONN_RETRY=1

utils/rpms/package_info.sh

@@ -51,7 +51,7 @@ export argobots_version="1.2"
 export argobots_release="4${distro_name}"
 export argobots_full="${argobots_version}-${argobots_release}"
 export pmdk_version="2.1.3"
-export pmdk_release="2${distro_name}"
+export pmdk_release="5${distro_name}"
 export pmdk_full="${pmdk_version}-${pmdk_release}"
 export isal_version="2.31.1"
 export isal_release="8${distro_name}"