diff --git a/Jenkinsfile b/Jenkinsfile index f586e6897a1..43564fa4e46 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -582,7 +582,7 @@ pipeline { agent { dockerfile { filename 'utils/docker/Dockerfile.el.9' - label 'docker_runner' + label 'fox-119_docker_1' additionalBuildArgs dockerBuildArgs(repo_type: 'stable', deps_build: false, parallel_build: true) + @@ -590,7 +590,8 @@ pipeline { ' --build-arg DAOS_PACKAGES_BUILD=no ' + ' --build-arg DAOS_KEEP_SRC=yes ' + ' --build-arg REPOS="' + prRepos() + '"' + - ' --build-arg POINT_RELEASE=.7 ' + ' --build-arg POINT_RELEASE=.7 ' + + ' --target basic' } } steps { diff --git a/ci/docker/Dockerfile.maldet.el.8 b/ci/docker/Dockerfile.maldet.el.8 index dd333d49aa0..baa2416f430 100755 --- a/ci/docker/Dockerfile.maldet.el.8 +++ b/ci/docker/Dockerfile.maldet.el.8 @@ -1,5 +1,5 @@ # Copyright 2018-2022 Intel Corporation -# Copyright 2025 Hewlett Packard Enterprise Development LP +# Copyright 2025-2026 Hewlett Packard Enterprise Development LP # All rights reserved. # # 'recipe' for Docker to build an environment scan DAOS for malware. @@ -19,31 +19,25 @@ ARG REPO_FILE_URL ARG JENKINS_URL ARG REPOS ARG DAOS_LAB_CA_FILE_URL -# script to translate lookup distro names to commonly used names -COPY ./utils/scripts/helpers/distro_info.sh /tmp/distro_info.sh -# script to install OS updates basic tools and maldet -COPY ./utils/scripts/helpers/install_maldet.sh /tmp/install.sh -# script to setup local repo if available -COPY ./utils/scripts/helpers/repo-helper-el8.sh /tmp/repo-helper.sh -RUN set -e; \ - chmod +x /tmp/repo-helper.sh /tmp/install.sh && \ - /tmp/repo-helper.sh +# setup local repo if available and install packages for maldet +RUN --mount=type=bind,source=utils/scripts/helpers/distro_info.sh,target=/tmp/distro_info.sh \ + --mount=type=bind,source=utils/scripts/helpers/install_maldet.sh,target=/tmp/install_maldet.sh \ + --mount=type=bind,source=utils/scripts/helpers/repo-helper-el8.sh,target=/tmp/repo-helper.sh \ + /tmp/repo-helper.sh && /tmp/install_maldet.sh # Add DAOS users ARG UID=1000 -COPY ./utils/scripts/helpers/daos-server-user-setup.sh \ - /tmp/daos-server-user-setup.sh -RUN set -e; \ - chmod +x /tmp/daos-server-user-setup.sh && \ +RUN --mount=type=bind,source=utils/scripts/helpers/daos-server-user-setup.sh,target=/tmp/daos-server-user-setup.sh \ /tmp/daos-server-user-setup.sh # force an upgrade to get any newly built RPMs, but only if CB1 is set. ARG CB1 RUN [ -z "$CB1" ] || { \ - dnf upgrade && \ + dnf upgrade && \ dnf clean all; \ } ARG BUILD_URL -COPY ./utils/scripts/helpers/build_url_repo.sh /tmp/build_url_repo.sh -RUN /tmp/build_url_repo.sh +RUN --mount=type=bind,source=utils/scripts/helpers/distro_info.sh,target=/tmp/distro_info.sh \ + --mount=type=bind,source=utils/scripts/helpers/build_url_repo.sh,target=/tmp/build_url_repo.sh \ + /tmp/build_url_repo.sh diff --git a/ci/docker/Dockerfile.maldet.leap.15 b/ci/docker/Dockerfile.maldet.leap.15 index a083c2e4fcd..037fe5fc9b8 100755 --- a/ci/docker/Dockerfile.maldet.leap.15 +++ b/ci/docker/Dockerfile.maldet.leap.15 @@ -1,5 +1,5 @@ # Copyright 2018-2022 Intel Corporation -# Copyright 2025 Hewlett Packard Enterprise Development LP +# Copyright 2025-2026 Hewlett Packard Enterprise Development LP # All rights reserved. # # 'recipe' for Docker to build an environment scan DAOS for malware. @@ -20,21 +20,16 @@ ARG JENKINS_URL ARG REPOS ARG DAOS_LAB_CA_FILE_URL # script to translate lookup distro names to commonly used names -COPY ./utils/scripts/helpers/distro_info.sh /tmp/distro_info.sh # script to install OS updates basic tools and maldet -COPY ./utils/scripts/helpers/install_maldet.sh /tmp/install.sh # script to setup local repo if available -COPY ./utils/scripts/helpers/repo-helper-leap15.sh /tmp/repo-helper.sh -RUN set -e; \ - chmod +x /tmp/repo-helper.sh /tmp/install.sh && \ - /tmp/repo-helper.sh +RUN --mount=type=bind,source=utils/scripts/helpers/distro_info.sh,target=/tmp/distro_info.sh \ + --mount=type=bind,source=utils/scripts/helpers/install_maldet.sh,target=/tmp/install_maldet.sh \ + --mount=type=bind,source=utils/scripts/helpers/repo-helper-leap15.sh,target=/tmp/repo-helper.sh \ + /tmp/repo-helper.sh && /tmp/install_maldet.sh # Add DAOS users ARG UID=1000 -COPY ./utils/scripts/helpers/daos-server-user-setup.sh \ - /tmp/daos-server-user-setup.sh -RUN set -e; \ - chmod +x /tmp/daos-server-user-setup.sh && \ +RUN --mount=type=bind,source=utils/scripts/helpers/daos-server-user-setup.sh,target=/tmp/daos-server-user-setup.sh \ /tmp/daos-server-user-setup.sh # force an upgrade to get any newly built RPMs, but only if CB1 is set. @@ -45,5 +40,6 @@ RUN [ -z "$CB1" ] || { \ } ARG BUILD_URL -COPY ./utils/scripts/helpers/build_url_repo.sh /tmp/build_url_repo.sh -RUN /tmp/build_url_repo.sh +RUN --mount=type=bind,source=utils/scripts/helpers/distro_info.sh,target=/tmp/distro_info.sh \ + --mount=type=bind,source=utils/scripts/helpers/build_url_repo.sh,target=/tmp/build_url_repo.sh \ + /tmp/build_url_repo.sh diff --git a/utils/docker/Dockerfile.code_scanning b/utils/docker/Dockerfile.code_scanning index a46acf6df55..1daf2e4d193 100644 --- a/utils/docker/Dockerfile.code_scanning +++ b/utils/docker/Dockerfile.code_scanning @@ -1,5 +1,6 @@ # -# Copyright 2018-2022, Intel Corporation +# Copyright 2018-2022 Intel Corporation. +# Copyright 2026 Hewlett Packard Enterprise Development LP # # 'recipe' for Docker for code scanning. # @@ -17,13 +18,10 @@ ARG CB0 ARG REPO_FILE_URL ARG DAOS_LAB_CA_FILE_URL # script to install OS updates basic tools and daos dependencies -# COPY ./utils/scripts/install-fedora.sh /tmp/install.sh # script to setup local repo if available -COPY ./utils/scripts/helpers/repo-helper-fedora.sh /tmp/repo-helper.sh - -RUN chmod +x /tmp/repo-helper.sh && \ - /tmp/repo-helper.sh && \ - rm -f /tmp/repo-helper.sh +RUN --mount=type=bind,source=utils/scripts/helpers/repo-helper-fedora.sh,target=/tmp/repo-helper.sh \ + chmod +x /tmp/repo-helper.sh && \ + /tmp/repo-helper.sh # Install Python Bandit scanner. # The unset commands are currently needed for the combination of running diff --git a/utils/docker/Dockerfile.el.8 b/utils/docker/Dockerfile.el.8 index bf80f3fc122..3e3c1cdb3c3 100644 --- a/utils/docker/Dockerfile.el.8 +++ b/utils/docker/Dockerfile.el.8 @@ -1,6 +1,6 @@ # Copyright 2018-2024 Intel Corporation # Copyright 2025 Google LLC -# Copyright 2025 Hewlett Packard Enterprise Development LP +# Copyright 2025-2026 Hewlett Packard Enterprise Development LP # All rights reserved. # # 'recipe' for Docker to build an image of EL 8 based @@ -33,13 +33,10 @@ RUN echo "no_proxy=${DAOS_NO_PROXY}" >> /etc/environment && \ echo "NO_PROXY=${DAOS_NO_PROXY}" >> /etc/environment # script to install OS updates basic tools and daos dependencies -COPY ./utils/scripts/install-el8.sh /tmp/install.sh # script to setup local repo if available -COPY ./utils/scripts/helpers/repo-helper-el8.sh /tmp/repo-helper.sh - -RUN chmod +x /tmp/repo-helper.sh /tmp/install.sh && \ - /tmp/repo-helper.sh && \ - rm -f /tmp/repo-helper.sh +RUN --mount=type=bind,source=utils/scripts/helpers/repo-helper-el8.sh,target=/tmp/repo-helper.sh \ + chmod +x /tmp/repo-helper.sh && \ + /tmp/repo-helper.sh FROM basic # Install OS updates and package. Include basic tools and daos dependencies @@ -47,17 +44,14 @@ FROM basic # with a local repository, yet needing a proxy to reach outside repositories. # This needs to be moved to a shell script like above in the future to # properly only remove the proxy variables only when they need to be removed -RUN dnf upgrade && \ +RUN --mount=type=bind,source=utils/scripts/install-el8.sh,target=/tmp/install.sh \ + dnf upgrade && \ /tmp/install.sh && \ - dnf clean all && \ - rm -f /tmp/install.sh + dnf clean all # Add DAOS users ARG UID=1000 -COPY ./utils/scripts/helpers/daos-server-user-setup.sh \ - /tmp/daos-server-user-setup.sh -RUN set -e; \ - chmod +x /tmp/daos-server-user-setup.sh && \ +RUN --mount=type=bind,source=utils/scripts/helpers/daos-server-user-setup.sh,target=/tmp/daos-server-user-setup.sh \ /tmp/daos-server-user-setup.sh RUN useradd --no-log-init --user-group --create-home --shell /bin/bash daos_agent RUN echo "daos_agent:daos_agent" | chpasswd @@ -75,9 +69,10 @@ ENV PATH=/home/daos/venv/bin:$PATH ENV VIRTUAL_ENV=/home/daos/venv/ # Install latest versions of python tools. -COPY requirements-build.txt requirements-utest.txt ./ -RUN python3 -m pip --no-cache-dir install --upgrade pip && \ - python3 -m pip --no-cache-dir install -r requirements-build.txt -r requirements-utest.txt +RUN --mount=type=bind,source=requirements-build.txt,target=/tmp/requirements-build.txt \ + --mount=type=bind,source=requirements-utest.txt,target=/tmp/requirements-utest.txt \ + python3 -m pip --no-cache-dir install --upgrade pip && \ + python3 -m pip --no-cache-dir install -r /tmp/requirements-build.txt -r /tmp/requirements-utest.txt WORKDIR /home/daos/pre RUN mkdir -p /home/daos/pre/site_scons/prereq_tools /home/daos/pre/site_scons/components diff --git a/utils/docker/Dockerfile.el.9 b/utils/docker/Dockerfile.el.9 index 82c5287ff26..0331c51a413 100644 --- a/utils/docker/Dockerfile.el.9 +++ b/utils/docker/Dockerfile.el.9 @@ -1,6 +1,6 @@ # Copyright 2022-2024 Intel Corporation # Copyright 2025 Google LLC -# Copyright 2025 Hewlett Packard Enterprise Development LP +# Copyright 2025-2026 Hewlett Packard Enterprise Development LP # All rights reserved. # # 'recipe' for Docker to build an image of EL 9 based @@ -32,29 +32,35 @@ ENV NO_PROXY=${DAOS_NO_PROXY} RUN echo "no_proxy=${DAOS_NO_PROXY}" >> /etc/environment && \ echo "NO_PROXY=${DAOS_NO_PROXY}" >> /etc/environment +ARG BASE_DISTRO +ARG REPO_FILE_URL +ARG JENKINS_URL +ARG REPOS +ARG DAOS_LAB_CA_FILE_URL # script to install OS updates basic tools and daos dependencies -COPY ./utils/scripts/install-el9.sh /tmp/install.sh # script to setup local repo if available COPY ./utils/scripts/helpers/repo-helper-el9.sh /tmp/repo-helper.sh - -RUN chmod +x /tmp/repo-helper.sh /tmp/install.sh && \ - /tmp/repo-helper.sh && \ +RUN chmod +x /tmp/repo-helper.sh && \ + /tmp/repo-helper.sh && \ rm -f /tmp/repo-helper.sh -FROM basic # Install OS updates and package. Include basic tools and daos dependencies RUN dnf upgrade && \ - /tmp/install.sh && \ - dnf clean all && \ + dnf clean all + +# Install OS updates and package. Include basic tools and daos dependencies +COPY ./utils/scripts/install-el9.sh /tmp/install.sh +RUN chmod +x /tmp/install.sh && \ + /tmp/install.sh && \ + dnf clean all && \ rm -f /tmp/install.sh # Add DAOS users ARG UID=1000 -COPY ./utils/scripts/helpers/daos-server-user-setup.sh \ - /tmp/daos-server-user-setup.sh -RUN set -e; \ - chmod +x /tmp/daos-server-user-setup.sh && \ - /tmp/daos-server-user-setup.sh +COPY ./utils/scripts/helpers/daos-server-user-setup.sh /tmp/daos-server-user-setup.sh +RUN chmod +x /tmp/daos-server-user-setup.sh && \ + /tmp/daos-server-user-setup.sh && \ + rm -f /tmp/daos-server-user-setup.sh RUN useradd --no-log-init --user-group --create-home --shell /bin/bash daos_agent RUN echo "daos_agent:daos_agent" | chpasswd @@ -73,7 +79,7 @@ ENV VIRTUAL_ENV=/home/daos/venv/ # Install latest versions of python tools. COPY requirements-build.txt requirements-utest.txt ./ RUN python3 -m pip --no-cache-dir install --upgrade pip && \ - python3 -m pip --no-cache-dir install -r requirements-build.txt -r requirements-utest.txt + python3 -m pip --no-cache-dir install -r requirements-build.txt WORKDIR /home/daos/pre RUN mkdir -p /home/daos/pre/site_scons/prereq_tools /home/daos/pre/site_scons/components @@ -192,3 +198,7 @@ WORKDIR /home/daos ARG DAOS_KEEP_SRC=no # Remove local copy RUN [ "$DAOS_KEEP_SRC" != "no" ] || rm -rf /home/daos/daos /home/daos/pre + +FROM basic AS full +COPY requirements-utest.txt /tmp/requirements-utest.txt +RUN python3 -m pip --no-cache-dir install -r /tmp/requirements-utest.txt diff --git a/utils/docker/Dockerfile.leap.15 b/utils/docker/Dockerfile.leap.15 index 41922a06ffd..f14c5cd053e 100644 --- a/utils/docker/Dockerfile.leap.15 +++ b/utils/docker/Dockerfile.leap.15 @@ -33,13 +33,10 @@ RUN echo "no_proxy=${DAOS_NO_PROXY}" >> /etc/environment && \ echo "NO_PROXY=${DAOS_NO_PROXY}" >> /etc/environment # script to install OS updates basic tools and daos dependencies -COPY ./utils/scripts/install-leap15.sh /tmp/install.sh # script to setup local repo if available -COPY ./utils/scripts/helpers/repo-helper-leap15.sh /tmp/repo-helper.sh - -RUN chmod +x /tmp/repo-helper.sh /tmp/install.sh && \ - /tmp/repo-helper.sh && \ - rm -f /tmp/repo-helper.sh +RUN --mount=type=bind,source=utils/scripts/helpers/repo-helper-leap15.sh,target=/tmp/repo-helper.sh \ + chmod +x /tmp/repo-helper.sh && \ + /tmp/repo-helper.sh FROM basic # Install OS updates and package. Include basic tools and daos dependencies @@ -48,10 +45,10 @@ FROM basic # with a local repository, yet needing a proxy to reach outside repositories. # This needs to be moved to a shell script like above in the future to # properly only remove the proxy variables only when they need to be removed -RUN dnf upgrade && \ +RUN --mount=type=bind,source=utils/scripts/install-leap15.sh,target=/tmp/install.sh \ + dnf upgrade && \ /tmp/install.sh && \ - dnf clean all && \ - rm -f /tmp/install.sh + dnf clean all # According to https://pkgs.org/search/?q=lua-lmod, Leap 15.6 only has lua-lmod-8.7.34. # This version has a problem with loading modules, as described in https://github.com/TACC/Lmod/issues/687. @@ -69,10 +66,7 @@ RUN if [ -n "$JENKINS_URL" ]; then \ # Add DAOS users ARG UID=1000 -COPY ./utils/scripts/helpers/daos-server-user-setup.sh \ - /tmp/daos-server-user-setup.sh -RUN set -e; \ - chmod +x /tmp/daos-server-user-setup.sh && \ +RUN --mount=type=bind,source=utils/scripts/helpers/daos-server-user-setup.sh,target=/tmp/daos-server-user-setup.sh \ /tmp/daos-server-user-setup.sh RUN useradd --no-log-init --user-group --create-home --shell /bin/bash daos_agent RUN echo "daos_agent:daos_agent" | chpasswd @@ -90,9 +84,10 @@ ENV PATH=/home/daos/venv/bin:$PATH ENV VIRTUAL_ENV=/home/daos/venv/ # Install latest versions of python tools. -COPY requirements-build.txt requirements-utest.txt ./ -RUN python3 -m pip --no-cache-dir install --upgrade pip && \ - python3 -m pip --no-cache-dir install -r requirements-build.txt -r requirements-utest.txt +RUN --mount=type=bind,source=requirements-build.txt,target=/tmp/requirements-build.txt \ + --mount=type=bind,source=requirements-utest.txt,target=/tmp/requirements-utest.txt \ + python3 -m pip --no-cache-dir install --upgrade pip && \ + python3 -m pip --no-cache-dir install -r /tmp/requirements-build.txt -r /tmp/requirements-utest.txt WORKDIR /home/daos/pre RUN mkdir -p /home/daos/pre/site_scons/prereq_tools /home/daos/pre/site_scons/components diff --git a/utils/docker/Dockerfile.ubuntu b/utils/docker/Dockerfile.ubuntu index 589c3f3dd9d..e1cbbb656f6 100644 --- a/utils/docker/Dockerfile.ubuntu +++ b/utils/docker/Dockerfile.ubuntu @@ -1,6 +1,6 @@ # Copyright 2018-2024 Intel Corporation # Copyright 2025 Google LLC -# Copyright 2025 Hewlett Packard Enterprise Development LP +# Copyright 2025-2026 Hewlett Packard Enterprise Development LP # All rights reserved. # # 'recipe' for Docker to build an image of Ubuntu-based environment for building the DAOS project. @@ -33,12 +33,10 @@ RUN echo "no_proxy=${DAOS_NO_PROXY}" >> /etc/environment && \ echo "NO_PROXY=${DAOS_NO_PROXY}" >> /etc/environment # script to setup local repo if available and install packages -COPY ./utils/scripts/helpers/repo-helper-ubuntu.sh /tmp/repo-helper.sh -COPY ./utils/scripts/install-ubuntu.sh /tmp/install.sh - -RUN chmod +x /tmp/repo-helper.sh /tmp/install.sh && \ - /tmp/repo-helper.sh && \ - rm -f /tmp/repo-helper.sh /tmp/install.sh +RUN --mount=type=bind,source=utils/scripts/helpers/repo-helper-ubuntu.sh,target=/tmp/repo-helper.sh \ + --mount=type=bind,source=utils/scripts/install-ubuntu.sh,target=/tmp/install.sh \ + chmod +x /tmp/repo-helper.sh /tmp/install.sh && \ + /tmp/repo-helper.sh RUN locale-gen en_US.UTF-8 @@ -48,10 +46,7 @@ RUN update-alternatives --set \ # Add DAOS users ARG UID=5000 -COPY ./utils/scripts/helpers/daos-server-user-setup.sh \ - /tmp/daos-server-user-setup.sh -RUN set -e; \ - chmod +x /tmp/daos-server-user-setup.sh && \ +RUN --mount=type=bind,source=utils/scripts/helpers/daos-server-user-setup.sh,target=/tmp/daos-server-user-setup.sh \ /tmp/daos-server-user-setup.sh RUN useradd --no-log-init --user-group --create-home --shell /bin/bash daos_agent RUN echo "daos_agent:daos_agent" | chpasswd @@ -69,9 +64,10 @@ ENV PATH=/home/daos/venv/bin:$PATH ENV VIRTUAL_ENV=/home/daos/venv/ # Install latest versions of python tools. -COPY requirements-build.txt requirements-utest.txt ./ -RUN python3 -m pip --no-cache-dir install --upgrade pip && \ - python3 -m pip --no-cache-dir install -r requirements-build.txt -r requirements-utest.txt +RUN --mount=type=bind,source=requirements-build.txt,target=/tmp/requirements-build.txt \ + --mount=type=bind,source=requirements-utest.txt,target=/tmp/requirements-utest.txt \ + python3 -m pip --no-cache-dir install --upgrade pip && \ + python3 -m pip --no-cache-dir install -r /tmp/requirements-build.txt -r /tmp/requirements-utest.txt WORKDIR /home/daos/pre RUN mkdir -p /home/daos/pre/site_scons/prereq_tools /home/daos/pre/site_scons/components diff --git a/utils/scripts/helpers/repo-helper-el9.sh b/utils/scripts/helpers/repo-helper-el9.sh old mode 100644 new mode 100755