diff --git a/Jenkinsfile b/Jenkinsfile index dd086e88a0e..270e3ba8c44 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -859,7 +859,7 @@ pipeline { scons_args: sconsArgs() + ' PREFIX=/opt/daos TARGET_TYPE=release')) sh label: 'Generate RPMs', - script: './ci/rpm/gen_rpms.sh el8 "' + env.DAOS_RELVAL + '"' + script: './ci/rpm/gen_rpms.sh el8 "' + env.DAOS_RELVAL + '" "' + env.PYTHON_VERSION + '"' } } post { @@ -913,7 +913,7 @@ pipeline { scons_args: sconsArgs() + ' PREFIX=/opt/daos TARGET_TYPE=release')) sh label: 'Generate RPMs', - script: './ci/rpm/gen_rpms.sh el9 "' + env.DAOS_RELVAL + '"' + script: './ci/rpm/gen_rpms.sh el9 "' + env.DAOS_RELVAL + '" "' + env.PYTHON_VERSION + '"' // For non-release builds, create a separate build with the valgrind // tag for NLT memcheck testing. This is necessary to avoid problems // caused by valgrind being confused by the Go runtime. We don't want @@ -981,7 +981,7 @@ pipeline { scons_args: sconsArgs() + ' PREFIX=/opt/daos TARGET_TYPE=release')) sh label: 'Generate RPMs', - script: './ci/rpm/gen_rpms.sh suse.lp156 "' + env.DAOS_RELVAL + '"' + script: './ci/rpm/gen_rpms.sh suse.lp156 "' + env.DAOS_RELVAL + '" "' + env.PYTHON_VERSION + '"' } } post { diff --git a/ci/rpm/gen_rpms.sh b/ci/rpm/gen_rpms.sh index 30a4e248952..c9cb57973ff 100755 --- a/ci/rpm/gen_rpms.sh +++ b/ci/rpm/gen_rpms.sh @@ -21,6 +21,7 @@ env pushd "${mydir}/../.." || exit 1 export DISTRO="${1}" export DAOS_RELVAL="${2}" +export PYTHON_VERSION="${3}" rm -f ./*.rpm rm -rf /home/daos/rpms/* utils/rpms/build_packages.sh deps diff --git a/ci/rpm/test_daos_node.sh b/ci/rpm/test_daos_node.sh index ce16704c884..9b688346f2d 100755 --- a/ci/rpm/test_daos_node.sh +++ b/ci/rpm/test_daos_node.sh @@ -24,6 +24,8 @@ SERVER_CONFIG="/etc/daos/daos_server.yml" AGENT_CONFIG="/etc/daos/daos_agent.yml" CONTROL_CONFIG="/etc/daos/daos_control.yml" +: "${PYTHON_VERSION:=3.11}" + if [ -n "$DAOS_PKG_VERSION" ]; then DAOS_PKG_VERSION="-${DAOS_PKG_VERSION}" fi @@ -35,6 +37,30 @@ if rpm -q daos-server; then exit 1 fi +# Verify daos command works +if ! daos version; then + echo "Error checking daos command version" + exit 1 +fi + +# Verify daos_agent command works +if ! daos_agent version; then + echo "Error checking daos_agent command version" + exit 1 +fi + +# Verify the python dependency matches what pydaos is packaged with +if ! rpm -qR daos-client | grep "python${PYTHON_VERSION}"; then + echo "daos-client should depend on python${PYTHON_VERSION}" + exit 1 +fi + +# Verify pydaos can be imported +if ! "python${PYTHON_VERSION}" -c "import pydaos"; then + echo "Should be able to import pydaos after installing daos-client" + exit 1 +fi + if ! sudo $YUM -y history undo last; then echo "Error trying to undo previous dnf transaction" $YUM history @@ -132,7 +158,6 @@ sudo chown "$me:$me" /tmp/daos_sockets FTEST=/usr/lib/daos/TESTING/ftest -: "${PYTHON_VERSION:=3.11}" "python${PYTHON_VERSION}" -m venv venv # shellcheck disable=SC1091 source venv/bin/activate diff --git a/utils/docker/Dockerfile.el.9 b/utils/docker/Dockerfile.el.9 index ac1a99798fc..d86b814edb9 100644 --- a/utils/docker/Dockerfile.el.9 +++ b/utils/docker/Dockerfile.el.9 @@ -118,10 +118,11 @@ COPY --chown=daos_server:daos_server utils/sl utils/sl # Build third party RPMs RUN [ "$DAOS_PACKAGES_BUILD" != "yes" ] || [ "$DAOS_DEPS_BUILD" != "yes" ] || { \ - export DISTRO="el9" && \ - utils/rpms/build_packages.sh deps && \ - mkdir -p /home/daos/rpms && \ - mv *.rpm /home/daos/rpms; \ + export DISTRO="el9" && \ + export PYTHON_VERSION=${PYTHON_VERSION} && \ + utils/rpms/build_packages.sh deps && \ + mkdir -p /home/daos/rpms && \ + mv *.rpm /home/daos/rpms; \ } USER root:root @@ -166,10 +167,11 @@ COPY --chown=daos_server:daos_server utils utils # Build DAOS RPMs RUN [ "$DAOS_PACKAGES_BUILD" != "yes" ] || [ "$DAOS_BUILD" != "yes" ] || { \ - export DISTRO="el9" && \ - utils/rpms/build_packages.sh daos && \ - mkdir -p /home/daos/rpms && \ - cp *.rpm /home/daos/rpms; \ + export DISTRO="el9" && \ + export PYTHON_VERSION=${PYTHON_VERSION} && \ + utils/rpms/build_packages.sh daos && \ + mkdir -p /home/daos/rpms && \ + cp *.rpm /home/daos/rpms; \ } # Set environment variables diff --git a/utils/docker/Dockerfile.leap.15 b/utils/docker/Dockerfile.leap.15 index a9c2129d9f2..667f5f01af9 100644 --- a/utils/docker/Dockerfile.leap.15 +++ b/utils/docker/Dockerfile.leap.15 @@ -138,10 +138,11 @@ COPY --chown=daos_server:daos_server utils/sl utils/sl # Build third party RPMs RUN [ "$DAOS_PACKAGES_BUILD" != "yes" ] || [ "$DAOS_DEPS_BUILD" != "yes" ] || { \ - export DISTRO="suse.lp15${POINT_RELEASE#\.}" && \ - utils/rpms/build_packages.sh deps && \ - mkdir -p /home/daos/rpms && \ - mv *.rpm /home/daos/rpms; \ + export DISTRO="suse.lp15${POINT_RELEASE#\.}" && \ + export PYTHON_VERSION=${PYTHON_VERSION} && \ + utils/rpms/build_packages.sh deps && \ + mkdir -p /home/daos/rpms && \ + mv *.rpm /home/daos/rpms; \ } USER root:root @@ -195,10 +196,11 @@ COPY --chown=daos_server:daos_server utils utils # Build DAOS RPMs RUN [ "$DAOS_PACKAGES_BUILD" != "yes" ] || [ "$DAOS_BUILD" != "yes" ] || { \ - export DISTRO="suse.lp15${POINT_RELEASE#\.}" && \ - utils/rpms/build_packages.sh daos && \ - mkdir -p /home/daos/rpms && \ - cp *.rpm /home/daos/rpms; \ + export DISTRO="suse.lp15${POINT_RELEASE#\.}" && \ + export PYTHON_VERSION=${PYTHON_VERSION} && \ + utils/rpms/build_packages.sh daos && \ + mkdir -p /home/daos/rpms && \ + cp *.rpm /home/daos/rpms; \ } # Set environment variables diff --git a/utils/docker/Dockerfile.ubuntu b/utils/docker/Dockerfile.ubuntu index b65e3eddad8..6331350d441 100644 --- a/utils/docker/Dockerfile.ubuntu +++ b/utils/docker/Dockerfile.ubuntu @@ -116,6 +116,7 @@ RUN [ "$DAOS_PACKAGES_BUILD" != "yes" ] || [ "$DAOS_DEPS_BUILD" != "yes" ] || { export DISTRO=ubuntu && \ export OUTPUT_TYPE=deb && \ export LIBDIR=/usr/lib/x86_64-linux-gnu && \ + export PYTHON_VERSION=${PYTHON_VERSION} && \ utils/rpms/build_packages.sh deps && \ mkdir -p /home/daos/dpkg && \ cp *.deb /home/daos/dpkg; \ @@ -164,6 +165,7 @@ RUN [ "$DAOS_PACKAGES_BUILD" != "yes" ] || [ "$DAOS_BUILD" != "yes" ] || { \ export DISTRO=ubuntu && \ export OUTPUT_TYPE=deb && \ export LIBDIR=/usr/lib/x86_64-linux-gnu && \ + export PYTHON_VERSION=${PYTHON_VERSION} && \ utils/rpms/build_packages.sh daos && \ mkdir -p /home/daos/dpkg && \ cp *.deb /home/daos/dpkg; \ diff --git a/utils/rpms/build_packages.sh b/utils/rpms/build_packages.sh index 6c72cfd52e6..79b2515cc2a 100755 --- a/utils/rpms/build_packages.sh +++ b/utils/rpms/build_packages.sh @@ -1,7 +1,11 @@ #!/bin/bash -set -eEuo pipefail +set -eEuox pipefail + +: "${PYTHON_VERSION:=}" + build_type="${1:-all}" source utils/sl/setup_local.sh +echo "PYTHON_VERSION=$PYTHON_VERSION" if [[ "${build_type}" =~ deps|all ]]; then utils/rpms/argobots.sh utils/rpms/fused.sh @@ -13,5 +17,6 @@ if [[ "${build_type}" =~ deps|all ]]; then utils/rpms/daos-spdk.sh fi if [[ "${build_type}" =~ daos|all ]]; then + PYTHON_VERSION=\"${PYTHON_VERSION}\" \ utils/rpms/daos.sh fi diff --git a/utils/rpms/daos.sh b/utils/rpms/daos.sh index a71991cc525..d2b920f15ab 100755 --- a/utils/rpms/daos.sh +++ b/utils/rpms/daos.sh @@ -38,6 +38,8 @@ URL="https://daos.io" RPM_CHANGELOG="daos.changelog" +: "${PYTHON_VERSION:=}" + # Some extra "install" steps # daos package files=() @@ -80,6 +82,7 @@ EXTRA_OPTS+=("--after-install" "${tmp}/post_install_daos") DEPENDS=( "mercury >= ${mercury_version}" ) DEPENDS+=( "${isal_lib} >= ${isal_version}" ) DEPENDS+=( "${isal_crypto_lib} >= ${isal_crypto_version}" ) +EXTERNAL_DEPENDS=("${protobufc_lib}") build_package "daos" # Only build server RPMs if we built the server @@ -331,6 +334,7 @@ EOF fi EXTERNAL_DEPENDS=("fuse3") +EXTERNAL_DEPENDS+=("python${PYTHON_VERSION}") DEPENDS=("daos = ${VERSION}-${RELEASE}") build_package "daos-client" @@ -379,8 +383,7 @@ append_install_list "${files[@]}" CONFIG_FILES+=("${TARGET_PATH}/fault-inject-cart.yaml") #todo add external depends -EXTERNAL_DEPENDS=("${protobufc_lib}") -EXTERNAL_DEPENDS+=("fio") +EXTERNAL_DEPENDS=("fio") EXTERNAL_DEPENDS+=("git") EXTERNAL_DEPENDS+=("dbench") EXTERNAL_DEPENDS+=("lbzip2") diff --git a/utils/sl/setup_local.sh b/utils/sl/setup_local.sh index 25b19a89483..c6525c16dca 100644 --- a/utils/sl/setup_local.sh +++ b/utils/sl/setup_local.sh @@ -42,28 +42,23 @@ then SL_PREFIX="$(pwd)/install" fi +# Export PYTHONPATH if a valid python installation is found function export_pythonpath() { - MAJOR="${1}" - MINOR="$(python3 -c 'import sys; print(sys.version_info.minor)')" - VERSION="${MAJOR}.${MINOR}" - if [ "${MAJOR}" -eq 3 ]; then - PYTHONPATH=${SL_PREFIX}/lib64/python${VERSION}/site-packages:${PYTHONPATH:-} - else - echo "unknown Python version: ${VERSION}" + # Default to PYTHON_VERSION to be compatible with packaging scripts + local python_version="${PYTHON_VERSION:=3}" + local python_cmd="python${python_version}" + if [ ! -x "$(command -v $python_cmd)" ]; then + echo "unknown Python version: ${python_version}" return 0 fi - export PYTHONPATH + local major="$($python_cmd -c 'import sys; print(sys.version_info.major)')" + local minor="$($python_cmd -c 'import sys; print(sys.version_info.minor)')" + python_version="${major}.${minor}" + export PYTHONPATH=${SL_PREFIX}/lib64/python${python_version}/site-packages:${PYTHONPATH:-} } - -# look for a valid installation of python -if [ -x "$(command -v python3)" ]; then - PYTHON_VERSION="$(python3 -c 'import sys; print(sys.version_info.major)')" - export_pythonpath "${PYTHON_VERSION}" -else - echo "python3 not found" -fi +export_pythonpath function in_list() {