diff --git a/Jenkinsfile b/Jenkinsfile index dd086e88a0e..674f8ef7bb9 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -21,6 +21,7 @@ import groovy.transform.Field // To use a test branch (i.e. PR) until it lands to master // I.e. for testing library changes //@Library(value='pipeline-lib@your_branch') _ +@Library(value='pipeline-lib@hendersp/DAOS-19247-fix') _ /* groovylint-disable-next-line CompileStatic */ job_status_internal = [:] @@ -469,19 +470,6 @@ String vm9_label(String distro) { def_val: params.FUNCTIONAL_VM_LABEL)) } -void rpm_test_post(String stageName, String node) { - // Extract first node from comma-delimited list - String firstNode = node.split(',')[0].trim() - sh label: 'Fetch and stage artifacts', - script: 'hostname; ssh -i ci_key jenkins@' + firstNode + - ' ls -ltar /tmp; mkdir -p "' + env.STAGE_NAME + '/" && ' + - 'scp -i ci_key jenkins@' + firstNode + - ':/tmp/{{suite_dmg,daos_{server_helper,{control,agent}}}.log,daos_server.log.*} "' + - stageName + '/"' - archiveArtifacts artifacts: env.STAGE_NAME + '/**' - job_status_update() -} - String sconsArgs() { if (!params.CI_SCONS_ARGS) { return sconsFaultsArgs() @@ -1175,262 +1163,143 @@ pipeline { beforeAgent true expression { shouldStageRun('Test') } } - parallel { - stage('Functional on EL 8.8 with Valgrind') { - when { - beforeAgent true - expression { shouldStageRun('Functional on EL 8.8 with Valgrind') } - } - agent { - label vm9_label('EL8') - } - steps { - job_step_update( - functionalTest( - inst_repos: daosRepos(), - inst_rpms: functionalPackages(1, next_version(), 'tests-internal') + - ' mercury-libfabric', - test_function: 'runTestFunctionalV2')) - } - post { - always { - functionalTestPostV2() - job_status_update() - } - } - } // stage('Functional on EL 8.8 with Valgrind') - stage('Functional on EL 8') { - when { - beforeAgent true - expression { shouldStageRun('Functional on EL 8') } - } - agent { - label vm9_label('EL8') - } - steps { - job_step_update( - functionalTest( - inst_repos: daosRepos(), - inst_rpms: functionalPackages(1, next_version(), 'tests-internal') + - ' mercury-libfabric', - test_function: 'runTestFunctionalV2', - image_version: 'el8.10')) - } - post { - always { - functionalTestPostV2() - job_status_update() - } - } - } // stage('Functional on EL 8') - stage('Functional on EL 9') { - when { - beforeAgent true - expression { shouldStageRun('Functional on EL 9') } - } - agent { - label vm9_label('EL9') - } - steps { - job_step_update( - functionalTest( - inst_repos: daosRepos(), - inst_rpms: functionalPackages(1, next_version(), 'tests-internal') + - ' mercury-libfabric', - test_function: 'runTestFunctionalV2', - image_version: 'el9.7')) - } - post { - always { - functionalTestPostV2() - job_status_update() - } - } - } // stage('Functional on EL 9') - stage('Functional on Leap 15') { - when { - beforeAgent true - expression { shouldStageRun('Functional on Leap 15') } - } - agent { - label vm9_label('Leap15') - } - steps { - job_step_update( - functionalTest( - inst_repos: daosRepos(), - inst_rpms: functionalPackages(1, next_version(), 'tests-internal') + - ' mercury-libfabric', - test_function: 'runTestFunctionalV2', - image_version: 'leap15.6')) - } - post { - always { - functionalTestPostV2() - job_status_update() - } - } // post - } // stage('Functional on Leap 15') - stage('Functional on SLES 15') { - when { - beforeAgent true - expression { shouldStageRun('Functional on SLES 15') } - } - agent { - label vm9_label('Leap15') - } - steps { - job_step_update( - functionalTest( - inst_repos: daosRepos(), - inst_rpms: functionalPackages(1, next_version(), 'tests-internal') + - ' mercury-libfabric', - test_function: 'runTestFunctionalV2', - image_version: 'sles15.7')) - } - post { - always { - functionalTestPostV2() - job_status_update() - } - } // post - } // stage('Functional on SLES 15') - stage('Functional on Ubuntu 20.04') { - when { - beforeAgent true - expression { shouldStageRun('Functional on Ubuntu 20.04') } - } - agent { - label vm9_label('Ubuntu') - } - steps { - job_step_update( - functionalTest( - inst_repos: daosRepos(), - inst_rpms: functionalPackages(1, next_version(), 'tests-internal') + - ' mercury-libfabric', - test_function: 'runTestFunctionalV2')) - } - post { - always { - functionalTestPostV2() - job_status_update() - } - } // post - } // stage('Functional on Ubuntu 20.04') - stage('Fault injection testing') { - when { - beforeAgent true - expression { shouldStageRun('Fault injection testing') } - } - agent { - label params.CI_FI_1_LABEL - } - steps { - job_step_update( - unitTest(timeout_time: 240, - inst_repos: daosRepos(), - test_script: 'ci/unit/test_nlt.sh --memcheck no' + - ' --system-ram-reserved 4 --server-debug WARN' + - ' --log-usage-import nltr.json' + - ' --log-usage-save nltr.xml' + - ' --class-name fault-injection fi', - with_valgrind: '', - always_script: 'ci/unit/test_nlt_post.sh', - testResults: 'nlt-junit.xml', - unstash_opt: true, - unstash_tests: false, - inst_rpms: unitPackages(target: 'el9') + ' daos-client-tests', - image_version: 'el9.7', - prov_env_vars: 'VM_CPUS=14')) - } - post { - always { - unitTestPost artifacts: ['nlt_logs/'], - testResults: 'nlt-junit.xml', - with_valgrind: '', - FI: true - archiveArtifacts artifacts: 'nlt_logs/fault-injection/', - allowEmptyArchive: true - job_status_update() - } - } - } // stage('Fault injection testing') - stage('Test RPMs on EL 9.6') { - when { - beforeAgent true - expression { shouldStageRun('Test RPMs on EL 9.6') } - } - agent { - label params.CI_UNIT_VM1_LABEL - } - steps { - job_step_update( - testRpm(inst_repos: daosRepos(), - daos_pkg_version: daosPackagesVersion(next_version()), - inst_rpms: 'mercury-libfabric') - ) - } - post { - always { - rpm_test_post(env.STAGE_NAME, env.NODELIST) - } - } - } // stage('Test RPMs on EL 9.6') - stage('Test RPMs on Leap 15.5') { - when { - beforeAgent true - expression { shouldStageRun('Test RPMs on Leap 15.5') } - } - agent { - label params.CI_UNIT_VM1_LABEL - } - steps { - /* neither of these work as FTest strips the first node - out of the pool requiring 2 node clusters at minimum - * additionally for this use-case, can't override - ftest_arg with this :-( - script { - 'Test RPMs on Leap 15.5': getFunctionalTestStage( - name: 'Test RPMs on Leap 15.5', - pragma_suffix: '', - label: params.CI_UNIT_VM1_LABEL, - next_version: next_version(), - stage_tags: '', - default_tags: 'test_daos_management', - nvme: 'auto', - run_if_pr: true, - run_if_landing: true, - job_status: job_status_internal - ) - } - job_step_update( - functionalTest( - test_tag: 'test_daos_management', - ftest_arg: '--yaml_extension single_host', - inst_repos: daosRepos(), - inst_rpms: functionalPackages(1, next_version(), 'tests-internal'), - test_function: 'runTestFunctionalV2')) - } - post { - always { - functionalTestPostV2() - job_status_update() - } - } */ - job_step_update( - testRpm(inst_repos: daosRepos(), - daos_pkg_version: daosPackagesVersion(next_version()), - inst_rpms: 'mercury-libfabric') + steps { + script { + parallel( + 'Functional on EL 8.8 with Valgrind': getFunctionalTestStage( + name: 'Functional on EL 8.8 with Valgrind', + runStage: shouldStageRun('Functional on EL 8.8 with Valgrind'), + pragma_suffix: '-vm', + label: vm9_label('EL8'), + next_version: next_version(), + other_packages: 'mercury-libfabric', + stage_tags: 'vm', + default_tags: 'memcheck', + nvme: 'auto', + job_status: job_status_internal, + image_version: 'el8.8' + ), + 'Functional on EL 8': getFunctionalTestStage( + name: 'Functional on EL 8', + runStage: shouldStageRun('Functional on EL 8'), + pragma_suffix: '-vm', + label: vm9_label('EL8'), + next_version: next_version(), + other_packages: 'mercury-libfabric', + stage_tags: 'vm', + default_tags: startedByTimer() ? 'pr daily_regression' : 'pr', + nvme: 'auto', + job_status: job_status_internal, + image_version: 'el8.10' + ), + 'Functional on EL 9': getFunctionalTestStage( + name: 'Functional on EL 9', + runStage: shouldStageRun('Functional on EL 9'), + pragma_suffix: '-vm', + label: vm9_label('EL9'), + next_version: next_version(), + other_packages: 'mercury-libfabric', + stage_tags: 'vm', + default_tags: startedByTimer() ? 'pr daily_regression' : 'pr', + nvme: 'auto', + job_status: job_status_internal, + image_version: 'el9.7' + ), + 'Functional on Leap 15': getFunctionalTestStage( + name: 'Functional on Leap 15', + runStage: shouldStageRun('Functional on Leap 15'), + pragma_suffix: '-vm', + label: vm9_label('Leap15'), + next_version: next_version(), + other_packages: 'mercury-libfabric', + stage_tags: 'vm', + default_tags: startedByTimer() ? 'pr daily_regression' : 'pr', + nvme: 'auto', + job_status: job_status_internal, + image_version: 'leap15.6' + ), + 'Functional on SLES 15': getFunctionalTestStage( + name: 'Functional on SLES 15', + runStage: shouldStageRun('Functional on SLES 15'), + pragma_suffix: '-vm', + label: vm9_label('Leap15'), + next_version: next_version(), + other_packages: 'mercury-libfabric', + stage_tags: 'vm', + default_tags: startedByTimer() ? 'pr daily_regression' : 'pr', + nvme: 'auto', + job_status: job_status_internal, + image_version: 'sles15.7' + ), + 'Functional on Ubuntu 20.04': getFunctionalTestStage( + name: 'Functional on Ubuntu 20.04', + runStage: shouldStageRun('Functional on Ubuntu 20.04'), + pragma_suffix: '-vm', + label: vm9_label('Ubuntu'), + next_version: next_version(), + other_packages: 'mercury-libfabric', + stage_tags: 'vm', + default_tags: startedByTimer() ? 'pr daily_regression' : 'pr', + nvme: 'auto', + job_status: job_status_internal + ), + 'Fault injection testing': scriptedUnitTestStage( + name: 'Fault injection testing', + runStage: shouldStageRun('Fault injection testing'), + label: params.CI_FI_1_LABEL, + jobStatus: job_status_internal, + distro: 'el9', + timeoutTime: 240, + testScript: 'ci/unit/test_nlt.sh --memcheck no' + + ' --system-ram-reserved 4 --server-debug WARN' + + ' --log-usage-import nltr.json' + + ' --log-usage-save nltr.xml' + + ' --class-name fault-injection fi', + withValgrind: '', + alwaysScript: 'ci/unit/test_nlt_post.sh', + testResults: 'nlt-junit.xml', + unstashOpt: true, + unstashTests: false, + imageVersion: 'el9.7', + provEnvVars: 'VM_CPUS=14', + unitTestPostArgs: [ + artifacts: ['nlt_logs/'], + testResults: 'nlt-junit.xml', + with_valgrind: '', + FI: true], + archiveArtifactsArgs: [ + artifacts: 'nlt_logs/fault-injection/', + allowEmptyArchive: true] + ), + 'Test RPMs on EL 9.6': scriptedTestRpmStage( + name: 'Test RPMs on EL 9.6', + runStage: shouldStageRun('Test RPMs on EL 9.6'), + label: params.CI_UNIT_VM1_LABEL, + jobStatus: job_status_internal, + nextVersion: next_version(), + imageVersion: 'el9.6', + instRpms: 'mercury-libfabric', + nodeList: env.NODELIST, + alwaysScript: 'ci/rpm/test_daos_post.sh \'Test RPMs on EL 9.6\'', + archiveArtifactsArgs: [ + artifacts: 'Test RPMs on EL 9.6/'] + ), + 'Test RPMs on Leap 15.5': scriptedTestRpmStage( + name: 'Test RPMs on Leap 15.5', + runStage: shouldStageRun('Test RPMs on Leap 15.5'), + label: params.CI_UNIT_VM1_LABEL, + jobStatus: job_status_internal, + nextVersion: next_version(), + imageVersion: 'leap15.5', + instRpms: 'mercury-libfabric', + nodeList: env.NODELIST, + alwaysScript: 'ci/rpm/test_daos_post.sh \'Test RPMs on Leap 15.5\'', + archiveArtifactsArgs: [ + artifacts: 'Test RPMs on Leap 15.5/'] ) - } - post { - always { - rpm_test_post(env.STAGE_NAME, env.NODELIST) - } - } - } // stage('Test RPMs on Leap 15.5') - } // parallel + ) + } + } } // stage('Test') stage('Test Storage Prep on EL 8.8') { when { diff --git a/ci/rpm/test_daos.sh b/ci/rpm/test_daos.sh index 6e1f415d237..0f732aa140e 100755 --- a/ci/rpm/test_daos.sh +++ b/ci/rpm/test_daos.sh @@ -11,7 +11,11 @@ mydir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" # shellcheck disable=SC2029 ssh "$SSH_KEY_ARGS" jenkins@"${nodelist[0]}" \ - "NODE=${nodelist[0]} \ - DAOS_PKG_VERSION=$DAOS_PKG_VERSION \ - PYTHON_VERSION=\"${PYTHON_VERSION}\" \ - $(cat "$mydir/test_daos_node.sh")" + "NODE=${nodelist[0]} \ + DAOS_PKG_VERSION=$DAOS_PKG_VERSION \ + PYTHON_VERSION=${PYTHON_VERSION} \ + bash -s 2>&1 | tee /tmp/test_daos_rpms.log" \ + < "$mydir/test_daos_node.sh" + + # bash -s > /tmp/test_daos_rpms.log 2>&1" + # < "$mydir/test_daos_node.sh" diff --git a/ci/rpm/test_daos_node.sh b/ci/rpm/test_daos_node.sh index ce16704c884..4cc9728d018 100755 --- a/ci/rpm/test_daos_node.sh +++ b/ci/rpm/test_daos_node.sh @@ -29,6 +29,9 @@ if [ -n "$DAOS_PKG_VERSION" ]; then fi set -uex + +echo "${PRETTY_NAME:-Unknown OS}" + sudo $YUM -y install daos-client"$DAOS_PKG_VERSION" if rpm -q daos-server; then echo "daos-server RPM should not be installed as a dependency of daos-client" diff --git a/ci/rpm/test_daos_post.sh b/ci/rpm/test_daos_post.sh new file mode 100755 index 00000000000..1ab04d9d63f --- /dev/null +++ b/ci/rpm/test_daos_post.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +# This is a post test processing script for post processing the +# run_utest.py stage CI run + +set -uex + +test_log_dir="${1:-}" +first_node="${NODELIST%%,*}" + +if [ -z "${test_log_dir}" ]; then + echo "test_daos_post: The test log directory argument is missing!" + exit 1 +fi + +# Copy logs from test_daos_node.sh execution +rm -rf "${test_log_dir}" +mkdir -p "${test_log_dir}/configs" +rsync -v -dpt -z -e "ssh ${SSH_KEY_ARGS}" jenkins@"${first_node}":/etc/daos/ \ + --filter="include daos_*.yml*" \ + --filter="exclude *" "${test_log_dir}/configs/" || true +mkdir -p "${test_log_dir}/logs" +rsync -v -dpt -z -e "ssh ${SSH_KEY_ARGS}" jenkins@"${first_node}":/tmp/ \ + --filter="include suite_dmg.log" \ + --filter="include daos_*.log*" \ + --filter="exclude *" "${test_log_dir}/logs/" || true +rsync -v -dpt -z -e "ssh ${SSH_KEY_ARGS}" jenkins@"${first_node}":/tmp/ \ + --filter="include test_daos_rpms.log" \ + --filter="exclude *" "${test_log_dir}/" || true