From 2e157fab922ea3af00bf4b5c90fcb54d0086083c Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Wed, 1 Jul 2026 14:02:51 -0400 Subject: [PATCH 01/11] DAOS-19248 test: Fix no tag match for Functional VM stages Ensure Functional VM stages are skipped if no test tags match. Test-tag: test_always_passes_hw Skip-functional-on-el-8: false Skip-functional-on-el-9: false Skip-functional-on-leap-15: false Skip-functional-on-sles-15: false Signed-off-by: Phil Henderson --- Jenkinsfile | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index dd086e88a0e..c918985e726 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1179,7 +1179,8 @@ pipeline { stage('Functional on EL 8.8 with Valgrind') { when { beforeAgent true - expression { shouldStageRun('Functional on EL 8.8 with Valgrind') } + expression { shouldStageRun('Functional on EL 8.8 with Valgrind') && + !testsInStage(getFunctionalTags('-vm', 'vm', 'pr')) } } agent { label vm9_label('EL8') @@ -1202,7 +1203,8 @@ pipeline { stage('Functional on EL 8') { when { beforeAgent true - expression { shouldStageRun('Functional on EL 8') } + expression { shouldStageRun('Functional on EL 8') && + !testsInStage(getFunctionalTags('-vm', 'vm', 'pr')) } } agent { label vm9_label('EL8') @@ -1226,7 +1228,8 @@ pipeline { stage('Functional on EL 9') { when { beforeAgent true - expression { shouldStageRun('Functional on EL 9') } + expression { shouldStageRun('Functional on EL 9') && + !testsInStage(getFunctionalTags('-vm', 'vm', 'pr'))} } agent { label vm9_label('EL9') @@ -1250,7 +1253,8 @@ pipeline { stage('Functional on Leap 15') { when { beforeAgent true - expression { shouldStageRun('Functional on Leap 15') } + expression { shouldStageRun('Functional on Leap 15') && + !testsInStage(getFunctionalTags('-vm', 'vm', 'pr'))} } agent { label vm9_label('Leap15') @@ -1274,7 +1278,8 @@ pipeline { stage('Functional on SLES 15') { when { beforeAgent true - expression { shouldStageRun('Functional on SLES 15') } + expression { shouldStageRun('Functional on SLES 15') && + !testsInStage(getFunctionalTags('-vm', 'vm', 'pr'))} } agent { label vm9_label('Leap15') @@ -1298,7 +1303,8 @@ pipeline { stage('Functional on Ubuntu 20.04') { when { beforeAgent true - expression { shouldStageRun('Functional on Ubuntu 20.04') } + expression { shouldStageRun('Functional on Ubuntu 20.04') && + !testsInStage(getFunctionalTags('-vm', 'vm', 'pr'))} } agent { label vm9_label('Ubuntu') From ab22926fb0aa4380cc622c989a4f3a6d71047b27 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Wed, 1 Jul 2026 18:54:26 -0400 Subject: [PATCH 02/11] Use scripted pipeline methods for the Test stages. Skip-test-hardware: true Test-tag: test_load_mpi test_setup_vm Signed-off-by: Phil Henderson --- Jenkinsfile | 388 ++++++++++++++++++---------------------------------- 1 file changed, 130 insertions(+), 258 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index c918985e726..59f3542fd75 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -21,6 +21,7 @@ import groovy.transform.Field // To use a test branch (i.e. PR) until it lands to master // I.e. for testing library changes //@Library(value='pipeline-lib@your_branch') _ +@Library(value='pipeline-lib@hendersp/DAOS-19247-fix') _ /* groovylint-disable-next-line CompileStatic */ job_status_internal = [:] @@ -1175,268 +1176,139 @@ pipeline { beforeAgent true expression { shouldStageRun('Test') } } - parallel { - stage('Functional on EL 8.8 with Valgrind') { - when { - beforeAgent true - expression { shouldStageRun('Functional on EL 8.8 with Valgrind') && - !testsInStage(getFunctionalTags('-vm', 'vm', 'pr')) } - } - agent { - label vm9_label('EL8') - } - steps { - job_step_update( - functionalTest( - inst_repos: daosRepos(), - inst_rpms: functionalPackages(1, next_version(), 'tests-internal') + - ' mercury-libfabric', - test_function: 'runTestFunctionalV2')) - } - post { - always { - functionalTestPostV2() - job_status_update() - } - } - } // stage('Functional on EL 8.8 with Valgrind') - stage('Functional on EL 8') { - when { - beforeAgent true - expression { shouldStageRun('Functional on EL 8') && - !testsInStage(getFunctionalTags('-vm', 'vm', 'pr')) } - } - agent { - label vm9_label('EL8') - } - steps { - job_step_update( - functionalTest( - inst_repos: daosRepos(), - inst_rpms: functionalPackages(1, next_version(), 'tests-internal') + - ' mercury-libfabric', - test_function: 'runTestFunctionalV2', - image_version: 'el8.10')) - } - post { - always { - functionalTestPostV2() - job_status_update() - } - } - } // stage('Functional on EL 8') - stage('Functional on EL 9') { - when { - beforeAgent true - expression { shouldStageRun('Functional on EL 9') && - !testsInStage(getFunctionalTags('-vm', 'vm', 'pr'))} - } - agent { - label vm9_label('EL9') - } - steps { - job_step_update( - functionalTest( - inst_repos: daosRepos(), - inst_rpms: functionalPackages(1, next_version(), 'tests-internal') + - ' mercury-libfabric', - test_function: 'runTestFunctionalV2', - image_version: 'el9.7')) - } - post { - always { - functionalTestPostV2() - job_status_update() - } - } - } // stage('Functional on EL 9') - stage('Functional on Leap 15') { - when { - beforeAgent true - expression { shouldStageRun('Functional on Leap 15') && - !testsInStage(getFunctionalTags('-vm', 'vm', 'pr'))} - } - agent { - label vm9_label('Leap15') - } - steps { - job_step_update( - functionalTest( - inst_repos: daosRepos(), - inst_rpms: functionalPackages(1, next_version(), 'tests-internal') + - ' mercury-libfabric', - test_function: 'runTestFunctionalV2', - image_version: 'leap15.6')) - } - post { - always { - functionalTestPostV2() - job_status_update() - } - } // post - } // stage('Functional on Leap 15') - stage('Functional on SLES 15') { - when { - beforeAgent true - expression { shouldStageRun('Functional on SLES 15') && - !testsInStage(getFunctionalTags('-vm', 'vm', 'pr'))} - } - agent { - label vm9_label('Leap15') - } - steps { - job_step_update( - functionalTest( + steps { + script { + parallel( + 'Functional on EL 8.8 with Valgrind': getFunctionalTestStage( + name: 'Functional on EL 8.8 with Valgrind', + runStage: shouldStageRun('Functional on EL 8.8 with Valgrind'), + pragma_suffix: '-vm', + label: vm9_label('EL8'), + next_version: next_version(), + other_packages: 'mercury-libfabric', + stage_tags: 'vm', + default_tags: 'memcheck', + nvme: 'auto', + job_status: job_status_internal, + image_version: 'el8.8' + ), + 'Functional on EL 8': getFunctionalTestStage( + name: 'Functional on EL 8', + runStage: shouldStageRun('Functional on EL 8'), + pragma_suffix: '-vm', + label: vm9_label('EL8'), + next_version: next_version(), + other_packages: 'mercury-libfabric', + stage_tags: 'vm', + default_tags: startedByTimer() ? 'pr daily_regression' : 'pr', + nvme: 'auto', + job_status: job_status_internal, + image_version: 'el8.10' + ), + 'Functional on EL 9': getFunctionalTestStage( + name: 'Functional on EL 9', + runStage: shouldStageRun('Functional on EL 9'), + pragma_suffix: '-vm', + label: vm9_label('EL9'), + next_version: next_version(), + other_packages: 'mercury-libfabric', + stage_tags: 'vm', + default_tags: startedByTimer() ? 'pr daily_regression' : 'pr', + nvme: 'auto', + job_status: job_status_internal, + image_version: 'el9.7' + ), + 'Functional on Leap 15': getFunctionalTestStage( + name: 'Functional on Leap 15', + runStage: shouldStageRun('Functional on Leap 15'), + pragma_suffix: '-vm', + label: vm9_label('Leap15'), + next_version: next_version(), + other_packages: 'mercury-libfabric', + stage_tags: 'vm', + default_tags: startedByTimer() ? 'pr daily_regression' : 'pr', + nvme: 'auto', + job_status: job_status_internal, + image_version: 'leap15.6' + ), + 'Functional on SLES 15': getFunctionalTestStage( + name: 'Functional on SLES 15', + runStage: shouldStageRun('Functional on SLES 15'), + pragma_suffix: '-vm', + label: vm9_label('Leap15'), + next_version: next_version(), + other_packages: 'mercury-libfabric', + stage_tags: 'vm', + default_tags: startedByTimer() ? 'pr daily_regression' : 'pr', + nvme: 'auto', + job_status: job_status_internal, + image_version: 'sles15.7' + ), + 'Functional on Ubuntu 20.04': getFunctionalTestStage( + name: 'Functional on Ubuntu 20.04', + runStage: shouldStageRun('Functional on Ubuntu 20.04'), + pragma_suffix: '-vm', + label: vm9_label('Ubuntu'), + next_version: next_version(), + other_packages: 'mercury-libfabric', + stage_tags: 'vm', + default_tags: startedByTimer() ? 'pr daily_regression' : 'pr', + nvme: 'auto', + job_status: job_status_internal + ), + 'Fault injection testing': scriptedTestStage( + name: 'Fault injection testing', + runStage: shouldStageRun('Fault injection testing'), + label: params.CI_FI_1_LABEL, + job_status: job_status_internal, + unitTestArgs: [ + timeout_time: 240, inst_repos: daosRepos(), - inst_rpms: functionalPackages(1, next_version(), 'tests-internal') + - ' mercury-libfabric', - test_function: 'runTestFunctionalV2', - image_version: 'sles15.7')) - } - post { - always { - functionalTestPostV2() - job_status_update() - } - } // post - } // stage('Functional on SLES 15') - stage('Functional on Ubuntu 20.04') { - when { - beforeAgent true - expression { shouldStageRun('Functional on Ubuntu 20.04') && - !testsInStage(getFunctionalTags('-vm', 'vm', 'pr'))} - } - agent { - label vm9_label('Ubuntu') - } - steps { - job_step_update( - functionalTest( + test_script: 'ci/unit/test_nlt.sh --memcheck no' + + ' --system-ram-reserved 4 --server-debug WARN' + + ' --log-usage-import nltr.json' + + ' --log-usage-save nltr.xml' + + ' --class-name fault-injection fi', + with_valgrind: '', + always_script: 'ci/unit/test_nlt_post.sh', + testResults: 'nlt-junit.xml', + unstash_opt: true, + unstash_tests: false, + inst_rpms: unitPackages(target: 'el9') + ' daos-client-tests', + image_version: 'el9.7', + prov_env_vars: 'VM_CPUS=14'], + unitTestPostArgs: [ + artifacts: ['nlt_logs/'], + testResults: 'nlt-junit.xml', + with_valgrind: '', + FI: true], + archiveArtifactsArgs: [ + artifacts: 'nlt_logs/fault-injection/', + allowEmptyArchive: true] + ), + 'Test RPMs on EL 9.6': scriptedTestStage( + name: 'Test RPMs on EL 9.6', + runStage: shouldStageRun('Test RPMs on EL 9.6'), + label: params.CI_UNIT_VM1_LABEL, + job_status: job_status_internal, + testRpmArgs: [ inst_repos: daosRepos(), - inst_rpms: functionalPackages(1, next_version(), 'tests-internal') + - ' mercury-libfabric', - test_function: 'runTestFunctionalV2')) - } - post { - always { - functionalTestPostV2() - job_status_update() - } - } // post - } // stage('Functional on Ubuntu 20.04') - stage('Fault injection testing') { - when { - beforeAgent true - expression { shouldStageRun('Fault injection testing') } - } - agent { - label params.CI_FI_1_LABEL - } - steps { - job_step_update( - unitTest(timeout_time: 240, - inst_repos: daosRepos(), - test_script: 'ci/unit/test_nlt.sh --memcheck no' + - ' --system-ram-reserved 4 --server-debug WARN' + - ' --log-usage-import nltr.json' + - ' --log-usage-save nltr.xml' + - ' --class-name fault-injection fi', - with_valgrind: '', - always_script: 'ci/unit/test_nlt_post.sh', - testResults: 'nlt-junit.xml', - unstash_opt: true, - unstash_tests: false, - inst_rpms: unitPackages(target: 'el9') + ' daos-client-tests', - image_version: 'el9.7', - prov_env_vars: 'VM_CPUS=14')) - } - post { - always { - unitTestPost artifacts: ['nlt_logs/'], - testResults: 'nlt-junit.xml', - with_valgrind: '', - FI: true - archiveArtifacts artifacts: 'nlt_logs/fault-injection/', - allowEmptyArchive: true - job_status_update() - } - } - } // stage('Fault injection testing') - stage('Test RPMs on EL 9.6') { - when { - beforeAgent true - expression { shouldStageRun('Test RPMs on EL 9.6') } - } - agent { - label params.CI_UNIT_VM1_LABEL - } - steps { - job_step_update( - testRpm(inst_repos: daosRepos(), - daos_pkg_version: daosPackagesVersion(next_version()), - inst_rpms: 'mercury-libfabric') - ) - } - post { - always { - rpm_test_post(env.STAGE_NAME, env.NODELIST) - } - } - } // stage('Test RPMs on EL 9.6') - stage('Test RPMs on Leap 15.5') { - when { - beforeAgent true - expression { shouldStageRun('Test RPMs on Leap 15.5') } - } - agent { - label params.CI_UNIT_VM1_LABEL - } - steps { - /* neither of these work as FTest strips the first node - out of the pool requiring 2 node clusters at minimum - * additionally for this use-case, can't override - ftest_arg with this :-( - script { - 'Test RPMs on Leap 15.5': getFunctionalTestStage( - name: 'Test RPMs on Leap 15.5', - pragma_suffix: '', - label: params.CI_UNIT_VM1_LABEL, - next_version: next_version(), - stage_tags: '', - default_tags: 'test_daos_management', - nvme: 'auto', - run_if_pr: true, - run_if_landing: true, - job_status: job_status_internal - ) - } - job_step_update( - functionalTest( - test_tag: 'test_daos_management', - ftest_arg: '--yaml_extension single_host', + daos_pkg_version: daosPackagesVersion(next_version()), + inst_rpms: 'mercury-libfabric'] + ), + 'Test RPMs on Leap 15.5': scriptedTestStage( + name: 'Test RPMs on Leap 15.5', + runStage: shouldStageRun('Test RPMs on Leap 15.5'), + label: params.CI_UNIT_VM1_LABEL, + job_status: job_status_internal, + testRpmArgs: [ inst_repos: daosRepos(), - inst_rpms: functionalPackages(1, next_version(), 'tests-internal'), - test_function: 'runTestFunctionalV2')) - } - post { - always { - functionalTestPostV2() - job_status_update() - } - } */ - job_step_update( - testRpm(inst_repos: daosRepos(), - daos_pkg_version: daosPackagesVersion(next_version()), - inst_rpms: 'mercury-libfabric') + daos_pkg_version: daosPackagesVersion(next_version()), + inst_rpms: 'mercury-libfabric'] ) - } - post { - always { - rpm_test_post(env.STAGE_NAME, env.NODELIST) - } - } - } // stage('Test RPMs on Leap 15.5') - } // parallel + ) + } + } } // stage('Test') stage('Test Storage Prep on EL 8.8') { when { From e2c7edaf89cf6c9872cdca2f72461d55ab6fccbf Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Wed, 1 Jul 2026 22:58:52 -0400 Subject: [PATCH 03/11] Updates. skip-unit-tests: true skip-test-hardware: true test-tag: test_load_mpi test_setup_vm Signed-off-by: Phil Henderson --- Jenkinsfile | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 59f3542fd75..322583259d4 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1260,15 +1260,15 @@ pipeline { name: 'Fault injection testing', runStage: shouldStageRun('Fault injection testing'), label: params.CI_FI_1_LABEL, - job_status: job_status_internal, + jobStatus: job_status_internal, unitTestArgs: [ timeout_time: 240, inst_repos: daosRepos(), test_script: 'ci/unit/test_nlt.sh --memcheck no' + - ' --system-ram-reserved 4 --server-debug WARN' + - ' --log-usage-import nltr.json' + - ' --log-usage-save nltr.xml' + - ' --class-name fault-injection fi', + ' --system-ram-reserved 4 --server-debug WARN' + + ' --log-usage-import nltr.json' + + ' --log-usage-save nltr.xml' + + ' --class-name fault-injection fi', with_valgrind: '', always_script: 'ci/unit/test_nlt_post.sh', testResults: 'nlt-junit.xml', @@ -1290,7 +1290,7 @@ pipeline { name: 'Test RPMs on EL 9.6', runStage: shouldStageRun('Test RPMs on EL 9.6'), label: params.CI_UNIT_VM1_LABEL, - job_status: job_status_internal, + jobStatus: job_status_internal, testRpmArgs: [ inst_repos: daosRepos(), daos_pkg_version: daosPackagesVersion(next_version()), @@ -1300,7 +1300,7 @@ pipeline { name: 'Test RPMs on Leap 15.5', runStage: shouldStageRun('Test RPMs on Leap 15.5'), label: params.CI_UNIT_VM1_LABEL, - job_status: job_status_internal, + jobStatus: job_status_internal, testRpmArgs: [ inst_repos: daosRepos(), daos_pkg_version: daosPackagesVersion(next_version()), From 231152d34d652cdc5bd6f9ffbb0b598f5c1f031c Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Thu, 2 Jul 2026 01:24:57 -0400 Subject: [PATCH 04/11] Updates. skip-unit-tests: true skip-test-hardware: true test-tag: test_load_mpi test_setup_vm Signed-off-by: Phil Henderson --- Jenkinsfile | 63 +++++++++++++++++++---------------------------------- 1 file changed, 23 insertions(+), 40 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 322583259d4..279943a63c5 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -470,19 +470,6 @@ String vm9_label(String distro) { def_val: params.FUNCTIONAL_VM_LABEL)) } -void rpm_test_post(String stageName, String node) { - // Extract first node from comma-delimited list - String firstNode = node.split(',')[0].trim() - sh label: 'Fetch and stage artifacts', - script: 'hostname; ssh -i ci_key jenkins@' + firstNode + - ' ls -ltar /tmp; mkdir -p "' + env.STAGE_NAME + '/" && ' + - 'scp -i ci_key jenkins@' + firstNode + - ':/tmp/{{suite_dmg,daos_{server_helper,{control,agent}}}.log,daos_server.log.*} "' + - stageName + '/"' - archiveArtifacts artifacts: env.STAGE_NAME + '/**' - job_status_update() -} - String sconsArgs() { if (!params.CI_SCONS_ARGS) { return sconsFaultsArgs() @@ -1256,27 +1243,25 @@ pipeline { nvme: 'auto', job_status: job_status_internal ), - 'Fault injection testing': scriptedTestStage( + 'Fault injection testing': scriptedUnitTestStage( name: 'Fault injection testing', runStage: shouldStageRun('Fault injection testing'), label: params.CI_FI_1_LABEL, jobStatus: job_status_internal, - unitTestArgs: [ - timeout_time: 240, - inst_repos: daosRepos(), - test_script: 'ci/unit/test_nlt.sh --memcheck no' + - ' --system-ram-reserved 4 --server-debug WARN' + - ' --log-usage-import nltr.json' + - ' --log-usage-save nltr.xml' + - ' --class-name fault-injection fi', - with_valgrind: '', - always_script: 'ci/unit/test_nlt_post.sh', - testResults: 'nlt-junit.xml', - unstash_opt: true, - unstash_tests: false, - inst_rpms: unitPackages(target: 'el9') + ' daos-client-tests', - image_version: 'el9.7', - prov_env_vars: 'VM_CPUS=14'], + distro: 'el9', + timeoutTime: 240, + testScript: 'ci/unit/test_nlt.sh --memcheck no' + + ' --system-ram-reserved 4 --server-debug WARN' + + ' --log-usage-import nltr.json' + + ' --log-usage-save nltr.xml' + + ' --class-name fault-injection fi', + withValgrind: '', + alwaysScript: 'ci/unit/test_nlt_post.sh', + testResults: 'nlt-junit.xml', + unstashOpt: true, + unstashTests: false, + imageVersion: 'el9.7', + provEnvVars: 'VM_CPUS=14', unitTestPostArgs: [ artifacts: ['nlt_logs/'], testResults: 'nlt-junit.xml', @@ -1286,25 +1271,23 @@ pipeline { artifacts: 'nlt_logs/fault-injection/', allowEmptyArchive: true] ), - 'Test RPMs on EL 9.6': scriptedTestStage( + 'Test RPMs on EL 9.6': scriptedTestRpmStage( name: 'Test RPMs on EL 9.6', runStage: shouldStageRun('Test RPMs on EL 9.6'), label: params.CI_UNIT_VM1_LABEL, jobStatus: job_status_internal, - testRpmArgs: [ - inst_repos: daosRepos(), - daos_pkg_version: daosPackagesVersion(next_version()), - inst_rpms: 'mercury-libfabric'] + nextVersion: next_version(), + instRpms: 'mercury-libfabric', + nodeList: env.NODELIST, ), - 'Test RPMs on Leap 15.5': scriptedTestStage( + 'Test RPMs on Leap 15.5': scriptedTestRpmStage( name: 'Test RPMs on Leap 15.5', runStage: shouldStageRun('Test RPMs on Leap 15.5'), label: params.CI_UNIT_VM1_LABEL, jobStatus: job_status_internal, - testRpmArgs: [ - inst_repos: daosRepos(), - daos_pkg_version: daosPackagesVersion(next_version()), - inst_rpms: 'mercury-libfabric'] + nextVersion: next_version(), + instRpms: 'mercury-libfabric', + nodeList: env.NODELIST, ) ) } From 9658ef1faf144febf0429d7796f3bc91ef81c55c Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Thu, 2 Jul 2026 11:57:06 -0400 Subject: [PATCH 05/11] Updates skip-unit-tests: true skip-test-hardware: true skip-functional-on-el-8: false skip-functional-on-leap-15: false skip-functional-on-sles-15: false test-tag: test_load_mpi test_setup_vm Signed-off-by: Phil Henderson --- Jenkinsfile | 6 ++++++ ci/rpm/test_daos_post.sh | 25 +++++++++++++++++++++++++ 2 files changed, 31 insertions(+) create mode 100755 ci/rpm/test_daos_post.sh diff --git a/Jenkinsfile b/Jenkinsfile index 279943a63c5..de7b257e7a4 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1279,6 +1279,9 @@ pipeline { nextVersion: next_version(), instRpms: 'mercury-libfabric', nodeList: env.NODELIST, + alwaysScript: 'ci/rpm/test_daos_post.sh test_rpms_el_logs', + archiveArtifactsArgs: [ + artifacts: 'test_rpms_el_logs/'] ), 'Test RPMs on Leap 15.5': scriptedTestRpmStage( name: 'Test RPMs on Leap 15.5', @@ -1288,6 +1291,9 @@ pipeline { nextVersion: next_version(), instRpms: 'mercury-libfabric', nodeList: env.NODELIST, + alwaysScript: 'ci/rpm/test_daos_post.sh test_rpms_leap_logs', + archiveArtifactsArgs: [ + artifacts: 'test_rpms_leap_logs/'] ) ) } diff --git a/ci/rpm/test_daos_post.sh b/ci/rpm/test_daos_post.sh new file mode 100755 index 00000000000..5f98e6d045c --- /dev/null +++ b/ci/rpm/test_daos_post.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# This is a post test processing script for post processing the +# run_utest.py stage CI run + +set -uex + +test_log_dir="${1:-}" +first_node="${NODELIST%%,*}" + +if [ -z "$test_log_dir" ]; then + echo "test_daos_post: The test log directory argument is missing!" + exit 1 +fi + +# Copy logs from test_daos_node.sh execution +rm -rf "$test_log_dir" +mkdir -p "$test_log_dir" +rsync -v -dpt -z -e "ssh ${SSH_KEY_ARGS}" jenkins@"${first_node}":/tmp/ \ + --filter="include suite_dmg.log" \ + --filter="include daos_server_helper.log" \ + --filter="include daos_control.log" \ + --filter="include daos_agent.log" \ + --filter="include daos_server.log.*" \ + --filter="exclude *" "${test_log_dir}/" From d8d25ab9e28f645b2c73e4f1d91ddd0b5fbfda1c Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Thu, 2 Jul 2026 17:56:40 -0400 Subject: [PATCH 06/11] Test artifact dir name. Skip-cancel-previous-builds: true Skip-functional-test: true SKip-fault-injection-testing: true Skip-unit-tests: true Signed-off-by: Phil Henderson --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index de7b257e7a4..774382bd72b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1291,9 +1291,9 @@ pipeline { nextVersion: next_version(), instRpms: 'mercury-libfabric', nodeList: env.NODELIST, - alwaysScript: 'ci/rpm/test_daos_post.sh test_rpms_leap_logs', + alwaysScript: 'ci/rpm/test_daos_post.sh \'Test RPMs on Leap 15.5\'', archiveArtifactsArgs: [ - artifacts: 'test_rpms_leap_logs/'] + artifacts: '\'Test RPMs on Leap 15.5/\''] ) ) } From 33213bbae20b479c08b6f6dbf17654d589e7500b Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Thu, 2 Jul 2026 18:42:59 -0400 Subject: [PATCH 07/11] Test RPM updates skip-unit-tests: true skip-test-hardware: true skip-functional-on-el-8: false skip-functional-on-leap-15: false skip-functional-on-sles-15: false test-tag: test_load_mpi test_setup_vm Signed-off-by: Phil Henderson --- Jenkinsfile | 8 +++++--- ci/rpm/test_daos_post.sh | 17 +++++++++-------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 774382bd72b..674f8ef7bb9 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1277,11 +1277,12 @@ pipeline { label: params.CI_UNIT_VM1_LABEL, jobStatus: job_status_internal, nextVersion: next_version(), + imageVersion: 'el9.6', instRpms: 'mercury-libfabric', nodeList: env.NODELIST, - alwaysScript: 'ci/rpm/test_daos_post.sh test_rpms_el_logs', + alwaysScript: 'ci/rpm/test_daos_post.sh \'Test RPMs on EL 9.6\'', archiveArtifactsArgs: [ - artifacts: 'test_rpms_el_logs/'] + artifacts: 'Test RPMs on EL 9.6/'] ), 'Test RPMs on Leap 15.5': scriptedTestRpmStage( name: 'Test RPMs on Leap 15.5', @@ -1289,11 +1290,12 @@ pipeline { label: params.CI_UNIT_VM1_LABEL, jobStatus: job_status_internal, nextVersion: next_version(), + imageVersion: 'leap15.5', instRpms: 'mercury-libfabric', nodeList: env.NODELIST, alwaysScript: 'ci/rpm/test_daos_post.sh \'Test RPMs on Leap 15.5\'', archiveArtifactsArgs: [ - artifacts: '\'Test RPMs on Leap 15.5/\''] + artifacts: 'Test RPMs on Leap 15.5/'] ) ) } diff --git a/ci/rpm/test_daos_post.sh b/ci/rpm/test_daos_post.sh index 5f98e6d045c..e1e9fa9b3ef 100755 --- a/ci/rpm/test_daos_post.sh +++ b/ci/rpm/test_daos_post.sh @@ -8,18 +8,19 @@ set -uex test_log_dir="${1:-}" first_node="${NODELIST%%,*}" -if [ -z "$test_log_dir" ]; then +if [ -z "${test_log_dir}" ]; then echo "test_daos_post: The test log directory argument is missing!" exit 1 fi # Copy logs from test_daos_node.sh execution -rm -rf "$test_log_dir" -mkdir -p "$test_log_dir" +rm -rf "${test_log_dir}" +mkdir -p "${test_log_dir}/configs" +rsync -v -dpt -z -e "ssh ${SSH_KEY_ARGS}" jenkins@"${first_node}":/etc/daos/ \ + --filter="include daos_*.yml*" \ + --filter="exclude *" "${test_log_dir}/configs/" +mkdir -p "${test_log_dir}/logs" rsync -v -dpt -z -e "ssh ${SSH_KEY_ARGS}" jenkins@"${first_node}":/tmp/ \ --filter="include suite_dmg.log" \ - --filter="include daos_server_helper.log" \ - --filter="include daos_control.log" \ - --filter="include daos_agent.log" \ - --filter="include daos_server.log.*" \ - --filter="exclude *" "${test_log_dir}/" + --filter="include daos_*.log*" \ + --filter="exclude *" "${test_log_dir}/logs/" From e57a1e0efbee3752a5675aab3bd15884e7471957 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Thu, 2 Jul 2026 22:48:16 -0400 Subject: [PATCH 08/11] Hack to install pciutils-3.13 skip-unit-tests: true skip-test-hardware: true skip-functional-on-el-8: false skip-functional-on-leap-15: false skip-functional-on-sles-15: false test-tag: test_load_mpi test_setup_vm Signed-off-by: Phil Henderson --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 674f8ef7bb9..e23b43cbaa1 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1291,7 +1291,7 @@ pipeline { jobStatus: job_status_internal, nextVersion: next_version(), imageVersion: 'leap15.5', - instRpms: 'mercury-libfabric', + instRpms: 'mercury-libfabric pciutils-3.13.0-150300.13.12.1.x86_64', nodeList: env.NODELIST, alwaysScript: 'ci/rpm/test_daos_post.sh \'Test RPMs on Leap 15.5\'', archiveArtifactsArgs: [ From 7b33944d9f67c6cf2e25d0e9d26b13ea31105b53 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Thu, 2 Jul 2026 23:06:31 -0400 Subject: [PATCH 09/11] Collect log for test_daos_node.sh execution sjip-cancel-previous-builds: true skip-unit-tests: true skip-test-hardware: true skip-functional-on-el-9: true skip-fault-injection-testing: true Signed-off-by: Phil Henderson --- ci/rpm/test_daos.sh | 10 +++++----- ci/rpm/test_daos_post.sh | 3 +++ 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/ci/rpm/test_daos.sh b/ci/rpm/test_daos.sh index 6e1f415d237..67fba6bc44d 100755 --- a/ci/rpm/test_daos.sh +++ b/ci/rpm/test_daos.sh @@ -9,9 +9,9 @@ set -uex IFS=" " read -r -a nodelist <<< "${NODELIST//,/ }" mydir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" -# shellcheck disable=SC2029 ssh "$SSH_KEY_ARGS" jenkins@"${nodelist[0]}" \ - "NODE=${nodelist[0]} \ - DAOS_PKG_VERSION=$DAOS_PKG_VERSION \ - PYTHON_VERSION=\"${PYTHON_VERSION}\" \ - $(cat "$mydir/test_daos_node.sh")" + "NODE=${nodelist[0]} \ + DAOS_PKG_VERSION=$DAOS_PKG_VERSION \ + PYTHON_VERSION=${PYTHON_VERSION} \ + bash -s 2>&1 | tee /tmp/test_daos_rpms.log" \ + < "$mydir/test_daos_node.sh" diff --git a/ci/rpm/test_daos_post.sh b/ci/rpm/test_daos_post.sh index e1e9fa9b3ef..392e1245960 100755 --- a/ci/rpm/test_daos_post.sh +++ b/ci/rpm/test_daos_post.sh @@ -24,3 +24,6 @@ rsync -v -dpt -z -e "ssh ${SSH_KEY_ARGS}" jenkins@"${first_node}":/tmp/ \ --filter="include suite_dmg.log" \ --filter="include daos_*.log*" \ --filter="exclude *" "${test_log_dir}/logs/" +rsync -v -dpt -z -e "ssh ${SSH_KEY_ARGS}" jenkins@"${first_node}":/tmp/ \ + --filter="include test_daos_rpms.log" \ + --filter="exclude *" "${test_log_dir}/" From a643198b05e97201975cfa258a7f366274e31329 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Thu, 2 Jul 2026 23:52:09 -0400 Subject: [PATCH 10/11] Log collection hardening skip-unit-tests: true skip-test-hardware: true skip-functional-on-el-9: true skip-fault-injection-testing: true Signed-off-by: Phil Henderson --- ci/rpm/test_daos.sh | 4 ++++ ci/rpm/test_daos_node.sh | 3 +++ ci/rpm/test_daos_post.sh | 6 +++--- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/ci/rpm/test_daos.sh b/ci/rpm/test_daos.sh index 67fba6bc44d..0f732aa140e 100755 --- a/ci/rpm/test_daos.sh +++ b/ci/rpm/test_daos.sh @@ -9,9 +9,13 @@ set -uex IFS=" " read -r -a nodelist <<< "${NODELIST//,/ }" mydir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +# shellcheck disable=SC2029 ssh "$SSH_KEY_ARGS" jenkins@"${nodelist[0]}" \ "NODE=${nodelist[0]} \ DAOS_PKG_VERSION=$DAOS_PKG_VERSION \ PYTHON_VERSION=${PYTHON_VERSION} \ bash -s 2>&1 | tee /tmp/test_daos_rpms.log" \ < "$mydir/test_daos_node.sh" + + # bash -s > /tmp/test_daos_rpms.log 2>&1" + # < "$mydir/test_daos_node.sh" diff --git a/ci/rpm/test_daos_node.sh b/ci/rpm/test_daos_node.sh index ce16704c884..4cc9728d018 100755 --- a/ci/rpm/test_daos_node.sh +++ b/ci/rpm/test_daos_node.sh @@ -29,6 +29,9 @@ if [ -n "$DAOS_PKG_VERSION" ]; then fi set -uex + +echo "${PRETTY_NAME:-Unknown OS}" + sudo $YUM -y install daos-client"$DAOS_PKG_VERSION" if rpm -q daos-server; then echo "daos-server RPM should not be installed as a dependency of daos-client" diff --git a/ci/rpm/test_daos_post.sh b/ci/rpm/test_daos_post.sh index 392e1245960..1ab04d9d63f 100755 --- a/ci/rpm/test_daos_post.sh +++ b/ci/rpm/test_daos_post.sh @@ -18,12 +18,12 @@ rm -rf "${test_log_dir}" mkdir -p "${test_log_dir}/configs" rsync -v -dpt -z -e "ssh ${SSH_KEY_ARGS}" jenkins@"${first_node}":/etc/daos/ \ --filter="include daos_*.yml*" \ - --filter="exclude *" "${test_log_dir}/configs/" + --filter="exclude *" "${test_log_dir}/configs/" || true mkdir -p "${test_log_dir}/logs" rsync -v -dpt -z -e "ssh ${SSH_KEY_ARGS}" jenkins@"${first_node}":/tmp/ \ --filter="include suite_dmg.log" \ --filter="include daos_*.log*" \ - --filter="exclude *" "${test_log_dir}/logs/" + --filter="exclude *" "${test_log_dir}/logs/" || true rsync -v -dpt -z -e "ssh ${SSH_KEY_ARGS}" jenkins@"${first_node}":/tmp/ \ --filter="include test_daos_rpms.log" \ - --filter="exclude *" "${test_log_dir}/" + --filter="exclude *" "${test_log_dir}/" || true From 19629f9635cdc042146c3baaaf3af3cab0a38c4c Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Fri, 3 Jul 2026 00:23:29 -0400 Subject: [PATCH 11/11] Remove hack skip-unit-tests: true skip-test-hardware: true skip-functional-on-el-8: false skip-functional-on-leap-15: false skip-functional-on-sles-15: false test-tag: test_load_mpi test_setup_vm Signed-off-by: Phil Henderson --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index e23b43cbaa1..674f8ef7bb9 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1291,7 +1291,7 @@ pipeline { jobStatus: job_status_internal, nextVersion: next_version(), imageVersion: 'leap15.5', - instRpms: 'mercury-libfabric pciutils-3.13.0-150300.13.12.1.x86_64', + instRpms: 'mercury-libfabric', nodeList: env.NODELIST, alwaysScript: 'ci/rpm/test_daos_post.sh \'Test RPMs on Leap 15.5\'', archiveArtifactsArgs: [