diff --git a/regression-test/suites/cloud_p0/recycler/check_meta.groovy b/regression-test/suites/cloud_p0/recycler/check_meta.groovy index a9f34f19787455..8dc82ca1227e2d 100644 --- a/regression-test/suites/cloud_p0/recycler/check_meta.groovy +++ b/regression-test/suites/cloud_p0/recycler/check_meta.groovy @@ -24,6 +24,8 @@ suite("check_meta", "check_meta") { def status = 200 def recyclerLastSuccessTime = -1 def recyclerLastFinishTime = -1 + def waitTimeoutMs = 30 * 60 * 1000L + def recyclerWaitDeadline = caseStartTime + waitTimeoutMs String jdbcUrl = context.config.jdbcUrl String urlWithoutSchema = jdbcUrl.substring(jdbcUrl.indexOf("://") + 3) @@ -89,6 +91,12 @@ suite("check_meta", "check_meta") { if (recyclerLastSuccessTime > caseStartTime) { break } + if (System.currentTimeMillis() >= recyclerWaitDeadline) { + throw new IllegalStateException( + "Timed out waiting for recycler job after ${waitTimeoutMs / 1000}s: " + + "caseStartTime=${caseStartTime}, recyclerLastFinishTime=${recyclerLastFinishTime}, " + + "recyclerLastSuccessTime=${recyclerLastSuccessTime}") + } } while (true) assertEquals(recyclerLastFinishTime, recyclerLastSuccessTime) diff --git a/regression-test/suites/cloud_p0/recycler/test_checker.groovy b/regression-test/suites/cloud_p0/recycler/test_checker.groovy index 6763db2c682e63..055ddaa6ae3922 100644 --- a/regression-test/suites/cloud_p0/recycler/test_checker.groovy +++ b/regression-test/suites/cloud_p0/recycler/test_checker.groovy @@ -180,6 +180,9 @@ suite("test_checker") { // Make sure to complete at least one round of checking def checkerLastSuccessTime = -1 def checkerLastFinishTime = -1 + def waitTimeoutMs = 30 * 60 * 1000L + def checkerWaitStartTime = System.currentTimeMillis() + def checkerWaitDeadline = checkerWaitStartTime + waitTimeoutMs def triggerChecker = { def triggerCheckerApi = { checkFunc -> @@ -233,6 +236,12 @@ suite("test_checker") { if (checkerLastFinishTime > caseStartTime) { break } + if (System.currentTimeMillis() >= checkerWaitDeadline) { + throw new IllegalStateException( + "Timed out waiting for checker job after ${waitTimeoutMs / 1000}s: " + + "caseStartTime=${caseStartTime}, checkerLastFinishTime=${checkerLastFinishTime}, " + + "checkerLastSuccessTime=${checkerLastSuccessTime}") + } } while (true) assertTrue(checkerLastSuccessTime < checkerLastFinishTime) // Check MUST fail @@ -298,4 +307,4 @@ suite("test_checker") { } } } -} \ No newline at end of file +} diff --git a/regression-test/suites/cloud_p0/recycler/test_recycler.groovy b/regression-test/suites/cloud_p0/recycler/test_recycler.groovy index 3a6e99f1778fc9..2d0b3e454f8b7e 100644 --- a/regression-test/suites/cloud_p0/recycler/test_recycler.groovy +++ b/regression-test/suites/cloud_p0/recycler/test_recycler.groovy @@ -25,6 +25,8 @@ suite("test_recycler") { def caseStartTime = System.currentTimeMillis() def recyclerLastSuccessTime = -1 def recyclerLastFinishTime = -1 + def waitTimeoutMs = 30 * 60 * 1000L + def recyclerWaitDeadline = caseStartTime + waitTimeoutMs // Make sure to complete at least one round of recycling def getRecycleJobInfo = { @@ -61,6 +63,12 @@ suite("test_recycler") { if (recyclerLastFinishTime > caseStartTime) { break } + if (System.currentTimeMillis() >= recyclerWaitDeadline) { + throw new IllegalStateException( + "Timed out waiting for recycler job after ${waitTimeoutMs / 1000}s: " + + "caseStartTime=${caseStartTime}, recyclerLastFinishTime=${recyclerLastFinishTime}, " + + "recyclerLastSuccessTime=${recyclerLastSuccessTime}") + } } while (true) assertEquals(recyclerLastFinishTime, recyclerLastSuccessTime) @@ -86,6 +94,8 @@ suite("test_recycler") { // Make sure to complete at least one round of checking def checkerLastSuccessTime = -1 def checkerLastFinishTime = -1 + def checkerWaitStartTime = System.currentTimeMillis() + def checkerWaitDeadline = checkerWaitStartTime + waitTimeoutMs def triggerChecker = { def triggerCheckerApi = { checkFunc -> @@ -139,6 +149,13 @@ suite("test_recycler") { if (checkerLastSuccessTime > recyclerLastSuccessTime) { break } + if (System.currentTimeMillis() >= checkerWaitDeadline) { + throw new IllegalStateException( + "Timed out waiting for checker job after ${waitTimeoutMs / 1000}s: " + + "recyclerLastSuccessTime=${recyclerLastSuccessTime}, " + + "checkerLastFinishTime=${checkerLastFinishTime}, " + + "checkerLastSuccessTime=${checkerLastSuccessTime}") + } } while (true) assertEquals(checkerLastFinishTime, checkerLastSuccessTime) }