branch-4.1: [fix](test) bound recycler polling waits

regression-test/suites/cloud_p0/recycler/check_meta.groovy

@@ -24,6 +24,8 @@ suite("check_meta", "check_meta") {
     def status = 200
     def recyclerLastSuccessTime = -1
     def recyclerLastFinishTime = -1
+    def waitTimeoutMs = 30 * 60 * 1000L
+    def recyclerWaitDeadline = caseStartTime + waitTimeoutMs
 
     String jdbcUrl = context.config.jdbcUrl
     String urlWithoutSchema = jdbcUrl.substring(jdbcUrl.indexOf("://") + 3)
@@ -89,6 +91,12 @@ suite("check_meta", "check_meta") {
         if (recyclerLastSuccessTime > caseStartTime) {
             break
         }
+        if (System.currentTimeMillis() >= recyclerWaitDeadline) {
+            throw new IllegalStateException(
+                    "Timed out waiting for recycler job after ${waitTimeoutMs / 1000}s: " +
+                    "caseStartTime=${caseStartTime}, recyclerLastFinishTime=${recyclerLastFinishTime}, " +
+                    "recyclerLastSuccessTime=${recyclerLastSuccessTime}")
+        }
     } while (true)
     assertEquals(recyclerLastFinishTime, recyclerLastSuccessTime)
 

regression-test/suites/cloud_p0/recycler/test_checker.groovy

@@ -180,6 +180,9 @@ suite("test_checker") {
         // Make sure to complete at least one round of checking
         def checkerLastSuccessTime = -1
         def checkerLastFinishTime = -1
+        def waitTimeoutMs = 30 * 60 * 1000L
+        def checkerWaitStartTime = System.currentTimeMillis()
+        def checkerWaitDeadline = checkerWaitStartTime + waitTimeoutMs
 
         def triggerChecker = {
             def triggerCheckerApi = { checkFunc ->
@@ -233,6 +236,12 @@ suite("test_checker") {
             if (checkerLastFinishTime > caseStartTime) {
                 break
             }
+            if (System.currentTimeMillis() >= checkerWaitDeadline) {
+                throw new IllegalStateException(
+                        "Timed out waiting for checker job after ${waitTimeoutMs / 1000}s: " +
+                        "caseStartTime=${caseStartTime}, checkerLastFinishTime=${checkerLastFinishTime}, " +
+                        "checkerLastSuccessTime=${checkerLastSuccessTime}")
+            }
         } while (true)
         assertTrue(checkerLastSuccessTime < checkerLastFinishTime) // Check MUST fail
 
@@ -298,4 +307,4 @@ suite("test_checker") {
             }
         }
     }
-}
+}

regression-test/suites/cloud_p0/recycler/test_recycler.groovy

@@ -25,6 +25,8 @@ suite("test_recycler") {
     def caseStartTime = System.currentTimeMillis()
     def recyclerLastSuccessTime = -1
     def recyclerLastFinishTime = -1
+    def waitTimeoutMs = 30 * 60 * 1000L
+    def recyclerWaitDeadline = caseStartTime + waitTimeoutMs
 
     // Make sure to complete at least one round of recycling
     def getRecycleJobInfo = {
@@ -61,6 +63,12 @@ suite("test_recycler") {
         if (recyclerLastFinishTime > caseStartTime) {
             break
         }
+        if (System.currentTimeMillis() >= recyclerWaitDeadline) {
+            throw new IllegalStateException(
+                    "Timed out waiting for recycler job after ${waitTimeoutMs / 1000}s: " +
+                    "caseStartTime=${caseStartTime}, recyclerLastFinishTime=${recyclerLastFinishTime}, " +
+                    "recyclerLastSuccessTime=${recyclerLastSuccessTime}")
+        }
     } while (true)
     assertEquals(recyclerLastFinishTime, recyclerLastSuccessTime)
 
@@ -86,6 +94,8 @@ suite("test_recycler") {
     // Make sure to complete at least one round of checking
     def checkerLastSuccessTime = -1
     def checkerLastFinishTime = -1
+    def checkerWaitStartTime = System.currentTimeMillis()
+    def checkerWaitDeadline = checkerWaitStartTime + waitTimeoutMs
 
     def triggerChecker = {
         def triggerCheckerApi = { checkFunc ->
@@ -139,6 +149,13 @@ suite("test_recycler") {
         if (checkerLastSuccessTime > recyclerLastSuccessTime) {
             break
         }
+        if (System.currentTimeMillis() >= checkerWaitDeadline) {
+            throw new IllegalStateException(
+                    "Timed out waiting for checker job after ${waitTimeoutMs / 1000}s: " +
+                    "recyclerLastSuccessTime=${recyclerLastSuccessTime}, " +
+                    "checkerLastFinishTime=${checkerLastFinishTime}, " +
+                    "checkerLastSuccessTime=${checkerLastSuccessTime}")
+        }
     } while (true)
     assertEquals(checkerLastFinishTime, checkerLastSuccessTime)
 }