From e28e5b56e54dd80ebfe29b5ec2144d1675bde084 Mon Sep 17 00:00:00 2001 From: Tim Carr Date: Mon, 4 May 2026 13:57:16 +0800 Subject: [PATCH 1/7] Tests: Retry `apiCheckSubscriberExists` --- tests/Support/Helper/KitAPI.php | 63 +++++++++++++++++++++++++++------ 1 file changed, 53 insertions(+), 10 deletions(-) diff --git a/tests/Support/Helper/KitAPI.php b/tests/Support/Helper/KitAPI.php index 8053a9851..b09aaf523 100644 --- a/tests/Support/Helper/KitAPI.php +++ b/tests/Support/Helper/KitAPI.php @@ -49,17 +49,25 @@ public function apiEncodeState($returnTo, $clientID) */ public function apiCheckSubscriberExists($I, $emailAddress, $firstName = false) { - // Run request. - $results = $this->apiRequest( - 'subscribers', - 'GET', - [ - 'email_address' => $emailAddress, - 'include_total_count' => true, + // Retry the API request as sometimes there's a lag before the subscriber is queryable via the API. + $results = $this->retryUntil( + function () use ($emailAddress) { + $results = $this->apiRequest( + 'subscribers', + 'GET', + [ + 'email_address' => $emailAddress, + 'include_total_count' => true, - // Check all subscriber states. - 'status' => 'all', - ] + // Check all subscriber states. + 'status' => 'all', + ] + ); + + // Return the results only if a subscriber was found, so + // retryUntil() will keep trying otherwise. + return ($results['pagination']['total_count'] > 0) ? $results : false; + } ); // Check at least one subscriber was returned and it matches the email address. @@ -324,4 +332,39 @@ public function apiRequest($endpoint, $method = 'GET', $params = array()) // Return JSON decoded response. return json_decode($result->getBody()->getContents(), true); } + + /** + * Repeatedly invokes the given callback until it returns a truthy value, or + * the maximum number of attempts is reached. + * + * Use this to wrap API checks that can be flaky due to ingestion lag at + * Kit's end (e.g. a subscriber created via a form submission isn't always + * immediately queryable via the `subscribers` endpoint). + * + * @since 3.3.2 + * + * @param callable $callback Callback to invoke. Should return the value + * to use, or false/null to indicate the + * check has not yet succeeded. + * @param int $attempts Maximum number of attempts. + * @param int $delay Seconds to wait between attempts. + * @return mixed The truthy value returned by $callback, or + * false if all attempts are exhausted. + */ + private function retryUntil(callable $callback, $attempts = 3, $delay = 2) + { + for ($i = 0; $i < $attempts; $i++) { + $result = $callback(); + if ($result) { + return $result; + } + + // Don't sleep after the final attempt. + if ($i < $attempts - 1) { + sleep($delay); + } + } + + return false; + } } From 3d24b1e819547d936a9c1520aaa2f68859ba53de Mon Sep 17 00:00:00 2001 From: Tim Carr Date: Mon, 4 May 2026 13:57:23 +0800 Subject: [PATCH 2/7] Isolate failing tests --- .github/workflows/tests.yml | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f46f553a1..1805abd6a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -93,25 +93,8 @@ jobs: # Folder names within the 'tests' folder to run tests in parallel. test-groups: [ - 'EndToEnd/broadcasts/blocks-shortcodes', - 'EndToEnd/broadcasts/import-export', - 'EndToEnd/forms/blocks-shortcodes', - 'EndToEnd/forms/general', - 'EndToEnd/forms/post-types', - 'EndToEnd/general/other', - 'EndToEnd/general/uninstall', - 'EndToEnd/general/plugin-screens', - 'EndToEnd/integrations/divi-builder', - 'EndToEnd/integrations/divi-theme', 'EndToEnd/integrations/other', - 'EndToEnd/integrations/wlm', - 'EndToEnd/integrations/woocommerce', - 'EndToEnd/landing-pages', - 'EndToEnd/products', - 'EndToEnd/restrict-content/general', - 'EndToEnd/restrict-content/post-types', - 'EndToEnd/tags', - 'Integration' + 'EndToEnd/integrations/wlm' ] # Steps to install, configure and run tests From edc31490cbfe08e95292c4f781e2c3716940f8f6 Mon Sep 17 00:00:00 2001 From: Tim Carr Date: Mon, 4 May 2026 13:57:58 +0800 Subject: [PATCH 3/7] Coding standards --- tests/Support/Helper/KitAPI.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Support/Helper/KitAPI.php b/tests/Support/Helper/KitAPI.php index b09aaf523..468f75ca3 100644 --- a/tests/Support/Helper/KitAPI.php +++ b/tests/Support/Helper/KitAPI.php @@ -66,7 +66,7 @@ function () use ($emailAddress) { // Return the results only if a subscriber was found, so // retryUntil() will keep trying otherwise. - return ($results['pagination']['total_count'] > 0) ? $results : false; + return ( $results['pagination']['total_count'] > 0 ) ? $results : false; } ); From 19c5d437607266d18ef3341472803fb1c574e1ea Mon Sep 17 00:00:00 2001 From: Tim Carr Date: Mon, 4 May 2026 14:15:31 +0800 Subject: [PATCH 4/7] Check results are not false and increase retries and delays --- tests/Support/Helper/KitAPI.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/Support/Helper/KitAPI.php b/tests/Support/Helper/KitAPI.php index 468f75ca3..2d0db65fd 100644 --- a/tests/Support/Helper/KitAPI.php +++ b/tests/Support/Helper/KitAPI.php @@ -71,6 +71,7 @@ function () use ($emailAddress) { ); // Check at least one subscriber was returned and it matches the email address. + $I->assertNotFalse($results); $I->assertGreaterThan(0, $results['pagination']['total_count']); $I->assertEquals($emailAddress, $results['subscribers'][0]['email_address']); @@ -351,7 +352,7 @@ public function apiRequest($endpoint, $method = 'GET', $params = array()) * @return mixed The truthy value returned by $callback, or * false if all attempts are exhausted. */ - private function retryUntil(callable $callback, $attempts = 3, $delay = 2) + private function retryUntil(callable $callback, $attempts = 4, $delay = 3) { for ($i = 0; $i < $attempts; $i++) { $result = $callback(); From 9026b08687a4c541f8f811832ca681e876181c46 Mon Sep 17 00:00:00 2001 From: Tim Carr Date: Mon, 4 May 2026 14:28:36 +0800 Subject: [PATCH 5/7] Move wait() to `apiCheckSubscriberExists` and `apiCheckSubscriberDoesNotExist` --- .../PageBlockFormBuilderCest.php | 8 ------- .../other/ContactForm7FormCest.php | 15 ------------ .../integrations/other/ForminatorCest.php | 24 ------------------- tests/Support/Helper/KitAPI.php | 6 +++++ 4 files changed, 6 insertions(+), 47 deletions(-) diff --git a/tests/EndToEnd/forms/blocks-shortcodes/PageBlockFormBuilderCest.php b/tests/EndToEnd/forms/blocks-shortcodes/PageBlockFormBuilderCest.php index 2f02ca7da..2d9daacf0 100644 --- a/tests/EndToEnd/forms/blocks-shortcodes/PageBlockFormBuilderCest.php +++ b/tests/EndToEnd/forms/blocks-shortcodes/PageBlockFormBuilderCest.php @@ -155,7 +155,6 @@ public function testFormBuilderBlockWithDefaultConfiguration(EndToEndTester $I) $I->seeElementInDOM('button[type="submit"]'); // Confirm that the email address was added to Kit. - $I->wait(3); $I->apiCheckSubscriberExists( $I, emailAddress: $emailAddress, @@ -366,7 +365,6 @@ public function testFormBuilderBlockWithFormEnabled(EndToEndTester $I) // Confirm that the email address was added to Kit. $I->waitForElementVisible('.convertkit-form-builder-subscribed-message'); - $I->wait(3); $subscriber = $I->apiCheckSubscriberExists( $I, emailAddress: $emailAddress, @@ -471,7 +469,6 @@ public function testFormBuilderBlockWithTaggingEnabled(EndToEndTester $I) // Confirm that the email address was added to Kit. $I->waitForElementVisible('.convertkit-form-builder-subscribed-message'); - $I->wait(3); $subscriber = $I->apiCheckSubscriberExists( $I, emailAddress: $emailAddress, @@ -575,7 +572,6 @@ public function testFormBuilderBlockWithSequenceEnabled(EndToEndTester $I) // Confirm that the email address was added to Kit. $I->waitForElementVisible('.convertkit-form-builder-subscribed-message'); - $I->wait(3); $subscriber = $I->apiCheckSubscriberExists( $I, emailAddress: $emailAddress, @@ -710,7 +706,6 @@ public function testFormBuilderBlockWithCustomField(EndToEndTester $I) // Confirm that the email address was added to Kit. $I->waitForElementVisible('.convertkit-form-builder-subscribed-message'); - $I->wait(3); $subscriber = $I->apiCheckSubscriberExists( $I, emailAddress: $emailAddress, @@ -1033,7 +1028,6 @@ public function testFormBuilderWithRecaptchaEnabled(EndToEndTester $I) // Confirm that the email address was added to Kit. $I->waitForElementVisible('.convertkit-form-builder-subscribed-message'); - $I->wait(3); $I->apiCheckSubscriberExists( $I, emailAddress: $emailAddress, @@ -1096,7 +1090,6 @@ public function testFormBuilderWithRecaptchaEnabledAndHighMinimumScore(EndToEndT $I->click('div.wp-block-convertkit-form-builder button[type="submit"]'); // Confirm that the email address was not added to Kit, as reCAPTCHA score failed. - $I->wait(3); $I->apiCheckSubscriberDoesNotExist($I, $emailAddress); } @@ -1167,7 +1160,6 @@ public function testFormBuilderWithStoreEntriesEnabled(EndToEndTester $I) // Confirm that the email address was added to Kit. $I->waitForElementVisible('.convertkit-form-builder-subscribed-message'); - $I->wait(3); $subscriber = $I->apiCheckSubscriberExists( $I, emailAddress: $emailAddress, diff --git a/tests/EndToEnd/integrations/other/ContactForm7FormCest.php b/tests/EndToEnd/integrations/other/ContactForm7FormCest.php index e9a1cc2b4..cef401d2e 100644 --- a/tests/EndToEnd/integrations/other/ContactForm7FormCest.php +++ b/tests/EndToEnd/integrations/other/ContactForm7FormCest.php @@ -66,9 +66,6 @@ public function testSettingsContactForm7ToKitFormMapping(EndToEndTester $I) emailAddress: $emailAddress ); - // Wait for the API to update. - $I->wait(2); - // Confirm that the email address was added to Kit. $subscriber = $I->apiCheckSubscriberExists($I, $emailAddress); @@ -106,9 +103,6 @@ public function testSettingsContactForm7ToKitLegacyFormMapping(EndToEndTester $I emailAddress: $emailAddress ); - // Wait for the API to update. - $I->wait(2); - // Confirm that the email address was added to Kit. $I->apiCheckSubscriberExists($I, $emailAddress); } @@ -138,9 +132,6 @@ public function testSettingsContactForm7ToKitTagMapping(EndToEndTester $I) emailAddress: $emailAddress ); - // Wait for the API to update. - $I->wait(2); - // Confirm that the email address was added to Kit. $subscriber = $I->apiCheckSubscriberExists($I, $emailAddress); @@ -177,9 +168,6 @@ public function testSettingsContactForm7ToKitSequenceMapping(EndToEndTester $I) emailAddress: $emailAddress ); - // Wait for the API to update. - $I->wait(2); - // Confirm that the email address was added to Kit. $subscriber = $I->apiCheckSubscriberExists($I, $emailAddress); @@ -245,9 +233,6 @@ public function testSettingsContactForm7SubscribeOption(EndToEndTester $I) emailAddress: $emailAddress ); - // Wait for the API to update. - $I->wait(2); - // Confirm that the email address was added to Kit. $I->apiCheckSubscriberExists($I, $emailAddress); } diff --git a/tests/EndToEnd/integrations/other/ForminatorCest.php b/tests/EndToEnd/integrations/other/ForminatorCest.php index a37fdb5d1..b33200ed9 100644 --- a/tests/EndToEnd/integrations/other/ForminatorCest.php +++ b/tests/EndToEnd/integrations/other/ForminatorCest.php @@ -66,9 +66,6 @@ public function testSettingsForminatorFormToKitFormMapping(EndToEndTester $I) emailAddress: $emailAddress ); - // Wait for the API to update. - $I->wait(2); - // Confirm that the email address was added to Kit. $subscriber = $I->apiCheckSubscriberExists($I, $emailAddress); @@ -106,9 +103,6 @@ public function testSettingsForminatorFormToKitLegacyFormMapping(EndToEndTester emailAddress: $emailAddress ); - // Wait for the API to update. - $I->wait(2); - // Confirm that the email address was added to Kit. $I->apiCheckSubscriberExists($I, $emailAddress); } @@ -138,9 +132,6 @@ public function testSettingsForminatorFormToKitTagMapping(EndToEndTester $I) emailAddress: $emailAddress ); - // Wait for the API to update. - $I->wait(2); - // Confirm that the email address was added to Kit. $subscriber = $I->apiCheckSubscriberExists($I, $emailAddress); @@ -177,9 +168,6 @@ public function testSettingsForminatorFormToKitSequenceMapping(EndToEndTester $I emailAddress: $emailAddress ); - // Wait for the API to update. - $I->wait(2); - // Confirm that the email address was added to Kit. $subscriber = $I->apiCheckSubscriberExists($I, $emailAddress); @@ -270,9 +258,6 @@ public function testSettingsForminatorQuizToKitFormMapping(EndToEndTester $I) emailAddress: $emailAddress ); - // Wait for the API to update. - $I->wait(2); - // Confirm that the email address was added to Kit. $I->apiCheckSubscriberExists($I, $emailAddress); } @@ -302,9 +287,6 @@ public function testSettingsForminatorQuizToKitTagMapping(EndToEndTester $I) emailAddress: $emailAddress ); - // Wait for the API to update. - $I->wait(2); - // Confirm that the email address was added to Kit. $subscriber = $I->apiCheckSubscriberExists($I, $emailAddress); @@ -341,9 +323,6 @@ public function testSettingsForminatorQuizToKitSequenceMapping(EndToEndTester $I emailAddress: $emailAddress ); - // Wait for the API to update. - $I->wait(2); - // Confirm that the email address was added to Kit. $subscriber = $I->apiCheckSubscriberExists($I, $emailAddress); @@ -409,9 +388,6 @@ public function testSettingsForminatorQuizSubscribeOption(EndToEndTester $I) emailAddress: $emailAddress ); - // Wait for the API to update. - $I->wait(2); - // Confirm that the email address was added to Kit. $I->apiCheckSubscriberExists($I, $emailAddress); } diff --git a/tests/Support/Helper/KitAPI.php b/tests/Support/Helper/KitAPI.php index 2d0db65fd..5d69eba9e 100644 --- a/tests/Support/Helper/KitAPI.php +++ b/tests/Support/Helper/KitAPI.php @@ -49,6 +49,9 @@ public function apiEncodeState($returnTo, $clientID) */ public function apiCheckSubscriberExists($I, $emailAddress, $firstName = false) { + // Wait for the API to update. + $I->wait(3); + // Retry the API request as sometimes there's a lag before the subscriber is queryable via the API. $results = $this->retryUntil( function () use ($emailAddress) { @@ -201,6 +204,9 @@ public function apiCheckSubscriberHasNoTags($I, $subscriberID) */ public function apiCheckSubscriberDoesNotExist($I, $emailAddress) { + // Wait for the API to update. + $I->wait(3); + // Run request. $results = $this->apiRequest( 'subscribers', From 4858465ae81607975765fba2708c8f52c4f8ea46 Mon Sep 17 00:00:00 2001 From: Tim Carr Date: Mon, 4 May 2026 15:02:36 +0800 Subject: [PATCH 6/7] Move Elementor tests to own folder --- .../integrations/{other => elementor}/ElementorBroadcastsCest.php | 0 .../integrations/{other => elementor}/ElementorFormCest.php | 0 .../{other => elementor}/ElementorFormTriggerCest.php | 0 .../integrations/{other => elementor}/ElementorProductCest.php | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename tests/EndToEnd/integrations/{other => elementor}/ElementorBroadcastsCest.php (100%) rename tests/EndToEnd/integrations/{other => elementor}/ElementorFormCest.php (100%) rename tests/EndToEnd/integrations/{other => elementor}/ElementorFormTriggerCest.php (100%) rename tests/EndToEnd/integrations/{other => elementor}/ElementorProductCest.php (100%) diff --git a/tests/EndToEnd/integrations/other/ElementorBroadcastsCest.php b/tests/EndToEnd/integrations/elementor/ElementorBroadcastsCest.php similarity index 100% rename from tests/EndToEnd/integrations/other/ElementorBroadcastsCest.php rename to tests/EndToEnd/integrations/elementor/ElementorBroadcastsCest.php diff --git a/tests/EndToEnd/integrations/other/ElementorFormCest.php b/tests/EndToEnd/integrations/elementor/ElementorFormCest.php similarity index 100% rename from tests/EndToEnd/integrations/other/ElementorFormCest.php rename to tests/EndToEnd/integrations/elementor/ElementorFormCest.php diff --git a/tests/EndToEnd/integrations/other/ElementorFormTriggerCest.php b/tests/EndToEnd/integrations/elementor/ElementorFormTriggerCest.php similarity index 100% rename from tests/EndToEnd/integrations/other/ElementorFormTriggerCest.php rename to tests/EndToEnd/integrations/elementor/ElementorFormTriggerCest.php diff --git a/tests/EndToEnd/integrations/other/ElementorProductCest.php b/tests/EndToEnd/integrations/elementor/ElementorProductCest.php similarity index 100% rename from tests/EndToEnd/integrations/other/ElementorProductCest.php rename to tests/EndToEnd/integrations/elementor/ElementorProductCest.php From 9ec23da5489a6e5462748e139c64b50ff1d992d9 Mon Sep 17 00:00:00 2001 From: Tim Carr Date: Mon, 4 May 2026 16:37:01 +0800 Subject: [PATCH 7/7] Reinstate all tests --- .github/workflows/tests.yml | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1805abd6a..aadff041b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -93,8 +93,26 @@ jobs: # Folder names within the 'tests' folder to run tests in parallel. test-groups: [ + 'EndToEnd/broadcasts/blocks-shortcodes', + 'EndToEnd/broadcasts/import-export', + 'EndToEnd/forms/blocks-shortcodes', + 'EndToEnd/forms/general', + 'EndToEnd/forms/post-types', + 'EndToEnd/general/other', + 'EndToEnd/general/uninstall', + 'EndToEnd/general/plugin-screens', + 'EndToEnd/integrations/divi-builder', + 'EndToEnd/integrations/divi-theme', + 'EndToEnd/integrations/elementor', 'EndToEnd/integrations/other', - 'EndToEnd/integrations/wlm' + 'EndToEnd/integrations/wlm', + 'EndToEnd/integrations/woocommerce', + 'EndToEnd/landing-pages', + 'EndToEnd/products', + 'EndToEnd/restrict-content/general', + 'EndToEnd/restrict-content/post-types', + 'EndToEnd/tags', + 'Integration' ] # Steps to install, configure and run tests