Load comments and users in chunks in wp comment list and wp user list

features/comment.feature

@@ -475,3 +475,24 @@ Feature: Manage WordPress comments
     And I run `wp comment unspam {COMMENT_ID} --url=www.example.com`
     And I run `wp comment trash {COMMENT_ID} --url=www.example.com`
     And I run `wp comment untrash {COMMENT_ID} --url=www.example.com`
+
+  Scenario: List more comments than are loaded from the database at a time
+    Given a WP install
+    And I run `wp comment generate --count=1200 --post_id=1`
+    And I run `wp comment list --orderby=comment_ID --order=asc --format=ids`
+    And save STDOUT as {IDS}
+
+    When I run `wp comment list --orderby=comment_ID --order=asc --field=comment_ID | tr '\n' ' ' | sed 's/ $//'`
+    Then STDOUT should be:
+      """
+      {IDS}
+      """
+
+    When I run `wp comment list --fields=comment_ID,comment_post_ID --format=csv | wc -l`
+    Then STDOUT should contain:
+      """
+      1202
+      """
+
+    When I run `wp comment list --orderby=comment_ID --order=asc --number=1 --fields=comment_ID,url --format=csv`
+    Then STDOUT should match /^comment_ID,url\n(\d+),https?:\/\/example\.com\/\?p=1#comment-\1$/

features/user.feature

@@ -1035,3 +1035,30 @@ Feature: Manage WordPress users
       """
       true
       """
+
+  Scenario: List more users than are loaded from the database at a time
+    Given a WP install
+    And I run `wp user generate --count=600`
+    And I run `wp user meta update 1 batch_meta admin-value`
+    And I run `wp user list --orderby=ID --order=asc --format=ids`
+    And save STDOUT as {IDS}
+
+    When I run `wp user list --orderby=ID --order=asc --field=ID | tr '\n' ' ' | sed 's/ $//'`
+    Then STDOUT should be:
+      """
+      {IDS}
+      """
+
+    When I run `wp user list --fields=ID,user_login,roles --format=csv | wc -l`
+    Then STDOUT should contain:
+      """
+      602
+      """
+
+    When I run `wp user list --orderby=ID --order=asc --number=2 --fields=ID,batch_meta,roles --format=csv`
+    Then STDOUT should be:
+      """
+      ID,batch_meta,roles
+      1,admin-value,administrator
+      2,,subscriber
+      """

src/Comment_Command.php

@@ -520,6 +520,14 @@ public function list_( $args, $assoc_args ) {
 			$assoc_args['count'] = true;
 		}
 
+		$need_url = 'url' === $formatter->field || in_array( 'url', (array) $formatter->fields, true );
+
+		// Threaded results are nested, so they can't be loaded in chunks.
+		if ( ! in_array( $formatter->format, [ 'count', 'ids' ], true ) && empty( $assoc_args['hierarchical'] ) ) {
+			$formatter->display_items( $this->query_comments_in_chunks( $assoc_args, $need_url ) );
+			return;
+		}
+
 		$query    = new WP_Comment_Query();
 		$comments = $query->query( $assoc_args );
 
@@ -541,7 +549,7 @@ public function list_( $args, $assoc_args ) {
 				$items = wp_list_pluck( $comments, 'comment_ID' );
 
 				$comments = $items;
-			} elseif ( is_array( $comments ) ) {
+			} elseif ( is_array( $comments ) && $need_url ) {
 				$comments = array_map(
 					function ( $comment ) {
 						/**
@@ -558,6 +566,78 @@ function ( $comment ) {
 		}
 	}
 
+	/**
+	 * Query comments in chunks, so that they don't all have to be held in memory at once.
+	 *
+	 * The IDs of all matching comments are queried first. The comments are then loaded
+	 * a chunk at a time, and the object cache is cleared after each chunk. Formats that
+	 * can be written item by item are then streamed by the formatter.
+	 *
+	 * Fields that WP_Comment reads from the comment's post are read from the object cache.
+	 * The formatter reads the requested fields of each comment while it is the current
+	 * item, before the cache of its chunk is cleared, even when it can't stream them.
+	 *
+	 * @param array<string, mixed> $query_args WP_Comment_Query arguments.
+	 * @param bool                 $need_url   Whether to add the `url` property to each comment.
+	 * @return \Generator<int, \WP_Comment>
+	 */
+	private function query_comments_in_chunks( $query_args, $need_url ) {
+		$result = ( new WP_Comment_Query() )->query(
+			array_merge(
+				$query_args,
+				[
+					'fields' => 'ids',
+					'count'  => false,
+				]
+			)
+		);
+
+		$ids = [];
+		foreach ( is_array( $result ) ? $result : [] as $id ) {
+			if ( is_numeric( $id ) ) {
+				$ids[] = (int) $id;
+			}
+		}
+
+		$chunk_args = $query_args;
+		unset( $chunk_args['number'], $chunk_args['offset'], $chunk_args['paged'] );
+		// The order is restored from $ids below. Ordering by `comment__in` in SQL would
+		// need a FIELD() call with one argument per comment, which SQLite does not allow.
+		$chunk_args['orderby']       = 'none';
+		$chunk_args['fields']        = '';
+		$chunk_args['count']         = false;
+		$chunk_args['no_found_rows'] = true;
+		// Load the posts of each chunk in one query, as fields like `post_title` read them.
+		$chunk_args['update_comment_post_cache'] = true;
+
+		foreach ( array_chunk( $ids, self::LIST_CHUNK_SIZE ) as $chunk ) {
+			$chunk_args['comment__in'] = $chunk;
+
+			$comments = [];
+			foreach ( (array) ( new WP_Comment_Query() )->query( $chunk_args ) as $comment ) {
+				if ( $comment instanceof \WP_Comment ) {
+					$comments[ (int) $comment->comment_ID ] = $comment;
+				}
+			}
+
+			foreach ( $chunk as $id ) {
+				if ( ! isset( $comments[ (int) $id ] ) ) {
+					continue;
+				}
+
+				$comment = $comments[ (int) $id ];
+				if ( $need_url ) {
+					// @phpstan-ignore property.notFound
+					$comment->url = get_comment_link( $comment );
+				}
+				yield $comment;
+			}
+
+			unset( $comments );
+			self::clear_runtime_object_cache();
+		}
+	}
+
 	/**
 	 * Deletes a comment.
 	 *

src/Post_Command.php

@@ -27,11 +27,6 @@
  */
 class Post_Command extends CommandWithDBObject {
 
-	/**
-	 * Number of posts `wp post list` loads from the database at a time.
-	 */
-	const LIST_CHUNK_SIZE = 500;
-
 	protected $obj_type   = 'post';
 	protected $obj_fields = [
 		'ID',
@@ -1125,19 +1120,6 @@ private function query_posts_in_chunks( $query_args, $need_url ) {
 		}
 	}
 
-	/**
-	 * Free the memory held by the in-process object cache.
-	 *
-	 * Persistent object caches are left alone unless they can flush their in-process part only.
-	 */
-	private static function clear_runtime_object_cache() {
-		if ( function_exists( 'wp_cache_flush_runtime' ) && function_exists( 'wp_cache_supports' ) && wp_cache_supports( 'flush_runtime' ) ) {
-			wp_cache_flush_runtime();
-		} elseif ( ! wp_using_ext_object_cache() ) {
-			wp_cache_flush();
-		}
-	}
-
 	/**
 	 * Generates some posts.
 	 *

src/Term_Command.php

@@ -161,13 +161,17 @@ public function list_( $args, $assoc_args ) {
 			}
 		}
 
+		$need_url = 'url' === $formatter->field || in_array( 'url', (array) $formatter->fields, true );
+
 		$terms = array_map(
-			function ( $term ) {
+			function ( $term ) use ( $need_url ) {
 					$term->count  = (int) $term->count;
 					$term->parent = (int) $term->parent;
 
+				if ( $need_url ) {
 					// @phpstan-ignore property.notFound
 					$term->url = get_term_link( $term );
+				}
 					return $term;
 			},
 			$terms

src/User_Command.php

@@ -172,33 +172,66 @@ public function list_( $args, $assoc_args ) {
 			}
 		}
 
-		$users = get_users( $assoc_args );
-
 		if ( 'ids' === $formatter->format ) {
-			echo implode( ' ', $users );
+			echo implode( ' ', get_users( $assoc_args ) );
 		} elseif ( 'count' === $formatter->format ) {
-			$formatter->display_items( $users );
+			$formatter->display_items( get_users( $assoc_args ) );
 		} else {
-			$iterator = Utils\iterator_map(
-				$users,
-				function ( $user ) {
-					if ( ! is_object( $user ) ) {
-						return $user;
-					}
+			$need_url = 'url' === $formatter->field || in_array( 'url', (array) $formatter->fields, true );
+			$formatter->display_items( $this->query_users_in_chunks( $assoc_args, $need_url ) );
+		}
+	}
+
+	/**
+	 * Query users in chunks, so that they don't all have to be held in memory at once.
+	 *
+	 * The IDs of all matching users are queried first. The users and their meta are then
+	 * loaded a chunk at a time, and the object cache is cleared after each chunk. Formats
+	 * that can be written item by item are then streamed by the formatter.
+	 *
+	 * User meta is read from the object cache. The formatter reads the requested fields of
+	 * each user while it is the current item, before the cache of its chunk is cleared,
+	 * even when it can't stream them.
+	 *
+	 * @param array $query_args WP_User_Query arguments.
+	 * @param bool  $need_url   Whether to add the `url` property to each user.
+	 * @return \Generator<int, \WP_User>
+	 */
+	private function query_users_in_chunks( $query_args, $need_url ) {
+		$ids = array_map( 'intval', (array) get_users( array_merge( $query_args, [ 'fields' => 'ids' ] ) ) );
 
-					/**
-					 * @var \WP_User $user
-					 */
+		$chunk_args = $query_args;
+		unset( $chunk_args['number'], $chunk_args['offset'], $chunk_args['paged'] );
+		// The order is restored from $ids below.
+		$chunk_args['fields'] = 'all_with_meta';
 
-					// @phpstan-ignore assign.propertyType
-					$user->roles = implode( ',', $user->roles );
+		foreach ( array_chunk( $ids, self::LIST_CHUNK_SIZE ) as $chunk ) {
+			$chunk_args['include'] = $chunk;
+
+			$users = [];
+			foreach ( get_users( $chunk_args ) as $user ) {
+				if ( $user instanceof \WP_User ) {
+					$users[ $user->ID ] = $user;
+				}
+			}
+
+			foreach ( $chunk as $id ) {
+				if ( ! isset( $users[ $id ] ) ) {
+					continue;
+				}
+
+				$user = $users[ $id ];
+				// @phpstan-ignore assign.propertyType
+				$user->roles = implode( ',', $user->roles );
+				if ( $need_url ) {
 					// @phpstan-ignore property.notFound
 					$user->url = get_author_posts_url( $user->ID, $user->user_nicename );
-					return $user;
 				}
-			);
+				yield $user;
+			}
 
-			$formatter->display_items( $iterator );
+			unset( $users );
+			self::clear_runtime_object_cache();
 		}
 	}
 

src/WP_CLI/CommandWithDBObject.php

@@ -14,6 +14,11 @@
  */
 abstract class CommandWithDBObject extends WP_CLI_Command {
 
+	/**
+	 * Number of objects that list commands load from the database at a time.
+	 */
+	const LIST_CHUNK_SIZE = 500;
+
 	/**
 	 * @var string $object_type WordPress' expected name for the object.
 	 */
@@ -184,4 +189,18 @@ protected function get_formatter( &$assoc_args ) {
 		}
 		return new Formatter( $assoc_args, $fields, $this->obj_type );
 	}
+
+	/**
+	 * Free the memory held by the in-process object cache.
+	 *
+	 * List commands call this after each chunk of objects. Persistent object
+	 * caches are left alone unless they can flush their in-process part only.
+	 */
+	protected static function clear_runtime_object_cache() {
+		if ( function_exists( 'wp_cache_flush_runtime' ) && function_exists( 'wp_cache_supports' ) && wp_cache_supports( 'flush_runtime' ) ) {
+			wp_cache_flush_runtime();
+		} elseif ( ! wp_using_ext_object_cache() ) {
+			wp_cache_flush();
+		}
+	}
 }