Load posts in chunks in wp post list

features/post.feature

@@ -1152,3 +1152,46 @@ Feature: Manage WordPress posts
       2021-02-02
       """
     And STDOUT should not be empty
+
+  Scenario: List more posts than are loaded from the database at a time
+    Given a WP install
+    And I run `wp post generate --count=1200 --post_date="2020-01-01 00:00:00"`
+    And I run `wp post list --post_status=publish --orderby=ID --order=asc --format=ids`
+    And save STDOUT as {IDS}
+
+    When I run `wp post list --post_status=publish --orderby=ID --order=asc --field=ID | tr '\n' ' ' | sed 's/ $//'`
+    Then STDOUT should be:
+      """
+      {IDS}
+      """
+
+    When I run `wp post list --post_status=publish --orderby=ID --order=asc --fields=ID,post_title --format=csv | wc -l`
+    Then STDOUT should contain:
+      """
+      1202
+      """
+
+    When I run `wp post list --post_status=publish --orderby=ID --order=desc --posts_per_page=2 --fields=ID,url --format=csv`
+    Then STDOUT should match /^ID,url\n(\d+),https?:\/\/example\.com\/\?p=\1\n(\d+),https?:\/\/example\.com\/\?p=\2$/
+
+  Scenario: List post meta as a field for some posts only
+    Given a WP install
+    And I run `wp post create --post_title=First --post_status=publish --porcelain`
+    And save STDOUT as {FIRST}
+    And I run `wp post create --post_title=Second --post_status=publish --porcelain`
+    And save STDOUT as {SECOND}
+    And I run `wp post meta add {SECOND} batch_meta second-value`
+
+    When I run `wp post list --post__in={FIRST},{SECOND} --orderby=ID --order=asc --fields=ID,batch_meta --format=csv`
+    Then STDOUT should be:
+      """
+      ID,batch_meta
+      {FIRST},
+      {SECOND},second-value
+      """
+
+    When I try `wp post list --post__in={FIRST},{SECOND} --fields=ID,missing_meta --format=csv`
+    Then STDERR should be:
+      """
+      Warning: Field not found in any item: missing_meta.
+      """

src/Post_Command.php

@@ -27,6 +27,11 @@
  */
 class Post_Command extends CommandWithDBObject {
 
+	/**
+	 * Number of posts `wp post list` loads from the database at a time.
+	 */
+	const LIST_CHUNK_SIZE = 500;
+
 	protected $obj_type   = 'post';
 	protected $obj_fields = [
 		'ID',
@@ -1057,20 +1062,79 @@ public function list_( $args, $assoc_args ) {
 			$query                = new WP_Query( $query_args );
 			$formatter->display_items( $query->posts ?? [] );
 		} else {
-			$query = new WP_Query( $query_args );
-			$posts = array_map(
-				function ( $post ) {
-					/**
-					 * @var \WP_Post $post
-					 */
+			$need_url = 'url' === $formatter->field || in_array( 'url', (array) $formatter->fields, true );
+			$formatter->display_items( $this->query_posts_in_chunks( $query_args, $need_url ) );
+		}
+	}
+
+	/**
+	 * Query posts in chunks, so that they don't all have to be held in memory at once.
+	 *
+	 * The IDs of all matching posts are queried first. The posts are then loaded a chunk
+	 * at a time with the same query arguments, and the object cache is cleared after each
+	 * chunk. Formats that can be written item by item are then streamed by the formatter.
+	 *
+	 * Fields that WP_Post computes on access, like post meta, are read from the object
+	 * cache. The formatter reads the requested fields of each post while it is the current
+	 * item, before the cache of its chunk is cleared, even when it can't stream them.
+	 *
+	 * @param array<string, mixed> $query_args WP_Query arguments.
+	 * @param bool                 $need_url   Whether to add the `url` property to each post.
+	 * @return \Generator<int, \WP_Post>
+	 */
+	private function query_posts_in_chunks( $query_args, $need_url ) {
+		/** @var int[] $ids */
+		$ids = ( new WP_Query( array_merge( $query_args, [ 'fields' => 'ids' ] ) ) )->posts;
+
+		$chunk_args = $query_args;
+		unset( $chunk_args['offset'], $chunk_args['paged'], $chunk_args['page'], $chunk_args['nopaging'] );
+		// The order is restored from $ids below. Ordering by `post__in` in SQL would
+		// need a FIELD() call with one argument per post, which SQLite does not allow.
+		$chunk_args['orderby']             = 'none';
+		$chunk_args['no_found_rows']       = true;
+		$chunk_args['ignore_sticky_posts'] = true;
+
+		foreach ( array_chunk( $ids, self::LIST_CHUNK_SIZE ) as $chunk ) {
+			$chunk_args['post__in']       = $chunk;
+			$chunk_args['posts_per_page'] = count( $chunk );
+
+			$query = new WP_Query( $chunk_args );
+
+			$posts = [];
+			foreach ( $query->posts ?? [] as $post ) {
+				if ( $post instanceof \WP_Post ) {
+					$posts[ $post->ID ] = $post;
+				}
+			}
+
+			foreach ( $chunk as $id ) {
+				if ( ! isset( $posts[ $id ] ) ) {
+					continue;
+				}
 
+				$post = $posts[ $id ];
+				if ( $need_url ) {
 					// @phpstan-ignore property.notFound
-					$post->url = get_permalink( $post->ID );
-					return $post;
-				},
-				$query->posts ?? []
-			);
-			$formatter->display_items( $posts );
+					$post->url = get_permalink( $post );
+				}
+				yield $post;
+			}
+
+			unset( $posts, $query );
+			self::clear_runtime_object_cache();
+		}
+	}
+
+	/**
+	 * Free the memory held by the in-process object cache.
+	 *
+	 * Persistent object caches are left alone unless they can flush their in-process part only.
+	 */
+	private static function clear_runtime_object_cache() {
+		if ( function_exists( 'wp_cache_flush_runtime' ) && function_exists( 'wp_cache_supports' ) && wp_cache_supports( 'flush_runtime' ) ) {
+			wp_cache_flush_runtime();
+		} elseif ( ! wp_using_ext_object_cache() ) {
+			wp_cache_flush();
 		}
 	}