From 5161ba1f152172fe310e11d4681e26b62cf15f08 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 5 Oct 2026 19:33:25 +0000 Subject: [PATCH 1/4] Load posts in chunks in `wp post list` `wp post list` loaded every matching post, including its full content, in a single query, primed their meta and term caches, and computed the permalink of every post even when the `url` field was not requested. On a site with ~100k posts this used ~900 MB of memory regardless of the requested fields. Query the IDs first, then load the posts in chunks with the same query arguments, clearing the object cache after each chunk, and only compute `url` when it is requested. The posts are passed to the formatter as a generator, which newer WP-CLI versions stream for CSV and JSON. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01D26yjkN2BiqCXT6p6o1WqS --- features/post.feature | 21 +++++++++++++ src/Post_Command.php | 71 +++++++++++++++++++++++++++++++++++-------- 2 files changed, 80 insertions(+), 12 deletions(-) diff --git a/features/post.feature b/features/post.feature index 313260c4f..c18e6f034 100644 --- a/features/post.feature +++ b/features/post.feature @@ -1152,3 +1152,24 @@ Feature: Manage WordPress posts 2021-02-02 """ And STDOUT should not be empty + + Scenario: List more posts than are loaded from the database at a time + Given a WP install + And I run `wp post generate --count=1200 --post_date="2020-01-01 00:00:00"` + And I run `wp post list --post_status=publish --orderby=ID --order=asc --format=ids` + And save STDOUT as {IDS} + + When I run `wp post list --post_status=publish --orderby=ID --order=asc --field=ID | tr '\n' ' ' | sed 's/ $//'` + Then STDOUT should be: + """ + {IDS} + """ + + When I run `wp post list --post_status=publish --orderby=ID --order=asc --fields=ID,post_title --format=csv | wc -l` + Then STDOUT should contain: + """ + 1202 + """ + + When I run `wp post list --post_status=publish --orderby=ID --order=desc --posts_per_page=2 --fields=ID,url --format=csv` + Then STDOUT should match /^ID,url\n(\d+),https?:\/\/example\.com\/\?p=\1\n(\d+),https?:\/\/example\.com\/\?p=\2$/ diff --git a/src/Post_Command.php b/src/Post_Command.php index b314e31b1..48c2a9a57 100644 --- a/src/Post_Command.php +++ b/src/Post_Command.php @@ -27,6 +27,11 @@ */ class Post_Command extends CommandWithDBObject { + /** + * Number of posts `wp post list` loads from the database at a time. + */ + const LIST_CHUNK_SIZE = 500; + protected $obj_type = 'post'; protected $obj_fields = [ 'ID', @@ -1057,20 +1062,62 @@ public function list_( $args, $assoc_args ) { $query = new WP_Query( $query_args ); $formatter->display_items( $query->posts ?? [] ); } else { - $query = new WP_Query( $query_args ); - $posts = array_map( - function ( $post ) { - /** - * @var \WP_Post $post - */ + $need_url = 'url' === $formatter->field || in_array( 'url', (array) $formatter->fields, true ); + $formatter->display_items( $this->query_posts_in_chunks( $query_args, $need_url ) ); + } + } + /** + * Query posts in chunks, so that they don't all have to be held in memory at once. + * + * The IDs of all matching posts are queried first. The posts are then loaded a chunk + * at a time with the same query arguments, and the object cache is cleared after each + * chunk. Formats that can be written item by item are then streamed by the formatter. + * + * @param array $query_args WP_Query arguments. + * @param bool $need_url Whether to add the `url` property to each post. + * @return \Generator + */ + private function query_posts_in_chunks( $query_args, $need_url ) { + /** @var int[] $ids */ + $ids = ( new WP_Query( array_merge( $query_args, [ 'fields' => 'ids' ] ) ) )->posts; + + $chunk_args = $query_args; + unset( $chunk_args['offset'], $chunk_args['paged'], $chunk_args['page'], $chunk_args['nopaging'] ); + // The order is restored from $ids below. Ordering by `post__in` in SQL would + // need a FIELD() call with one argument per post, which SQLite does not allow. + $chunk_args['orderby'] = 'none'; + $chunk_args['no_found_rows'] = true; + $chunk_args['ignore_sticky_posts'] = true; + + foreach ( array_chunk( $ids, self::LIST_CHUNK_SIZE ) as $chunk ) { + $chunk_args['post__in'] = $chunk; + $chunk_args['posts_per_page'] = count( $chunk ); + + $query = new WP_Query( $chunk_args ); + + $posts = []; + foreach ( $query->posts as $post ) { + if ( $post instanceof \WP_Post ) { + $posts[ $post->ID ] = $post; + } + } + + foreach ( $chunk as $id ) { + if ( ! isset( $posts[ $id ] ) ) { + continue; + } + + $post = $posts[ $id ]; + if ( $need_url ) { // @phpstan-ignore property.notFound - $post->url = get_permalink( $post->ID ); - return $post; - }, - $query->posts ?? [] - ); - $formatter->display_items( $posts ); + $post->url = get_permalink( $post ); + } + yield $post; + } + + unset( $posts, $query ); + Utils\wp_clear_object_cache(); // phpcs:ignore PHPCompatibility.FunctionUse.RemovedFunctions.wp_clear_object_cacheDeprecatedRemoved @phpstan-ignore-line } } From f78534de3b8244418d298e58f8ccd83b9f265a45 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 5 Oct 2026 20:17:31 +0000 Subject: [PATCH 2/4] Fix PHPStan findings, avoid the deprecated wp_clear_object_cache() Guard against WP_Query::$posts being null, and free the in-process object cache directly instead of through the deprecated helper, whose deprecation is reported differently depending on the PHPStan setup. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01D26yjkN2BiqCXT6p6o1WqS --- src/Post_Command.php | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/Post_Command.php b/src/Post_Command.php index 48c2a9a57..8b9698bf4 100644 --- a/src/Post_Command.php +++ b/src/Post_Command.php @@ -1097,7 +1097,7 @@ private function query_posts_in_chunks( $query_args, $need_url ) { $query = new WP_Query( $chunk_args ); $posts = []; - foreach ( $query->posts as $post ) { + foreach ( $query->posts ?? [] as $post ) { if ( $post instanceof \WP_Post ) { $posts[ $post->ID ] = $post; } @@ -1117,7 +1117,20 @@ private function query_posts_in_chunks( $query_args, $need_url ) { } unset( $posts, $query ); - Utils\wp_clear_object_cache(); // phpcs:ignore PHPCompatibility.FunctionUse.RemovedFunctions.wp_clear_object_cacheDeprecatedRemoved @phpstan-ignore-line + self::clear_runtime_object_cache(); + } + } + + /** + * Free the memory held by the in-process object cache. + * + * Persistent object caches are left alone unless they can flush their in-process part only. + */ + private static function clear_runtime_object_cache() { + if ( function_exists( 'wp_cache_flush_runtime' ) && function_exists( 'wp_cache_supports' ) && wp_cache_supports( 'flush_runtime' ) ) { + wp_cache_flush_runtime(); + } elseif ( ! wp_using_ext_object_cache() ) { + wp_cache_flush(); } } From c3563e64358b82375e29eb57b515c478ef82d6b3 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 6 Oct 2026 15:54:35 +0000 Subject: [PATCH 3/4] Keep the object cache when listing computed post fields Fields like post meta are read through WP_Post::__get(), which uses the object cache. When the first post lacks such a field, the formatter can't stream and only reads the fields once all posts are loaded. By then the cache was cleared after each chunk, so every post needed another query, which made `wp post list --fields=ID,` slower than before. Only clear the cache when all displayed fields are plain post properties. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01D26yjkN2BiqCXT6p6o1WqS --- features/post.feature | 22 ++++++++++++++++++++++ src/Post_Command.php | 24 +++++++++++++++++++----- 2 files changed, 41 insertions(+), 5 deletions(-) diff --git a/features/post.feature b/features/post.feature index c18e6f034..5d5a9ac4e 100644 --- a/features/post.feature +++ b/features/post.feature @@ -1173,3 +1173,25 @@ Feature: Manage WordPress posts When I run `wp post list --post_status=publish --orderby=ID --order=desc --posts_per_page=2 --fields=ID,url --format=csv` Then STDOUT should match /^ID,url\n(\d+),https?:\/\/example\.com\/\?p=\1\n(\d+),https?:\/\/example\.com\/\?p=\2$/ + + Scenario: List post meta as a field for some posts only + Given a WP install + And I run `wp post create --post_title=First --post_status=publish --porcelain` + And save STDOUT as {FIRST} + And I run `wp post create --post_title=Second --post_status=publish --porcelain` + And save STDOUT as {SECOND} + And I run `wp post meta add {SECOND} batch_meta second-value` + + When I run `wp post list --post__in={FIRST},{SECOND} --orderby=ID --order=asc --fields=ID,batch_meta --format=csv` + Then STDOUT should be: + """ + ID,batch_meta + {FIRST}, + {SECOND},second-value + """ + + When I try `wp post list --post__in={FIRST},{SECOND} --fields=ID,missing_meta --format=csv` + Then STDERR should be: + """ + Warning: Field not found in any item: missing_meta. + """ diff --git a/src/Post_Command.php b/src/Post_Command.php index 8b9698bf4..22c569001 100644 --- a/src/Post_Command.php +++ b/src/Post_Command.php @@ -1062,8 +1062,8 @@ public function list_( $args, $assoc_args ) { $query = new WP_Query( $query_args ); $formatter->display_items( $query->posts ?? [] ); } else { - $need_url = 'url' === $formatter->field || in_array( 'url', (array) $formatter->fields, true ); - $formatter->display_items( $this->query_posts_in_chunks( $query_args, $need_url ) ); + $fields = $formatter->field ? [ $formatter->field ] : (array) $formatter->fields; + $formatter->display_items( $this->query_posts_in_chunks( $query_args, $fields ) ); } } @@ -1074,11 +1074,23 @@ public function list_( $args, $assoc_args ) { * at a time with the same query arguments, and the object cache is cleared after each * chunk. Formats that can be written item by item are then streamed by the formatter. * + * Fields that WP_Post computes on access, like post meta, are read from the object + * cache. If any of them are displayed, the cache is kept, because the formatter may + * only read them once all posts have been loaded. + * * @param array $query_args WP_Query arguments. - * @param bool $need_url Whether to add the `url` property to each post. + * @param string[] $fields Fields that will be displayed. * @return \Generator */ - private function query_posts_in_chunks( $query_args, $need_url ) { + private function query_posts_in_chunks( $query_args, $fields ) { + $need_url = in_array( 'url', $fields, true ); + $clear_cache = true; + foreach ( $fields as $field ) { + if ( 'url' !== $field && ! property_exists( 'WP_Post', $field ) && ! property_exists( 'WP_Post', 'post_' . $field ) ) { + $clear_cache = false; + } + } + /** @var int[] $ids */ $ids = ( new WP_Query( array_merge( $query_args, [ 'fields' => 'ids' ] ) ) )->posts; @@ -1117,7 +1129,9 @@ private function query_posts_in_chunks( $query_args, $need_url ) { } unset( $posts, $query ); - self::clear_runtime_object_cache(); + if ( $clear_cache ) { + self::clear_runtime_object_cache(); + } } } From 922c519fc44a7593406752f36ed20db90b50f7b0 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 7 Oct 2026 14:24:51 +0000 Subject: [PATCH 4/4] Always clear the object cache after each chunk of posts wp-cli/wp-cli#6426 makes the formatter read the requested fields of each item while it is current, also when it can't stream them. Post meta is therefore read before the cache of its chunk is cleared, so the cache no longer needs to be kept when computed fields are listed. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01D26yjkN2BiqCXT6p6o1WqS --- src/Post_Command.php | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/src/Post_Command.php b/src/Post_Command.php index 22c569001..4fa14359c 100644 --- a/src/Post_Command.php +++ b/src/Post_Command.php @@ -1062,8 +1062,8 @@ public function list_( $args, $assoc_args ) { $query = new WP_Query( $query_args ); $formatter->display_items( $query->posts ?? [] ); } else { - $fields = $formatter->field ? [ $formatter->field ] : (array) $formatter->fields; - $formatter->display_items( $this->query_posts_in_chunks( $query_args, $fields ) ); + $need_url = 'url' === $formatter->field || in_array( 'url', (array) $formatter->fields, true ); + $formatter->display_items( $this->query_posts_in_chunks( $query_args, $need_url ) ); } } @@ -1075,22 +1075,14 @@ public function list_( $args, $assoc_args ) { * chunk. Formats that can be written item by item are then streamed by the formatter. * * Fields that WP_Post computes on access, like post meta, are read from the object - * cache. If any of them are displayed, the cache is kept, because the formatter may - * only read them once all posts have been loaded. + * cache. The formatter reads the requested fields of each post while it is the current + * item, before the cache of its chunk is cleared, even when it can't stream them. * * @param array $query_args WP_Query arguments. - * @param string[] $fields Fields that will be displayed. + * @param bool $need_url Whether to add the `url` property to each post. * @return \Generator */ - private function query_posts_in_chunks( $query_args, $fields ) { - $need_url = in_array( 'url', $fields, true ); - $clear_cache = true; - foreach ( $fields as $field ) { - if ( 'url' !== $field && ! property_exists( 'WP_Post', $field ) && ! property_exists( 'WP_Post', 'post_' . $field ) ) { - $clear_cache = false; - } - } - + private function query_posts_in_chunks( $query_args, $need_url ) { /** @var int[] $ids */ $ids = ( new WP_Query( array_merge( $query_args, [ 'fields' => 'ids' ] ) ) )->posts; @@ -1129,9 +1121,7 @@ private function query_posts_in_chunks( $query_args, $fields ) { } unset( $posts, $query ); - if ( $clear_cache ) { - self::clear_runtime_object_cache(); - } + self::clear_runtime_object_cache(); } }