From 5bd5d04e6dd659e43c330098cba5f972d2ac5806 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 6 Oct 2026 16:08:36 +0000 Subject: [PATCH 1/2] Load comments and users in chunks in `wp comment list` and `wp user list` Like `wp post list`, query the IDs of all matching comments or users first, then load the objects a chunk at a time and clear the object cache after each chunk. Combined with the formatter streaming CSV and JSON output for iterators, the memory use no longer grows with the number of items. If a displayed field is read through the object cache, like user meta or the post title of a comment, the cache is kept. Also only compute the `url` field of comments, users and terms when it is displayed. For comments it took about 40% of the time. The chunk size and the cache helper move to CommandWithDBObject. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01D26yjkN2BiqCXT6p6o1WqS --- features/comment.feature | 21 +++++++ features/user.feature | 27 +++++++++ src/Comment_Command.php | 91 +++++++++++++++++++++++++++++- src/Post_Command.php | 18 ------ src/Term_Command.php | 6 +- src/User_Command.php | 84 +++++++++++++++++++++------ src/WP_CLI/CommandWithDBObject.php | 19 +++++++ 7 files changed, 228 insertions(+), 38 deletions(-) diff --git a/features/comment.feature b/features/comment.feature index c51a5095a..ca7514d62 100644 --- a/features/comment.feature +++ b/features/comment.feature @@ -475,3 +475,24 @@ Feature: Manage WordPress comments And I run `wp comment unspam {COMMENT_ID} --url=www.example.com` And I run `wp comment trash {COMMENT_ID} --url=www.example.com` And I run `wp comment untrash {COMMENT_ID} --url=www.example.com` + + Scenario: List more comments than are loaded from the database at a time + Given a WP install + And I run `wp comment generate --count=1200 --post_id=1` + And I run `wp comment list --orderby=comment_ID --order=asc --format=ids` + And save STDOUT as {IDS} + + When I run `wp comment list --orderby=comment_ID --order=asc --field=comment_ID | tr '\n' ' ' | sed 's/ $//'` + Then STDOUT should be: + """ + {IDS} + """ + + When I run `wp comment list --fields=comment_ID,comment_post_ID --format=csv | wc -l` + Then STDOUT should contain: + """ + 1202 + """ + + When I run `wp comment list --orderby=comment_ID --order=asc --number=1 --fields=comment_ID,url --format=csv` + Then STDOUT should match /^comment_ID,url\n(\d+),https?:\/\/example\.com\/\?p=1#comment-\1$/ diff --git a/features/user.feature b/features/user.feature index ff97085e0..78ecb2038 100644 --- a/features/user.feature +++ b/features/user.feature @@ -1035,3 +1035,30 @@ Feature: Manage WordPress users """ true """ + + Scenario: List more users than are loaded from the database at a time + Given a WP install + And I run `wp user generate --count=600` + And I run `wp user meta update 1 batch_meta admin-value` + And I run `wp user list --orderby=ID --order=asc --format=ids` + And save STDOUT as {IDS} + + When I run `wp user list --orderby=ID --order=asc --field=ID | tr '\n' ' ' | sed 's/ $//'` + Then STDOUT should be: + """ + {IDS} + """ + + When I run `wp user list --fields=ID,user_login,roles --format=csv | wc -l` + Then STDOUT should contain: + """ + 602 + """ + + When I run `wp user list --orderby=ID --order=asc --number=2 --fields=ID,batch_meta,roles --format=csv` + Then STDOUT should be: + """ + ID,batch_meta,roles + 1,admin-value,administrator + 2,,subscriber + """ diff --git a/src/Comment_Command.php b/src/Comment_Command.php index 2bb32d2b8..c84ab327f 100644 --- a/src/Comment_Command.php +++ b/src/Comment_Command.php @@ -520,6 +520,15 @@ public function list_( $args, $assoc_args ) { $assoc_args['count'] = true; } + $need_url = 'url' === $formatter->field || in_array( 'url', (array) $formatter->fields, true ); + + // Threaded results are nested, so they can't be loaded in chunks. + if ( ! in_array( $formatter->format, [ 'count', 'ids' ], true ) && empty( $assoc_args['hierarchical'] ) ) { + $fields = $formatter->field ? [ $formatter->field ] : (array) $formatter->fields; + $formatter->display_items( $this->query_comments_in_chunks( $assoc_args, $fields ) ); + return; + } + $query = new WP_Comment_Query(); $comments = $query->query( $assoc_args ); @@ -541,7 +550,7 @@ public function list_( $args, $assoc_args ) { $items = wp_list_pluck( $comments, 'comment_ID' ); $comments = $items; - } elseif ( is_array( $comments ) ) { + } elseif ( is_array( $comments ) && $need_url ) { $comments = array_map( function ( $comment ) { /** @@ -558,6 +567,86 @@ function ( $comment ) { } } + /** + * Query comments in chunks, so that they don't all have to be held in memory at once. + * + * The IDs of all matching comments are queried first. The comments are then loaded + * a chunk at a time, and the object cache is cleared after each chunk. Formats that + * can be written item by item are then streamed by the formatter. + * + * Fields that WP_Comment reads from the comment's post are read from the object + * cache. If any of them are displayed, the cache is kept, because the formatter may + * only read them once all comments have been loaded. + * + * @param array $query_args WP_Comment_Query arguments. + * @param string[] $fields Fields that will be displayed. + * @return \Generator + */ + private function query_comments_in_chunks( $query_args, $fields ) { + $need_url = in_array( 'url', $fields, true ); + $clear_cache = true; + foreach ( $fields as $field ) { + if ( 'url' !== $field && ! property_exists( 'WP_Comment', $field ) && ! property_exists( 'WP_Comment', 'comment_' . $field ) ) { + $clear_cache = false; + } + } + + $result = ( new WP_Comment_Query() )->query( + array_merge( + $query_args, + [ + 'fields' => 'ids', + 'count' => false, + ] + ) + ); + + $ids = []; + foreach ( is_array( $result ) ? $result : [] as $id ) { + if ( is_numeric( $id ) ) { + $ids[] = (int) $id; + } + } + + $chunk_args = $query_args; + unset( $chunk_args['number'], $chunk_args['offset'], $chunk_args['paged'] ); + // The order is restored from $ids below. Ordering by `comment__in` in SQL would + // need a FIELD() call with one argument per comment, which SQLite does not allow. + $chunk_args['orderby'] = 'none'; + $chunk_args['fields'] = ''; + $chunk_args['count'] = false; + $chunk_args['no_found_rows'] = true; + + foreach ( array_chunk( $ids, self::LIST_CHUNK_SIZE ) as $chunk ) { + $chunk_args['comment__in'] = $chunk; + + $comments = []; + foreach ( (array) ( new WP_Comment_Query() )->query( $chunk_args ) as $comment ) { + if ( $comment instanceof \WP_Comment ) { + $comments[ (int) $comment->comment_ID ] = $comment; + } + } + + foreach ( $chunk as $id ) { + if ( ! isset( $comments[ (int) $id ] ) ) { + continue; + } + + $comment = $comments[ (int) $id ]; + if ( $need_url ) { + // @phpstan-ignore property.notFound + $comment->url = get_comment_link( $comment ); + } + yield $comment; + } + + unset( $comments ); + if ( $clear_cache ) { + self::clear_runtime_object_cache(); + } + } + } + /** * Deletes a comment. * diff --git a/src/Post_Command.php b/src/Post_Command.php index 22c569001..7e2aa78a1 100644 --- a/src/Post_Command.php +++ b/src/Post_Command.php @@ -27,11 +27,6 @@ */ class Post_Command extends CommandWithDBObject { - /** - * Number of posts `wp post list` loads from the database at a time. - */ - const LIST_CHUNK_SIZE = 500; - protected $obj_type = 'post'; protected $obj_fields = [ 'ID', @@ -1135,19 +1130,6 @@ private function query_posts_in_chunks( $query_args, $fields ) { } } - /** - * Free the memory held by the in-process object cache. - * - * Persistent object caches are left alone unless they can flush their in-process part only. - */ - private static function clear_runtime_object_cache() { - if ( function_exists( 'wp_cache_flush_runtime' ) && function_exists( 'wp_cache_supports' ) && wp_cache_supports( 'flush_runtime' ) ) { - wp_cache_flush_runtime(); - } elseif ( ! wp_using_ext_object_cache() ) { - wp_cache_flush(); - } - } - /** * Generates some posts. * diff --git a/src/Term_Command.php b/src/Term_Command.php index c9dfb769e..e3a235f19 100644 --- a/src/Term_Command.php +++ b/src/Term_Command.php @@ -161,13 +161,17 @@ public function list_( $args, $assoc_args ) { } } + $need_url = 'url' === $formatter->field || in_array( 'url', (array) $formatter->fields, true ); + $terms = array_map( - function ( $term ) { + function ( $term ) use ( $need_url ) { $term->count = (int) $term->count; $term->parent = (int) $term->parent; + if ( $need_url ) { // @phpstan-ignore property.notFound $term->url = get_term_link( $term ); + } return $term; }, $terms diff --git a/src/User_Command.php b/src/User_Command.php index b7849ea08..796526af8 100644 --- a/src/User_Command.php +++ b/src/User_Command.php @@ -172,33 +172,81 @@ public function list_( $args, $assoc_args ) { } } - $users = get_users( $assoc_args ); - if ( 'ids' === $formatter->format ) { - echo implode( ' ', $users ); + echo implode( ' ', get_users( $assoc_args ) ); } elseif ( 'count' === $formatter->format ) { - $formatter->display_items( $users ); + $formatter->display_items( get_users( $assoc_args ) ); } else { - $iterator = Utils\iterator_map( - $users, - function ( $user ) { - if ( ! is_object( $user ) ) { - return $user; - } + $fields = $formatter->field ? [ $formatter->field ] : (array) $formatter->fields; + $formatter->display_items( $this->query_users_in_chunks( $assoc_args, $fields ) ); + } + } + + /** + * Query users in chunks, so that they don't all have to be held in memory at once. + * + * The IDs of all matching users are queried first. The users and their meta are then + * loaded a chunk at a time, and the object cache is cleared after each chunk. Formats + * that can be written item by item are then streamed by the formatter. + * + * User meta is read from the object cache. If any meta is displayed, the cache is + * kept, because the formatter may only read it once all users have been loaded. + * + * @param array $query_args WP_User_Query arguments. + * @param string[] $fields Fields that will be displayed. + * @return \Generator + */ + private function query_users_in_chunks( $query_args, $fields ) { + $columns = [ 'ID', 'user_login', 'user_pass', 'user_nicename', 'user_email', 'user_url', 'user_registered', 'user_activation_key', 'user_status', 'display_name' ]; + $need_url = in_array( 'url', $fields, true ); + $clear_cache = true; + foreach ( $fields as $field ) { + if ( + ! in_array( $field, [ 'url', 'roles' ], true ) + && ! in_array( $field, $columns, true ) + && ! in_array( 'user_' . $field, $columns, true ) + && ! property_exists( 'WP_User', $field ) + ) { + $clear_cache = false; + } + } - /** - * @var \WP_User $user - */ + $ids = array_map( 'intval', (array) get_users( array_merge( $query_args, [ 'fields' => 'ids' ] ) ) ); - // @phpstan-ignore assign.propertyType - $user->roles = implode( ',', $user->roles ); + $chunk_args = $query_args; + unset( $chunk_args['number'], $chunk_args['offset'], $chunk_args['paged'] ); + // The order is restored from $ids below. + $chunk_args['fields'] = 'all_with_meta'; + + foreach ( array_chunk( $ids, self::LIST_CHUNK_SIZE ) as $chunk ) { + $chunk_args['include'] = $chunk; + + $users = []; + foreach ( get_users( $chunk_args ) as $user ) { + if ( $user instanceof \WP_User ) { + $users[ $user->ID ] = $user; + } + } + + foreach ( $chunk as $id ) { + if ( ! isset( $users[ $id ] ) ) { + continue; + } + + $user = $users[ $id ]; + // @phpstan-ignore assign.propertyType + $user->roles = implode( ',', $user->roles ); + if ( $need_url ) { // @phpstan-ignore property.notFound $user->url = get_author_posts_url( $user->ID, $user->user_nicename ); - return $user; } - ); + yield $user; + } - $formatter->display_items( $iterator ); + unset( $users ); + if ( $clear_cache ) { + self::clear_runtime_object_cache(); + } } } diff --git a/src/WP_CLI/CommandWithDBObject.php b/src/WP_CLI/CommandWithDBObject.php index 767a0bb27..617e185eb 100644 --- a/src/WP_CLI/CommandWithDBObject.php +++ b/src/WP_CLI/CommandWithDBObject.php @@ -14,6 +14,11 @@ */ abstract class CommandWithDBObject extends WP_CLI_Command { + /** + * Number of objects that list commands load from the database at a time. + */ + const LIST_CHUNK_SIZE = 500; + /** * @var string $object_type WordPress' expected name for the object. */ @@ -184,4 +189,18 @@ protected function get_formatter( &$assoc_args ) { } return new Formatter( $assoc_args, $fields, $this->obj_type ); } + + /** + * Free the memory held by the in-process object cache. + * + * List commands call this after each chunk of objects. Persistent object + * caches are left alone unless they can flush their in-process part only. + */ + protected static function clear_runtime_object_cache() { + if ( function_exists( 'wp_cache_flush_runtime' ) && function_exists( 'wp_cache_supports' ) && wp_cache_supports( 'flush_runtime' ) ) { + wp_cache_flush_runtime(); + } elseif ( ! wp_using_ext_object_cache() ) { + wp_cache_flush(); + } + } } From ae6bbcfda9b7fd39e5c1bd1095c3a40e1ce9e093 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 7 Oct 2026 14:36:37 +0000 Subject: [PATCH 2/2] Always clear the object cache after each chunk of comments and users wp-cli/wp-cli#6426 makes the formatter read the requested fields of each item while it is current, also when it can't stream them. User meta and the post fields of comments are therefore read before the cache of their chunk is cleared, so the cache no longer needs to be kept for them. The posts of each chunk of comments are now loaded in one query, so that fields like post_title don't load each post again in every chunk. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01D26yjkN2BiqCXT6p6o1WqS --- src/Comment_Command.php | 27 +++++++++------------------ src/User_Command.php | 33 +++++++++------------------------ 2 files changed, 18 insertions(+), 42 deletions(-) diff --git a/src/Comment_Command.php b/src/Comment_Command.php index c84ab327f..45b4244ee 100644 --- a/src/Comment_Command.php +++ b/src/Comment_Command.php @@ -524,8 +524,7 @@ public function list_( $args, $assoc_args ) { // Threaded results are nested, so they can't be loaded in chunks. if ( ! in_array( $formatter->format, [ 'count', 'ids' ], true ) && empty( $assoc_args['hierarchical'] ) ) { - $fields = $formatter->field ? [ $formatter->field ] : (array) $formatter->fields; - $formatter->display_items( $this->query_comments_in_chunks( $assoc_args, $fields ) ); + $formatter->display_items( $this->query_comments_in_chunks( $assoc_args, $need_url ) ); return; } @@ -574,23 +573,15 @@ function ( $comment ) { * a chunk at a time, and the object cache is cleared after each chunk. Formats that * can be written item by item are then streamed by the formatter. * - * Fields that WP_Comment reads from the comment's post are read from the object - * cache. If any of them are displayed, the cache is kept, because the formatter may - * only read them once all comments have been loaded. + * Fields that WP_Comment reads from the comment's post are read from the object cache. + * The formatter reads the requested fields of each comment while it is the current + * item, before the cache of its chunk is cleared, even when it can't stream them. * * @param array $query_args WP_Comment_Query arguments. - * @param string[] $fields Fields that will be displayed. + * @param bool $need_url Whether to add the `url` property to each comment. * @return \Generator */ - private function query_comments_in_chunks( $query_args, $fields ) { - $need_url = in_array( 'url', $fields, true ); - $clear_cache = true; - foreach ( $fields as $field ) { - if ( 'url' !== $field && ! property_exists( 'WP_Comment', $field ) && ! property_exists( 'WP_Comment', 'comment_' . $field ) ) { - $clear_cache = false; - } - } - + private function query_comments_in_chunks( $query_args, $need_url ) { $result = ( new WP_Comment_Query() )->query( array_merge( $query_args, @@ -616,6 +607,8 @@ private function query_comments_in_chunks( $query_args, $fields ) { $chunk_args['fields'] = ''; $chunk_args['count'] = false; $chunk_args['no_found_rows'] = true; + // Load the posts of each chunk in one query, as fields like `post_title` read them. + $chunk_args['update_comment_post_cache'] = true; foreach ( array_chunk( $ids, self::LIST_CHUNK_SIZE ) as $chunk ) { $chunk_args['comment__in'] = $chunk; @@ -641,9 +634,7 @@ private function query_comments_in_chunks( $query_args, $fields ) { } unset( $comments ); - if ( $clear_cache ) { - self::clear_runtime_object_cache(); - } + self::clear_runtime_object_cache(); } } diff --git a/src/User_Command.php b/src/User_Command.php index 796526af8..e8026fcbf 100644 --- a/src/User_Command.php +++ b/src/User_Command.php @@ -177,8 +177,8 @@ public function list_( $args, $assoc_args ) { } elseif ( 'count' === $formatter->format ) { $formatter->display_items( get_users( $assoc_args ) ); } else { - $fields = $formatter->field ? [ $formatter->field ] : (array) $formatter->fields; - $formatter->display_items( $this->query_users_in_chunks( $assoc_args, $fields ) ); + $need_url = 'url' === $formatter->field || in_array( 'url', (array) $formatter->fields, true ); + $formatter->display_items( $this->query_users_in_chunks( $assoc_args, $need_url ) ); } } @@ -189,28 +189,15 @@ public function list_( $args, $assoc_args ) { * loaded a chunk at a time, and the object cache is cleared after each chunk. Formats * that can be written item by item are then streamed by the formatter. * - * User meta is read from the object cache. If any meta is displayed, the cache is - * kept, because the formatter may only read it once all users have been loaded. + * User meta is read from the object cache. The formatter reads the requested fields of + * each user while it is the current item, before the cache of its chunk is cleared, + * even when it can't stream them. * - * @param array $query_args WP_User_Query arguments. - * @param string[] $fields Fields that will be displayed. + * @param array $query_args WP_User_Query arguments. + * @param bool $need_url Whether to add the `url` property to each user. * @return \Generator */ - private function query_users_in_chunks( $query_args, $fields ) { - $columns = [ 'ID', 'user_login', 'user_pass', 'user_nicename', 'user_email', 'user_url', 'user_registered', 'user_activation_key', 'user_status', 'display_name' ]; - $need_url = in_array( 'url', $fields, true ); - $clear_cache = true; - foreach ( $fields as $field ) { - if ( - ! in_array( $field, [ 'url', 'roles' ], true ) - && ! in_array( $field, $columns, true ) - && ! in_array( 'user_' . $field, $columns, true ) - && ! property_exists( 'WP_User', $field ) - ) { - $clear_cache = false; - } - } - + private function query_users_in_chunks( $query_args, $need_url ) { $ids = array_map( 'intval', (array) get_users( array_merge( $query_args, [ 'fields' => 'ids' ] ) ) ); $chunk_args = $query_args; @@ -244,9 +231,7 @@ private function query_users_in_chunks( $query_args, $fields ) { } unset( $users ); - if ( $clear_cache ) { - self::clear_runtime_object_cache(); - } + self::clear_runtime_object_cache(); } }