diff --git a/features/comment.feature b/features/comment.feature index c51a5095a..ca7514d62 100644 --- a/features/comment.feature +++ b/features/comment.feature @@ -475,3 +475,24 @@ Feature: Manage WordPress comments And I run `wp comment unspam {COMMENT_ID} --url=www.example.com` And I run `wp comment trash {COMMENT_ID} --url=www.example.com` And I run `wp comment untrash {COMMENT_ID} --url=www.example.com` + + Scenario: List more comments than are loaded from the database at a time + Given a WP install + And I run `wp comment generate --count=1200 --post_id=1` + And I run `wp comment list --orderby=comment_ID --order=asc --format=ids` + And save STDOUT as {IDS} + + When I run `wp comment list --orderby=comment_ID --order=asc --field=comment_ID | tr '\n' ' ' | sed 's/ $//'` + Then STDOUT should be: + """ + {IDS} + """ + + When I run `wp comment list --fields=comment_ID,comment_post_ID --format=csv | wc -l` + Then STDOUT should contain: + """ + 1202 + """ + + When I run `wp comment list --orderby=comment_ID --order=asc --number=1 --fields=comment_ID,url --format=csv` + Then STDOUT should match /^comment_ID,url\n(\d+),https?:\/\/example\.com\/\?p=1#comment-\1$/ diff --git a/features/user.feature b/features/user.feature index ff97085e0..78ecb2038 100644 --- a/features/user.feature +++ b/features/user.feature @@ -1035,3 +1035,30 @@ Feature: Manage WordPress users """ true """ + + Scenario: List more users than are loaded from the database at a time + Given a WP install + And I run `wp user generate --count=600` + And I run `wp user meta update 1 batch_meta admin-value` + And I run `wp user list --orderby=ID --order=asc --format=ids` + And save STDOUT as {IDS} + + When I run `wp user list --orderby=ID --order=asc --field=ID | tr '\n' ' ' | sed 's/ $//'` + Then STDOUT should be: + """ + {IDS} + """ + + When I run `wp user list --fields=ID,user_login,roles --format=csv | wc -l` + Then STDOUT should contain: + """ + 602 + """ + + When I run `wp user list --orderby=ID --order=asc --number=2 --fields=ID,batch_meta,roles --format=csv` + Then STDOUT should be: + """ + ID,batch_meta,roles + 1,admin-value,administrator + 2,,subscriber + """ diff --git a/src/Comment_Command.php b/src/Comment_Command.php index 2bb32d2b8..45b4244ee 100644 --- a/src/Comment_Command.php +++ b/src/Comment_Command.php @@ -520,6 +520,14 @@ public function list_( $args, $assoc_args ) { $assoc_args['count'] = true; } + $need_url = 'url' === $formatter->field || in_array( 'url', (array) $formatter->fields, true ); + + // Threaded results are nested, so they can't be loaded in chunks. + if ( ! in_array( $formatter->format, [ 'count', 'ids' ], true ) && empty( $assoc_args['hierarchical'] ) ) { + $formatter->display_items( $this->query_comments_in_chunks( $assoc_args, $need_url ) ); + return; + } + $query = new WP_Comment_Query(); $comments = $query->query( $assoc_args ); @@ -541,7 +549,7 @@ public function list_( $args, $assoc_args ) { $items = wp_list_pluck( $comments, 'comment_ID' ); $comments = $items; - } elseif ( is_array( $comments ) ) { + } elseif ( is_array( $comments ) && $need_url ) { $comments = array_map( function ( $comment ) { /** @@ -558,6 +566,78 @@ function ( $comment ) { } } + /** + * Query comments in chunks, so that they don't all have to be held in memory at once. + * + * The IDs of all matching comments are queried first. The comments are then loaded + * a chunk at a time, and the object cache is cleared after each chunk. Formats that + * can be written item by item are then streamed by the formatter. + * + * Fields that WP_Comment reads from the comment's post are read from the object cache. + * The formatter reads the requested fields of each comment while it is the current + * item, before the cache of its chunk is cleared, even when it can't stream them. + * + * @param array $query_args WP_Comment_Query arguments. + * @param bool $need_url Whether to add the `url` property to each comment. + * @return \Generator + */ + private function query_comments_in_chunks( $query_args, $need_url ) { + $result = ( new WP_Comment_Query() )->query( + array_merge( + $query_args, + [ + 'fields' => 'ids', + 'count' => false, + ] + ) + ); + + $ids = []; + foreach ( is_array( $result ) ? $result : [] as $id ) { + if ( is_numeric( $id ) ) { + $ids[] = (int) $id; + } + } + + $chunk_args = $query_args; + unset( $chunk_args['number'], $chunk_args['offset'], $chunk_args['paged'] ); + // The order is restored from $ids below. Ordering by `comment__in` in SQL would + // need a FIELD() call with one argument per comment, which SQLite does not allow. + $chunk_args['orderby'] = 'none'; + $chunk_args['fields'] = ''; + $chunk_args['count'] = false; + $chunk_args['no_found_rows'] = true; + // Load the posts of each chunk in one query, as fields like `post_title` read them. + $chunk_args['update_comment_post_cache'] = true; + + foreach ( array_chunk( $ids, self::LIST_CHUNK_SIZE ) as $chunk ) { + $chunk_args['comment__in'] = $chunk; + + $comments = []; + foreach ( (array) ( new WP_Comment_Query() )->query( $chunk_args ) as $comment ) { + if ( $comment instanceof \WP_Comment ) { + $comments[ (int) $comment->comment_ID ] = $comment; + } + } + + foreach ( $chunk as $id ) { + if ( ! isset( $comments[ (int) $id ] ) ) { + continue; + } + + $comment = $comments[ (int) $id ]; + if ( $need_url ) { + // @phpstan-ignore property.notFound + $comment->url = get_comment_link( $comment ); + } + yield $comment; + } + + unset( $comments ); + self::clear_runtime_object_cache(); + } + } + /** * Deletes a comment. * diff --git a/src/Post_Command.php b/src/Post_Command.php index 4fa14359c..6c30182fe 100644 --- a/src/Post_Command.php +++ b/src/Post_Command.php @@ -27,11 +27,6 @@ */ class Post_Command extends CommandWithDBObject { - /** - * Number of posts `wp post list` loads from the database at a time. - */ - const LIST_CHUNK_SIZE = 500; - protected $obj_type = 'post'; protected $obj_fields = [ 'ID', @@ -1125,19 +1120,6 @@ private function query_posts_in_chunks( $query_args, $need_url ) { } } - /** - * Free the memory held by the in-process object cache. - * - * Persistent object caches are left alone unless they can flush their in-process part only. - */ - private static function clear_runtime_object_cache() { - if ( function_exists( 'wp_cache_flush_runtime' ) && function_exists( 'wp_cache_supports' ) && wp_cache_supports( 'flush_runtime' ) ) { - wp_cache_flush_runtime(); - } elseif ( ! wp_using_ext_object_cache() ) { - wp_cache_flush(); - } - } - /** * Generates some posts. * diff --git a/src/Term_Command.php b/src/Term_Command.php index c9dfb769e..e3a235f19 100644 --- a/src/Term_Command.php +++ b/src/Term_Command.php @@ -161,13 +161,17 @@ public function list_( $args, $assoc_args ) { } } + $need_url = 'url' === $formatter->field || in_array( 'url', (array) $formatter->fields, true ); + $terms = array_map( - function ( $term ) { + function ( $term ) use ( $need_url ) { $term->count = (int) $term->count; $term->parent = (int) $term->parent; + if ( $need_url ) { // @phpstan-ignore property.notFound $term->url = get_term_link( $term ); + } return $term; }, $terms diff --git a/src/User_Command.php b/src/User_Command.php index b7849ea08..e8026fcbf 100644 --- a/src/User_Command.php +++ b/src/User_Command.php @@ -172,33 +172,66 @@ public function list_( $args, $assoc_args ) { } } - $users = get_users( $assoc_args ); - if ( 'ids' === $formatter->format ) { - echo implode( ' ', $users ); + echo implode( ' ', get_users( $assoc_args ) ); } elseif ( 'count' === $formatter->format ) { - $formatter->display_items( $users ); + $formatter->display_items( get_users( $assoc_args ) ); } else { - $iterator = Utils\iterator_map( - $users, - function ( $user ) { - if ( ! is_object( $user ) ) { - return $user; - } + $need_url = 'url' === $formatter->field || in_array( 'url', (array) $formatter->fields, true ); + $formatter->display_items( $this->query_users_in_chunks( $assoc_args, $need_url ) ); + } + } + + /** + * Query users in chunks, so that they don't all have to be held in memory at once. + * + * The IDs of all matching users are queried first. The users and their meta are then + * loaded a chunk at a time, and the object cache is cleared after each chunk. Formats + * that can be written item by item are then streamed by the formatter. + * + * User meta is read from the object cache. The formatter reads the requested fields of + * each user while it is the current item, before the cache of its chunk is cleared, + * even when it can't stream them. + * + * @param array $query_args WP_User_Query arguments. + * @param bool $need_url Whether to add the `url` property to each user. + * @return \Generator + */ + private function query_users_in_chunks( $query_args, $need_url ) { + $ids = array_map( 'intval', (array) get_users( array_merge( $query_args, [ 'fields' => 'ids' ] ) ) ); - /** - * @var \WP_User $user - */ + $chunk_args = $query_args; + unset( $chunk_args['number'], $chunk_args['offset'], $chunk_args['paged'] ); + // The order is restored from $ids below. + $chunk_args['fields'] = 'all_with_meta'; - // @phpstan-ignore assign.propertyType - $user->roles = implode( ',', $user->roles ); + foreach ( array_chunk( $ids, self::LIST_CHUNK_SIZE ) as $chunk ) { + $chunk_args['include'] = $chunk; + + $users = []; + foreach ( get_users( $chunk_args ) as $user ) { + if ( $user instanceof \WP_User ) { + $users[ $user->ID ] = $user; + } + } + + foreach ( $chunk as $id ) { + if ( ! isset( $users[ $id ] ) ) { + continue; + } + + $user = $users[ $id ]; + // @phpstan-ignore assign.propertyType + $user->roles = implode( ',', $user->roles ); + if ( $need_url ) { // @phpstan-ignore property.notFound $user->url = get_author_posts_url( $user->ID, $user->user_nicename ); - return $user; } - ); + yield $user; + } - $formatter->display_items( $iterator ); + unset( $users ); + self::clear_runtime_object_cache(); } } diff --git a/src/WP_CLI/CommandWithDBObject.php b/src/WP_CLI/CommandWithDBObject.php index 767a0bb27..617e185eb 100644 --- a/src/WP_CLI/CommandWithDBObject.php +++ b/src/WP_CLI/CommandWithDBObject.php @@ -14,6 +14,11 @@ */ abstract class CommandWithDBObject extends WP_CLI_Command { + /** + * Number of objects that list commands load from the database at a time. + */ + const LIST_CHUNK_SIZE = 500; + /** * @var string $object_type WordPress' expected name for the object. */ @@ -184,4 +189,18 @@ protected function get_formatter( &$assoc_args ) { } return new Formatter( $assoc_args, $fields, $this->obj_type ); } + + /** + * Free the memory held by the in-process object cache. + * + * List commands call this after each chunk of objects. Persistent object + * caches are left alone unless they can flush their in-process part only. + */ + protected static function clear_runtime_object_cache() { + if ( function_exists( 'wp_cache_flush_runtime' ) && function_exists( 'wp_cache_supports' ) && wp_cache_supports( 'flush_runtime' ) ) { + wp_cache_flush_runtime(); + } elseif ( ! wp_using_ext_object_cache() ) { + wp_cache_flush(); + } + } }