Skip to content

Commit a4b6975

Browse files
vdusekvdusek
authored
feat: add chunk_size to collection iterate() methods (#1146)
Adds `chunk_size` to the `iterate()` methods of all paginated collection clients, matching `iterate_items()`, `iterate_keys()` and `iterate_requests()` and the JS client's `chunkSize`. It caps the `limit` of each page request; a page never asks for more than what `limit` has left. Without it, the API picks the page size, as before. Closes: #1149 *✍️ Drafted by Claude Code*
1 parent 91df454 commit a4b6975

13 files changed

Lines changed: 133 additions & 23 deletions

‎docs/02_concepts/08_pagination.mdx‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,10 @@ to retrieve every item matching the filters. For dataset, key-value store and re
5353
matching helpers are `iterate_items`, `iterate_keys` and `iterate_requests`. They handle pagination
5454
automatically, so you don't need to manage offsets, limits or cursors yourself.
5555

56+
To control how many items each API call fetches, pass `chunk_size`. It caps the size of every page, and a page
57+
never asks for more items than `limit` has left. Without `chunk_size`, collection clients let the API pick the
58+
page size, while the storage helpers fetch 1,000 items per call.
59+
5660
The example below iterates over every Actor owned by the current user using a collection client's `iterate`
5761
method:
5862

‎src/apify_client/_resource_clients/actor_collection.py‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,7 @@ def iterate(
8181
offset: int | None = None,
8282
desc: bool | None = None,
8383
sort_by: Literal['createdAt', 'stats.lastRunStartedAt'] | None = 'createdAt',
84+
chunk_size: int | None = None,
8485
timeout: Timeout = 'medium',
8586
) -> Iterator[ActorListItem]:
8687
"""Iterate over the Actors the user has created or used.
@@ -96,6 +97,7 @@ def iterate(
9697
offset: What Actor to include as first when retrieving the list.
9798
desc: Whether to sort the Actors in descending order based on their creation date.
9899
sort_by: Field to sort the results by.
100+
chunk_size: Maximum number of Actors requested per API call when iterating across pages.
99101
timeout: Timeout for the API HTTP request.
100102
101103
Yields:
@@ -105,7 +107,7 @@ def iterate(
105107
def _callback(*, limit: int | None = None, offset: int | None = None) -> ListOfActors:
106108
return self.list(my=my, limit=limit, offset=offset, desc=desc, sort_by=sort_by, timeout=timeout)
107109

108-
return get_items_iterator(_callback, limit=limit, offset=offset)
110+
return get_items_iterator(_callback, limit=limit, offset=offset, chunk_size=chunk_size)
109111

110112
def create(
111113
self,
@@ -273,6 +275,7 @@ def iterate(
273275
offset: int | None = None,
274276
desc: bool | None = None,
275277
sort_by: Literal['createdAt', 'stats.lastRunStartedAt'] | None = 'createdAt',
278+
chunk_size: int | None = None,
276279
timeout: Timeout = 'medium',
277280
) -> AsyncIterator[ActorListItem]:
278281
"""Iterate over the Actors the user has created or used.
@@ -288,6 +291,7 @@ def iterate(
288291
offset: What Actor to include as first when retrieving the list.
289292
desc: Whether to sort the Actors in descending order based on their creation date.
290293
sort_by: Field to sort the results by.
294+
chunk_size: Maximum number of Actors requested per API call when iterating across pages.
291295
timeout: Timeout for the API HTTP request.
292296
293297
Yields:
@@ -297,7 +301,7 @@ def iterate(
297301
async def _callback(*, limit: int | None = None, offset: int | None = None) -> ListOfActors:
298302
return await self.list(my=my, limit=limit, offset=offset, desc=desc, sort_by=sort_by, timeout=timeout)
299303

300-
return get_items_iterator_async(_callback, limit=limit, offset=offset)
304+
return get_items_iterator_async(_callback, limit=limit, offset=offset, chunk_size=chunk_size)
301305

302306
async def create(
303307
self,

‎src/apify_client/_resource_clients/build_collection.py‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@ def iterate(
6767
limit: int | None = None,
6868
offset: int | None = None,
6969
desc: bool | None = None,
70+
chunk_size: int | None = None,
7071
timeout: Timeout = 'medium',
7172
) -> Iterator[BuildListItem]:
7273
"""Iterate over all Actor builds.
@@ -81,6 +82,7 @@ def iterate(
8182
limit: How many builds to retrieve.
8283
offset: What build to include as first when retrieving the list.
8384
desc: Whether to sort the builds in descending order based on their start date.
85+
chunk_size: Maximum number of builds requested per API call when iterating across pages.
8486
timeout: Timeout for the API HTTP request.
8587
8688
Yields:
@@ -90,7 +92,7 @@ def iterate(
9092
def _callback(*, limit: int | None = None, offset: int | None = None) -> ListOfBuilds:
9193
return self.list(limit=limit, offset=offset, desc=desc, timeout=timeout)
9294

93-
return get_items_iterator(_callback, limit=limit, offset=offset)
95+
return get_items_iterator(_callback, limit=limit, offset=offset, chunk_size=chunk_size)
9496

9597

9698
@docs_group('Resource clients')
@@ -146,6 +148,7 @@ def iterate(
146148
limit: int | None = None,
147149
offset: int | None = None,
148150
desc: bool | None = None,
151+
chunk_size: int | None = None,
149152
timeout: Timeout = 'medium',
150153
) -> AsyncIterator[BuildListItem]:
151154
"""Iterate over all Actor builds.
@@ -160,6 +163,7 @@ def iterate(
160163
limit: How many builds to retrieve.
161164
offset: What build to include as first when retrieving the list.
162165
desc: Whether to sort the builds in descending order based on their start date.
166+
chunk_size: Maximum number of builds requested per API call when iterating across pages.
163167
timeout: Timeout for the API HTTP request.
164168
165169
Yields:
@@ -169,4 +173,4 @@ def iterate(
169173
async def _callback(*, limit: int | None = None, offset: int | None = None) -> ListOfBuilds:
170174
return await self.list(limit=limit, offset=offset, desc=desc, timeout=timeout)
171175

172-
return get_items_iterator_async(_callback, limit=limit, offset=offset)
176+
return get_items_iterator_async(_callback, limit=limit, offset=offset, chunk_size=chunk_size)

‎src/apify_client/_resource_clients/dataset_collection.py‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,7 @@ def iterate(
8383
offset: int | None = None,
8484
desc: bool | None = None,
8585
ownership: StorageOwnership | None = None,
86+
chunk_size: int | None = None,
8687
timeout: Timeout = 'medium',
8788
) -> Iterator[DatasetListItem]:
8889
"""Iterate over the available datasets.
@@ -99,6 +100,7 @@ def iterate(
99100
desc: Whether to sort the datasets in descending order based on their modification date.
100101
ownership: Filter by ownership. 'ownedByMe' returns only user's own datasets,
101102
'sharedWithMe' returns only datasets shared with the user.
103+
chunk_size: Maximum number of datasets requested per API call when iterating across pages.
102104
timeout: Timeout for the API HTTP request.
103105
104106
Yields:
@@ -110,7 +112,7 @@ def _callback(*, limit: int | None = None, offset: int | None = None) -> ListOfD
110112
unnamed=unnamed, limit=limit, offset=offset, desc=desc, ownership=ownership, timeout=timeout
111113
)
112114

113-
return get_items_iterator(_callback, limit=limit, offset=offset)
115+
return get_items_iterator(_callback, limit=limit, offset=offset, chunk_size=chunk_size)
114116

115117
def get_or_create(
116118
self,
@@ -198,6 +200,7 @@ def iterate(
198200
offset: int | None = None,
199201
desc: bool | None = None,
200202
ownership: StorageOwnership | None = None,
203+
chunk_size: int | None = None,
201204
timeout: Timeout = 'medium',
202205
) -> AsyncIterator[DatasetListItem]:
203206
"""Iterate over the available datasets.
@@ -214,6 +217,7 @@ def iterate(
214217
desc: Whether to sort the datasets in descending order based on their modification date.
215218
ownership: Filter by ownership. 'ownedByMe' returns only user's own datasets,
216219
'sharedWithMe' returns only datasets shared with the user.
220+
chunk_size: Maximum number of datasets requested per API call when iterating across pages.
217221
timeout: Timeout for the API HTTP request.
218222
219223
Yields:
@@ -225,7 +229,7 @@ async def _callback(*, limit: int | None = None, offset: int | None = None) -> L
225229
unnamed=unnamed, limit=limit, offset=offset, desc=desc, ownership=ownership, timeout=timeout
226230
)
227231

228-
return get_items_iterator_async(_callback, limit=limit, offset=offset)
232+
return get_items_iterator_async(_callback, limit=limit, offset=offset, chunk_size=chunk_size)
229233

230234
async def get_or_create(
231235
self,

‎src/apify_client/_resource_clients/key_value_store_collection.py‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,7 @@ def iterate(
8282
offset: int | None = None,
8383
desc: bool | None = None,
8484
ownership: StorageOwnership | None = None,
85+
chunk_size: int | None = None,
8586
timeout: Timeout = 'medium',
8687
) -> Iterator[KeyValueStoreResource]:
8788
"""Iterate over the available key-value stores.
@@ -98,6 +99,7 @@ def iterate(
9899
desc: Whether to sort the key-value stores in descending order based on their modification date.
99100
ownership: Filter by ownership. 'ownedByMe' returns only user's own key-value stores,
100101
'sharedWithMe' returns only key-value stores shared with the user.
102+
chunk_size: Maximum number of key-value stores requested per API call when iterating across pages.
101103
timeout: Timeout for the API HTTP request.
102104
103105
Yields:
@@ -109,7 +111,7 @@ def _callback(*, limit: int | None = None, offset: int | None = None) -> ListOfK
109111
unnamed=unnamed, limit=limit, offset=offset, desc=desc, ownership=ownership, timeout=timeout
110112
)
111113

112-
return get_items_iterator(_callback, limit=limit, offset=offset)
114+
return get_items_iterator(_callback, limit=limit, offset=offset, chunk_size=chunk_size)
113115

114116
def get_or_create(
115117
self,
@@ -197,6 +199,7 @@ def iterate(
197199
offset: int | None = None,
198200
desc: bool | None = None,
199201
ownership: StorageOwnership | None = None,
202+
chunk_size: int | None = None,
200203
timeout: Timeout = 'medium',
201204
) -> AsyncIterator[KeyValueStoreResource]:
202205
"""Iterate over the available key-value stores.
@@ -213,6 +216,7 @@ def iterate(
213216
desc: Whether to sort the key-value stores in descending order based on their modification date.
214217
ownership: Filter by ownership. 'ownedByMe' returns only user's own key-value stores,
215218
'sharedWithMe' returns only key-value stores shared with the user.
219+
chunk_size: Maximum number of key-value stores requested per API call when iterating across pages.
216220
timeout: Timeout for the API HTTP request.
217221
218222
Yields:
@@ -224,7 +228,7 @@ async def _callback(*, limit: int | None = None, offset: int | None = None) -> L
224228
unnamed=unnamed, limit=limit, offset=offset, desc=desc, ownership=ownership, timeout=timeout
225229
)
226230

227-
return get_items_iterator_async(_callback, limit=limit, offset=offset)
231+
return get_items_iterator_async(_callback, limit=limit, offset=offset, chunk_size=chunk_size)
228232

229233
async def get_or_create(
230234
self,

‎src/apify_client/_resource_clients/request_queue_collection.py‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,7 @@ def iterate(
8383
offset: int | None = None,
8484
desc: bool | None = None,
8585
ownership: StorageOwnership | None = None,
86+
chunk_size: int | None = None,
8687
timeout: Timeout = 'medium',
8788
) -> Iterator[RequestQueueListItem]:
8889
"""Iterate over the available request queues.
@@ -99,6 +100,7 @@ def iterate(
99100
desc: Whether to sort the request queues in descending order based on their modification date.
100101
ownership: Filter by ownership. 'ownedByMe' returns only user's own request queues,
101102
'sharedWithMe' returns only request queues shared with the user.
103+
chunk_size: Maximum number of request queues requested per API call when iterating across pages.
102104
timeout: Timeout for the API HTTP request.
103105
104106
Yields:
@@ -110,7 +112,7 @@ def _callback(*, limit: int | None = None, offset: int | None = None) -> ListOfR
110112
unnamed=unnamed, limit=limit, offset=offset, desc=desc, ownership=ownership, timeout=timeout
111113
)
112114

113-
return get_items_iterator(_callback, limit=limit, offset=offset)
115+
return get_items_iterator(_callback, limit=limit, offset=offset, chunk_size=chunk_size)
114116

115117
def get_or_create(
116118
self,
@@ -196,6 +198,7 @@ def iterate(
196198
offset: int | None = None,
197199
desc: bool | None = None,
198200
ownership: StorageOwnership | None = None,
201+
chunk_size: int | None = None,
199202
timeout: Timeout = 'medium',
200203
) -> AsyncIterator[RequestQueueListItem]:
201204
"""Iterate over the available request queues.
@@ -212,6 +215,7 @@ def iterate(
212215
desc: Whether to sort the request queues in descending order based on their modification date.
213216
ownership: Filter by ownership. 'ownedByMe' returns only user's own request queues,
214217
'sharedWithMe' returns only request queues shared with the user.
218+
chunk_size: Maximum number of request queues requested per API call when iterating across pages.
215219
timeout: Timeout for the API HTTP request.
216220
217221
Yields:
@@ -223,7 +227,7 @@ async def _callback(*, limit: int | None = None, offset: int | None = None) -> L
223227
unnamed=unnamed, limit=limit, offset=offset, desc=desc, ownership=ownership, timeout=timeout
224228
)
225229

226-
return get_items_iterator_async(_callback, limit=limit, offset=offset)
230+
return get_items_iterator_async(_callback, limit=limit, offset=offset, chunk_size=chunk_size)
227231

228232
async def get_or_create(
229233
self,

‎src/apify_client/_resource_clients/run_collection.py‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,7 @@ def iterate(
8888
status: ActorJobStatus | list[ActorJobStatus] | None = None, # ty: ignore[invalid-type-form]
8989
started_before: str | datetime | None = None,
9090
started_after: str | datetime | None = None,
91+
chunk_size: int | None = None,
9192
timeout: Timeout = 'medium',
9293
) -> Iterator[RunListItem]:
9394
"""Iterate over all Actor runs.
@@ -105,6 +106,7 @@ def iterate(
105106
status: Retrieve only runs with the provided statuses.
106107
started_before: Only return runs started before this date (inclusive).
107108
started_after: Only return runs started after this date (inclusive).
109+
chunk_size: Maximum number of runs requested per API call when iterating across pages.
108110
timeout: Timeout for the API HTTP request.
109111
110112
Yields:
@@ -122,7 +124,7 @@ def _callback(*, limit: int | None = None, offset: int | None = None) -> ListOfR
122124
timeout=timeout,
123125
)
124126

125-
return get_items_iterator(_callback, limit=limit, offset=offset)
127+
return get_items_iterator(_callback, limit=limit, offset=offset, chunk_size=chunk_size)
126128

127129

128130
@docs_group('Resource clients')
@@ -197,6 +199,7 @@ def iterate(
197199
status: ActorJobStatus | list[ActorJobStatus] | None = None, # ty: ignore[invalid-type-form]
198200
started_before: str | datetime | None = None,
199201
started_after: str | datetime | None = None,
202+
chunk_size: int | None = None,
200203
timeout: Timeout = 'medium',
201204
) -> AsyncIterator[RunListItem]:
202205
"""Iterate over all Actor runs.
@@ -214,6 +217,7 @@ def iterate(
214217
status: Retrieve only runs with the provided statuses.
215218
started_before: Only return runs started before this date (inclusive).
216219
started_after: Only return runs started after this date (inclusive).
220+
chunk_size: Maximum number of runs requested per API call when iterating across pages.
217221
timeout: Timeout for the API HTTP request.
218222
219223
Yields:
@@ -231,4 +235,4 @@ async def _callback(*, limit: int | None = None, offset: int | None = None) -> L
231235
timeout=timeout,
232236
)
233237

234-
return get_items_iterator_async(_callback, limit=limit, offset=offset)
238+
return get_items_iterator_async(_callback, limit=limit, offset=offset, chunk_size=chunk_size)

‎src/apify_client/_resource_clients/schedule_collection.py‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@ def iterate(
6969
limit: int | None = None,
7070
offset: int | None = None,
7171
desc: bool | None = None,
72+
chunk_size: int | None = None,
7273
timeout: Timeout = 'medium',
7374
) -> Iterator[ScheduleListItem]:
7475
"""Iterate over the available schedules.
@@ -82,6 +83,7 @@ def iterate(
8283
limit: How many schedules to retrieve.
8384
offset: What schedules to include as first when retrieving the list.
8485
desc: Whether to sort the schedules in descending order based on their modification date.
86+
chunk_size: Maximum number of schedules requested per API call when iterating across pages.
8587
timeout: Timeout for the API HTTP request.
8688
8789
Yields:
@@ -91,7 +93,7 @@ def iterate(
9193
def _callback(*, limit: int | None = None, offset: int | None = None) -> ListOfSchedules:
9294
return self.list(limit=limit, offset=offset, desc=desc, timeout=timeout)
9395

94-
return get_items_iterator(_callback, limit=limit, offset=offset)
96+
return get_items_iterator(_callback, limit=limit, offset=offset, chunk_size=chunk_size)
9597

9698
def create(
9799
self,
@@ -192,6 +194,7 @@ def iterate(
192194
limit: int | None = None,
193195
offset: int | None = None,
194196
desc: bool | None = None,
197+
chunk_size: int | None = None,
195198
timeout: Timeout = 'medium',
196199
) -> AsyncIterator[ScheduleListItem]:
197200
"""Iterate over the available schedules.
@@ -205,6 +208,7 @@ def iterate(
205208
limit: How many schedules to retrieve.
206209
offset: What schedules to include as first when retrieving the list.
207210
desc: Whether to sort the schedules in descending order based on their modification date.
211+
chunk_size: Maximum number of schedules requested per API call when iterating across pages.
208212
timeout: Timeout for the API HTTP request.
209213
210214
Yields:
@@ -214,7 +218,7 @@ def iterate(
214218
async def _callback(*, limit: int | None = None, offset: int | None = None) -> ListOfSchedules:
215219
return await self.list(limit=limit, offset=offset, desc=desc, timeout=timeout)
216220

217-
return get_items_iterator_async(_callback, limit=limit, offset=offset)
221+
return get_items_iterator_async(_callback, limit=limit, offset=offset, chunk_size=chunk_size)
218222

219223
async def create(
220224
self,

0 commit comments

Comments
 (0)