Skip to content

Commit

Permalink
executors: imap_unordered: use for loop when jobs == 1
Browse files Browse the repository at this point in the history
There is a visible overhead when dealing with lots of items.
  • Loading branch information
efiop committed Dec 2, 2023
1 parent 2db6438 commit 7e07012
Showing 1 changed file with 6 additions and 1 deletion.
7 changes: 6 additions & 1 deletion src/dvc_objects/executors.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,10 +45,15 @@ def imap_unordered(
It does not create all the futures at once to reduce memory usage.
"""

it = zip(*iterables)
if self.max_workers == 1:
for args in it:
yield fn(*args)
return

def create_taskset(n: int) -> Set[futures.Future]:
return {self.submit(fn, *args) for args in islice(it, n)}

it = zip(*iterables)
tasks = create_taskset(self.max_workers * 5)
while tasks:
done, tasks = futures.wait(tasks, return_when=futures.FIRST_COMPLETED)
Expand Down

0 comments on commit 7e07012

Please sign in to comment.