Skip to content

Commit

Permalink
fs: copyfile: don't show progress for files < 1G
Browse files Browse the repository at this point in the history
There is no reason to create a progress bar overhead for anything that
takes < 2sec to copy on a modern machine.

Related iterative/dvc#9914
  • Loading branch information
efiop committed Nov 30, 2023
1 parent 231e527 commit 4a68565
Showing 1 changed file with 20 additions and 16 deletions.
36 changes: 20 additions & 16 deletions src/dvc_objects/fs/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@


LOCAL_CHUNK_SIZE = 2**20 # 1 MB
COPY_PBAR_MIN_SIZE = 2**30 # 1 GB


def is_exec(mode: int) -> bool:
Expand Down Expand Up @@ -161,22 +162,25 @@ def copyfile(
try:
system.reflink(src, dest)
except OSError:
from .callbacks import Callback

with open(src, "rb") as fsrc, open(dest, "wb+") as fdest:
with Callback.as_tqdm_callback(
callback,
size=total,
bytes=True,
disable=no_progress_bar,
desc=name,
) as cb:
wrapped = cb.wrap_attr(fdest, "write")
while True:
buf = fsrc.read(LOCAL_CHUNK_SIZE)
if not buf:
break
wrapped.write(buf)
if total < COPY_PBAR_MIN_SIZE:
shutil.copyfile(src, dest)
else:
from .callbacks import Callback

with open(src, "rb") as fsrc, open(dest, "wb+") as fdest:
with Callback.as_tqdm_callback(
callback,
size=total,
bytes=True,
disable=no_progress_bar,
desc=name,
) as cb:
wrapped = cb.wrap_attr(fdest, "write")
while True:
buf = fsrc.read(LOCAL_CHUNK_SIZE)
if not buf:
break
wrapped.write(buf)

if callback:
callback.absolute_update(total)
Expand Down

0 comments on commit 4a68565

Please sign in to comment.