From 4a68565d6e811ad34c807a79785dbe1127c07d48 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Thu, 30 Nov 2023 01:58:59 +0200 Subject: [PATCH] fs: copyfile: don't show progress for files < 1G There is no reason to create a progress bar overhead for anything that takes < 2sec to copy on a modern machine. Related https://github.com/iterative/dvc/issues/9914 --- src/dvc_objects/fs/utils.py | 36 ++++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/src/dvc_objects/fs/utils.py b/src/dvc_objects/fs/utils.py index 9165f1f..4c3155e 100644 --- a/src/dvc_objects/fs/utils.py +++ b/src/dvc_objects/fs/utils.py @@ -22,6 +22,7 @@ LOCAL_CHUNK_SIZE = 2**20 # 1 MB +COPY_PBAR_MIN_SIZE = 2**30 # 1 GB def is_exec(mode: int) -> bool: @@ -161,22 +162,25 @@ def copyfile( try: system.reflink(src, dest) except OSError: - from .callbacks import Callback - - with open(src, "rb") as fsrc, open(dest, "wb+") as fdest: - with Callback.as_tqdm_callback( - callback, - size=total, - bytes=True, - disable=no_progress_bar, - desc=name, - ) as cb: - wrapped = cb.wrap_attr(fdest, "write") - while True: - buf = fsrc.read(LOCAL_CHUNK_SIZE) - if not buf: - break - wrapped.write(buf) + if total < COPY_PBAR_MIN_SIZE: + shutil.copyfile(src, dest) + else: + from .callbacks import Callback + + with open(src, "rb") as fsrc, open(dest, "wb+") as fdest: + with Callback.as_tqdm_callback( + callback, + size=total, + bytes=True, + disable=no_progress_bar, + desc=name, + ) as cb: + wrapped = cb.wrap_attr(fdest, "write") + while True: + buf = fsrc.read(LOCAL_CHUNK_SIZE) + if not buf: + break + wrapped.write(buf) if callback: callback.absolute_update(total)