From 060a98557e0bf6c9a7b5272f8df5b497bf8ce571 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Sat, 2 Dec 2023 01:25:32 +0200 Subject: [PATCH] fs: copyfile: don't touch callback at all for <1G files This again wastes a lot of time, as `set_size` might trigger a pbar refresh. Related https://github.com/iterative/dvc-objects/pull/229 Related https://github.com/iterative/dvc/issues/9914 --- src/dvc_objects/fs/utils.py | 66 ++++++++++++++++++++++--------------- 1 file changed, 40 insertions(+), 26 deletions(-) diff --git a/src/dvc_objects/fs/utils.py b/src/dvc_objects/fs/utils.py index 4c3155e..77f0391 100644 --- a/src/dvc_objects/fs/utils.py +++ b/src/dvc_objects/fs/utils.py @@ -142,48 +142,62 @@ def makedirs(path, exist_ok: bool = False, mode: Optional[int] = None) -> None: ) -def copyfile( +def _copyfile_with_pbar( src: "AnyFSPath", dest: "AnyFSPath", + total: int, callback: Optional["Callback"] = None, no_progress_bar: bool = False, name: Optional[str] = None, +): + from .callbacks import Callback + + name = name if name else os.path.basename(dest) + + if callback: + callback.set_size(total) + + with open(src, "rb") as fsrc, open(dest, "wb+") as fdest: + with Callback.as_tqdm_callback( + callback, + size=total, + bytes=True, + disable=no_progress_bar, + desc=name, + ) as cb: + wrapped = cb.wrap_attr(fdest, "write") + while True: + buf = fsrc.read(LOCAL_CHUNK_SIZE) + if not buf: + break + wrapped.write(buf) + + if callback: + callback.absolute_update(total) + + +def copyfile( + src: "AnyFSPath", + dest: "AnyFSPath", + **kwargs, ) -> None: """Copy file with progress bar""" - name = name if name else os.path.basename(dest) total = os.stat(src).st_size if os.path.isdir(dest): dest = os.path.join(dest, os.path.basename(src)) - if callback: - callback.set_size(total) - try: system.reflink(src, dest) + return except OSError: - if total < COPY_PBAR_MIN_SIZE: - shutil.copyfile(src, dest) - else: - from .callbacks import Callback - - with open(src, "rb") as fsrc, open(dest, "wb+") as fdest: - with Callback.as_tqdm_callback( - callback, - size=total, - bytes=True, - disable=no_progress_bar, - desc=name, - ) as cb: - wrapped = cb.wrap_attr(fdest, "write") - while True: - buf = fsrc.read(LOCAL_CHUNK_SIZE) - if not buf: - break - wrapped.write(buf) + pass - if callback: - callback.absolute_update(total) + if total < COPY_PBAR_MIN_SIZE: + shutil.copyfile(src, dest) + return + + _copyfile_with_pbar(src, dest, total, **kwargs) def tmp_fname(fname: "AnyFSPath" = "") -> "AnyFSPath":