From 041d4ff7c84cd563766dd9aab1aac33a976e4922 Mon Sep 17 00:00:00 2001 From: Petr Chmelar Date: Tue, 28 Jan 2025 13:23:45 +0000 Subject: [PATCH 1/2] import: fix broken progressbar when importing folder Add child progress bar to dvc fs to report directory import progress correctly. Fixes #10677 --- dvc/fs/dvc.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/dvc/fs/dvc.py b/dvc/fs/dvc.py index 6d79cb8f9f..86d075c5f2 100644 --- a/dvc/fs/dvc.py +++ b/dvc/fs/dvc.py @@ -500,7 +500,7 @@ def get( **kwargs, ) - def _get( # noqa: C901, PLR0912 + def _get( # noqa: C901, PLR0912, PLR0915 self, rpath, lpath, @@ -578,7 +578,9 @@ def get_file(arg: tuple[FileSystem, tuple[str, str, Optional[dict]]]): kw = kwargs if isinstance(fs, DataFileSystem): kw = kw | {"info": info} - return fs.get_file(src, dest, callback=callback, **kw) + with callback.branched(src, dest) as child: + fs.get_file(src, dest, callback=child, **kw) + callback.relative_update() if batch_size == 1: ctx: AbstractContextManager = nullcontext() From e8a680d2ad4d87f64d93619fb6f58ffd2e687671 Mon Sep 17 00:00:00 2001 From: skshetry <18718008+skshetry@users.noreply.github.com> Date: Fri, 31 Jan 2025 19:17:44 +0545 Subject: [PATCH 2/2] use callback.wrap to iterate on the main thread --- dvc/fs/dvc.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dvc/fs/dvc.py b/dvc/fs/dvc.py index 86d075c5f2..f1370de95f 100644 --- a/dvc/fs/dvc.py +++ b/dvc/fs/dvc.py @@ -580,7 +580,6 @@ def get_file(arg: tuple[FileSystem, tuple[str, str, Optional[dict]]]): kw = kw | {"info": info} with callback.branched(src, dest) as child: fs.get_file(src, dest, callback=child, **kw) - callback.relative_update() if batch_size == 1: ctx: AbstractContextManager = nullcontext() @@ -591,7 +590,7 @@ def get_file(arg: tuple[FileSystem, tuple[str, str, Optional[dict]]]): with ctx: it = ((fs, f) for fs, files in _files.items() for f in files) - deque(map_fn(get_file, it), maxlen=0) + deque(callback.wrap(map_fn(get_file, it)), maxlen=0) return result def get_file(self, rpath, lpath, **kwargs):