feat: skip dataset re-download

luxonis_ml/data/datasets/luxonis_dataset.py

@@ -418,15 +418,22 @@ def get_skeletons(
     def get_tasks(self) -> List[str]:
         return self.metadata.get("tasks", [])
 
-    def sync_from_cloud(self, force: bool = False) -> None:
+    def sync_from_cloud(
+        self, force: bool = False, skip_redownload_dataset: bool = False
+    ) -> None:
         """Downloads data from a remote cloud bucket."""
-
         if not self.is_remote:
             logger.warning("This is a local dataset! Cannot sync")
         else:
+            local_dir = self.base_path / "data" / self.team_id / "datasets"
+            if local_dir.exists() and skip_redownload_dataset and not force:
+                logger.info(
+                    "Local dataset directory already exists. Skipping download."
+                )
+                return
+
             if not self._is_synced or force:
                 logger.info("Syncing from cloud...")
-                local_dir = self.base_path / "data" / self.team_id / "datasets"
                 local_dir.mkdir(exist_ok=True, parents=True)
 
                 self.fs.get_dir(remote_paths="", local_dir=local_dir)

luxonis_ml/data/loaders/luxonis_loader.py

@@ -49,6 +49,7 @@ def __init__(
         out_image_format: Literal["RGB", "BGR"] = "RGB",
         *,
         force_resync: bool = False,
+        skip_redownload_dataset: bool = False,
     ) -> None:
         """A loader class used for loading data from L{LuxonisDataset}.
 
@@ -87,6 +88,8 @@ def __init__(
         @type force_resync: bool
         @param force_resync: Flag to force resync from cloud. Defaults
             to C{False}.
+        @param skip_redownload_dataset: If True, skip downloading when local dataset
+        already exists. If False, force redownload (unless force_resync is True).
         """
 
         self.logger = logging.getLogger(__name__)
@@ -96,7 +99,10 @@ def __init__(
         self.sync_mode = self.dataset.is_remote
 
         if self.sync_mode:
-            self.dataset.sync_from_cloud(force=force_resync)
+            self.dataset.sync_from_cloud(
+                force=force_resync,
+                skip_redownload_dataset=skip_redownload_dataset,
+            )
 
         if isinstance(view, str):
             view = [view]