Skip to content

Commit

Permalink
Fulfill the proposed deletions
Browse files Browse the repository at this point in the history
  • Loading branch information
saskra committed Feb 21, 2025
1 parent 001eb6b commit b4fe279
Show file tree
Hide file tree
Showing 4 changed files with 20 additions and 7 deletions.
8 changes: 6 additions & 2 deletions src/cleanvision/dataset/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,13 @@ def build_dataset(
verbose: bool = True,
) -> Dataset:
if data_path:
return FSDataset(data_folder=data_path, storage_opts=storage_opts, verbose=verbose)
return FSDataset(
data_folder=data_path, storage_opts=storage_opts, verbose=verbose
)
elif filepaths:
return FSDataset(filepaths=filepaths, storage_opts=storage_opts, verbose=verbose)
return FSDataset(
filepaths=filepaths, storage_opts=storage_opts, verbose=verbose
)
elif hf_dataset and image_key:
return HFDataset(hf_dataset, image_key)
elif torchvision_dataset:
Expand Down
9 changes: 7 additions & 2 deletions src/cleanvision/issue_managers/duplicate_issue_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,9 @@ def find_issues(

results: List[Dict[str, Union[str, int]]] = []
if n_jobs == 1:
for idx in tqdm(dataset.index, leave=verbose, desc="Computing hashes", smoothing=0):
for idx in tqdm(
dataset.index, leave=verbose, desc="Computing hashes", smoothing=0
):
results.append(compute_hash(idx, dataset, to_compute, self.params))
else:
args = [
Expand All @@ -145,7 +147,10 @@ def find_issues(
p.imap_unordered(
compute_hash_wrapper, args, chunksize=chunksize
),
total=len(dataset), leave=verbose, desc="Computing hashes", smoothing=0,
total=len(dataset),
leave=verbose,
desc="Computing hashes",
smoothing=0,
)
)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,9 @@ def find_issues(
if to_be_computed:
results: List[Dict[str, Union[int, float, str]]] = []
if n_jobs == 1:
for idx in tqdm(dataset.index, leave=verbose, desc="Computing scores", smoothing=0):
for idx in tqdm(
dataset.index, leave=verbose, desc="Computing scores", smoothing=0
):
results.append(
compute_scores(
idx, dataset, to_be_computed, self.image_properties
Expand All @@ -162,7 +164,10 @@ def find_issues(
p.imap_unordered(
compute_scores_wrapper, args, chunksize=chunksize
),
total=len(dataset), leave=verbose, desc="Computing scores", smoothing=0,
total=len(dataset),
leave=verbose,
desc="Computing scores",
smoothing=0,
)
)

Expand Down
1 change: 0 additions & 1 deletion src/cleanvision/utils/base_issue_manager.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
from abc import ABC, abstractmethod
from tabnanny import verbose
from typing import Dict, Any

import pandas as pd
Expand Down

0 comments on commit b4fe279

Please sign in to comment.