Skip to content

Commit

Permalink
[FSTORE-1272] Statistics cleaner should not delete statistics during …
Browse files Browse the repository at this point in the history
…migration (#1724)
  • Loading branch information
SirOibaf committed Feb 26, 2024
1 parent de70cff commit d4043be
Showing 1 changed file with 6 additions and 4 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
import io.hops.hopsworks.common.hdfs.DistributedFsService;
import io.hops.hopsworks.common.util.PayaraClusterManager;
import io.hops.hopsworks.common.util.Settings;
import io.hops.hopsworks.exceptions.FeaturestoreException;
import io.hops.hopsworks.persistence.entity.featurestore.statistics.FeatureDescriptiveStatistics;
import org.javatuples.Pair;

Expand Down Expand Up @@ -115,7 +114,10 @@ public void deleteOrphanStatistics(Timer timer) {

LOG.log(Level.FINE, "DeleteOrphanStatistics start");
List<FeatureDescriptiveStatistics> fds =
featureDescriptiveStatisticsFacade.findOrphaned(new Pair<>(0, batchSize));
featureDescriptiveStatisticsFacade.findOrphaned(new Pair<>(0, batchSize))
.stream()
.filter(f -> !f.getFeatureName().equals("for-migration") && !f.getFeatureName().equals("to-be-deleted"))
.collect(Collectors.toList());

while (!fds.isEmpty()) {
String fdsIds =
Expand All @@ -137,10 +139,10 @@ public void deleteOrphanStatistics(Timer timer) {
}

private void deleteExtendedStatsFiles(List<FeatureDescriptiveStatistics> fds, DistributedFileSystemOps udfso)
throws FeaturestoreException, IOException {
throws IOException {
for (FeatureDescriptiveStatistics stats : fds) {
if (stats.getExtendedStatisticsPath() != null) {
udfso.rm(stats.getExtendedStatisticsPath(), false);
udfso.rm(stats.getExtendedStatisticsPath(), true);
}
}
}
Expand Down

0 comments on commit d4043be

Please sign in to comment.