Dev/repair develop

environment.yaml

@@ -47,3 +47,6 @@ dependencies:
     - git+https://github.com/kornia/kornia#egg=kornia
     - git+https://github.com/mmattamala/liegroups#egg=liegroups
     - --editable git+https://github.com/leggedrobotics/stego.git#egg=stego==0.0.1
+    - pytictac
+    - prettytable
+    - termcolor

scripts/dataset_generation/extract_binary_images_and_labels.py

@@ -66,9 +66,11 @@ def do(n, dry_run):
 
     valid_topics = ["/state_estimator/anymal_state", "/wide_angle_camera_front/img_out"]
 
-    rosbags = ["/home/rschmid/RosBags/6/images.bag",
-               "/home/rschmid/RosBags/6/2023-03-02-11-13-08_anymal-d020-lpc_mission_0.bag",
-               "/home/rschmid/RosBags/6/2023-03-02-11-13-08_anymal-d020-lpc_mission_1.bag"]
+    rosbags = [
+        "/home/rschmid/RosBags/6/images.bag",
+        "/home/rschmid/RosBags/6/2023-03-02-11-13-08_anymal-d020-lpc_mission_0.bag",
+        "/home/rschmid/RosBags/6/2023-03-02-11-13-08_anymal-d020-lpc_mission_1.bag",
+    ]
 
     output_bag_wvn = s + "_wvn.bag"
     output_bag_tf = s + "_tf.bag"
@@ -115,7 +117,20 @@ def do(n, dry_run):
     info_msg.distortion_model = "equidistant"
     info_msg.D = [0.4316922809468283, 0.09279900476637248, -0.4010909691803734, 0.4756163338479413]
     info_msg.K = [575.6050407221768, 0.0, 745.7312198525915, 0.0, 578.564849365178, 519.5207040671075, 0.0, 0.0, 1.0]
-    info_msg.P = [575.6050407221768, 0.0, 745.7312198525915, 0.0, 0.0, 578.564849365178, 519.5207040671075, 0.0, 0.0, 0.0, 1.0, 0.0]
+    info_msg.P = [
+        575.6050407221768,
+        0.0,
+        745.7312198525915,
+        0.0,
+        0.0,
+        578.564849365178,
+        519.5207040671075,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+    ]
     info_msg.R = [1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0]
 
     rosbag_info_dict = get_bag_info(output_bag_wvn)
@@ -150,7 +165,7 @@ def do(n, dry_run):
 
                     info_msg.header = msg.header
                     camera_options = {}
-                    camera_options['name'] = "wide_angle_camera_front"
+                    camera_options["name"] = "wide_angle_camera_front"
                     camera_options["use_for_training"] = True
 
                     info_msg.header = msg.header

wild_visual_navigation/cfg/experiment_params.py

@@ -43,11 +43,18 @@ class LossParams:
         w_reco: float = 0.5
         w_temp: float = 0.0  # 0.75
         method: str = "latest_measurment"
-        confidence_std_factor: float = 1.0
+        confidence_std_factor: float = 0.5
         trav_cross_entropy: bool = False
 
     loss: LossParams = LossParams()
 
+    @dataclass
+    class LossAnomalyParams:
+        method: str = "running_mean"    # "latest_measurment", "running_mean", "moving_average"
+        confidence_std_factor: float = 0.5
+
+    loss_anomaly: LossAnomalyParams = LossAnomalyParams()
+
     @dataclass
     class TrainerParams:
         precision: int = 32
@@ -84,12 +91,12 @@ class AblationDataModuleParams:
 
     @dataclass
     class ModelParams:
-        name: str = "SimpleMLP"
+        name: str = "LinearRnvp"  # LinearRnvp, SimpleMLP, SimpleGCN, DoubleMLP
         load_ckpt: Optional[str] = None
 
         @dataclass
         class SimpleMlpCfgParams:
-            input_size: int = 90
+            input_size: int = 384
             hidden_sizes: List[int] = field(default_factory=lambda: [256, 32, 1])
             reconstruction: bool = True
 
@@ -110,6 +117,17 @@ class SimpleGcnCfgParams:
 
         simple_gcn_cfg: SimpleGcnCfgParams = SimpleGcnCfgParams()
 
+        @dataclass
+        class LinearRnvpCfgParams:
+            input_dim: int = 384
+            coupling_topology: List[int] = field(default_factory=lambda: [200])
+            mask_type: str = "odds"
+            conditioning_size: int = 0
+            use_permutation: bool = True
+            single_function: bool = False
+
+        linear_rnvp_cfg: LinearRnvpCfgParams = LinearRnvpCfgParams()
+
     model: ModelParams = ModelParams()
 
     @dataclass

wild_visual_navigation/feature_extractor/dino_interface.py

@@ -6,6 +6,7 @@
 from omegaconf import DictConfig
 from torchvision import transforms as T
 from stego.src.train_segmentation import DinoFeaturizer
+from kornia.filters import filter2d
 
 
 class DinoInterface:
@@ -16,16 +17,19 @@ def __init__(
         input_interp: str = "bilinear",
         model_type: str = "vit_small",
         patch_size: int = 8,
+        dim: int = 384,
+        projection_type: str = None,  # nonlinear or None
+        dropout: bool = False,  # True or False
     ):
-        self.dim = 90
+        self.dim = dim
         self.cfg = DictConfig(
             {
                 "dino_patch_size": patch_size,
                 "dino_feat_type": "feat",
                 "model_type": model_type,
-                "projection_type": "nonlinear",
+                "projection_type": projection_type,
                 "pretrained_weights": None,
-                "dropout": True,
+                "dropout": dropout,
             }
         )
 
@@ -136,7 +140,6 @@ def inference(self, img: torch.tensor, interpolate: bool = False):
         pad = int((W - H) / 2)
         features = F.interpolate(features, new_size, mode="bilinear", align_corners=True)
         features = F.pad(features, pad=[pad, pad, 0, 0])
-
         return features
 
     @property

wild_visual_navigation/feature_extractor/feature_extractor.py

@@ -17,8 +17,7 @@
 
 class FeatureExtractor:
     def __init__(
-        self, device: str, segmentation_type: str = "slic", feature_type: str = "dino", input_size: int = 448, **kwargs
-    ):
+        self, device: str, segmentation_type: str = "slic", feature_type: str = "dino", input_size: int = 448, **kwargs):
         """Feature extraction from image
 
         Args:
@@ -40,7 +39,7 @@ def __init__(
         elif self._feature_type == "dino":
             self._feature_dim = 90
 
-            self.extractor = DinoInterface(device=device, input_size=input_size, patch_size=kwargs.get("patch_size", 8))
+            self.extractor = DinoInterface(device=device, input_size=input_size, patch_size=kwargs.get("patch_size", 8), dim=kwargs.get("dino_dim", 384))
         elif self._feature_type == "sift":
             self._feature_dim = 128
             self.extractor = DenseSIFTDescriptor().to(device)
@@ -66,7 +65,7 @@ def __init__(
             pass
 
     def extract(self, img, **kwargs):
-        if kwargs.get("fast_random", False):
+        if self._segmentation_type == "random":
             dense_feat = self.compute_features(img, None, None, **kwargs)
 
             H, W = img.shape[2:]
@@ -83,16 +82,16 @@ def extract(self, img, **kwargs):
             return None, feat, seg, None
 
         # Compute segments, their centers, and edges connecting them (graph structure)
-        with Timer("feature_extractor - compute_segments"):
-            edges, seg, center = self.compute_segments(img, **kwargs)
+        # with Timer("feature_extractor - compute_segments"):
+        edges, seg, center = self.compute_segments(img, **kwargs)
 
         # Compute features
-        with Timer("feature_extractor - compute_features"):
-            dense_feat = self.compute_features(img, seg, center, **kwargs)
+        # with Timer("feature_extractor - compute_features"):
+        dense_feat = self.compute_features(img, seg, center, **kwargs)
 
-        with Timer("feature_extractor - compute_features"):
-            # Sparsify features to match the centers if required
-            feat = self.sparsify_features(dense_feat, seg)
+        # with Timer("feature_extractor - compute_features"):
+        # Sparsify features to match the centers if required
+        feat = self.sparsify_features(dense_feat, seg)
 
         if kwargs.get("return_dense_features", False):
             return edges, feat, seg, center, dense_feat

wild_visual_navigation/learning/model/__init__.py

@@ -1,3 +1,4 @@
 from .simple_mlp import SimpleMLP, DoubleMLP
 from .simple_gcn import SimpleGCN
+from .linear_rnvp import LinearRnvp
 from .network_register import get_model