update

gitignore_global

@@ -23,4 +23,4 @@ __pycache__/
 .~*
 
 out/
-runs/
+mlruns

pylintrc

@@ -27,8 +27,8 @@ disable=unused-argument,
         abstract-class-instantiated,
         logging-fstring-interpolation,
         assignment-from-none,
-        redefined-builtin
-        # C0326, C0115, C0116 # for alignment
+        redefined-builtin,
+        too-many-return-statements
 
 # [VARIABLES]
 # allow-global-unused-variables=yes

site-packages/endaaman/trainer2.py

@@ -10,6 +10,9 @@
 from collections import OrderedDict
 from datetime import datetime
 
+from tqdm import tqdm
+import pydantic
+from pydantic import BaseModel
 import numpy as np
 import matplotlib
 from matplotlib import ticker, pyplot as plt
@@ -18,13 +21,10 @@
 from torch.optim.lr_scheduler import LambdaLR
 from torch.utils.data import DataLoader, Dataset
 # from torch.utils.tensorboard import SummaryWriter
-from tensorboardX import SummaryWriter
-import mlflow
 from torchvision import transforms
 from torchvision.utils import make_grid
-from tqdm import tqdm
-import pydantic
-from pydantic import BaseModel
+from tensorboardX import SummaryWriter
+import mlflow
 
 from .utils import wrap_iter, yes_no_input
 from .metrics import BaseMetrics
@@ -66,14 +66,17 @@ def __init__(self,
                  train_dataset:Dataset,
                  val_dataset:Dataset = None,
                  use_gpu = True,
+                 experiment_name = 'Default',
                  loss_fmt = '{:3f}',
                  main_metrics = 'loss',
                  ):
         self.config = config
         self.train_dataset = train_dataset
         self.val_dataset = val_dataset
+        self.experiment_name = experiment_name
         self.use_gpu = use_gpu
         self.loss_fmt = loss_fmt
+        self.main_metrics = main_metrics
 
         self.scheduler = None
         self.optimizer = None
@@ -229,6 +232,26 @@ def prepare_mlops(self):
             self.writer = SummaryWriter(log_dir=tb_log_dir)
             print(f'using log dir: tensorboard=`{tb_log_dir}`')
 
+        # mlflow ui --backend-store-uri ./runs/mlflow
+        if mlflow.active_run() is None:
+            # mlflow.set_tracking_uri('mlruns')
+            experiment = mlflow.get_experiment_by_name(name=self.experiment_name)
+            if experiment:
+                experiment_id = experiment.experiment_id
+            else:
+                experiment_id = mlflow.create_experiment(name=self.experiment_name)
+
+            s = mlflow.search_runs()
+            if len(s) > 0:
+                r = s[(s['experiment_id'] == experiment_id) & (s['tags.mlflow.runName'] == self.out_dir)]
+                if len(r) > 1:
+                    print('Multiple runs exist. Resuming last run.')
+                mlflow.start_run(experiment_id=experiment_id, run_id=r.iloc[0]['run_id'])
+            else:
+                mlflow.start_run(experiment_id=experiment_id, run_name=self.out_dir)
+            for k, v in self.config.dict().items():
+                mlflow.log_param(k, v)
+
     @abstractmethod
     def eval(self, inputs, gts):
         # DO:
@@ -241,6 +264,7 @@ def eval(self, inputs, gts):
 
     def record_value(self, k, value, step):
         self.writer.add_scalar(k, value, step)
+        mlflow.log_metric(k, value, step)
 
     def restore(self, checkpoint: Checkpoint):
         self.model.cpu()
@@ -381,6 +405,11 @@ def start(self, total_epoch):
             self.save_checkpoint(J(self.out_dir, LAST_CHECKPOINT))
             if self.is_achieved_best():
                 self.save_checkpoint(J(self.out_dir, BEST_CHECKPOINT))
+                for h, t in ((self.train_history, 'train'), (self.val_history, 'val')):
+                    if not h:
+                        continue
+                    for k, v in h.items():
+                        mlflow.log_metric(f'best/{k}', v[-1], len(v))
                 print('save best checkpoint')
 
             self.step(self.train_history['loss'][-1])

vim/plugin.d/telescope.lua

@@ -9,14 +9,15 @@ local pickers = require('telescope.pickers')
 local finders = require('telescope.finders')
 local conf = require('telescope.config').values
 
-vim.keymap.set('n', '<Space>b', builtin.buffers, {})
+vim.keymap.set('n', '<Space><Space>', builtin.buffers, {})
 vim.keymap.set('n', '<Space>o', builtin.oldfiles, {})
 vim.keymap.set('n', '<Space>h', builtin.help_tags, {})
 vim.keymap.set('n', '<Space>g', builtin.live_grep, {})
 vim.keymap.set('n', '<Space>m', builtin.marks, {})
 vim.keymap.set('n', '<Space>f', builtin.git_files, {})
 vim.keymap.set('n', '<Space>d', builtin.git_status, {})
 vim.keymap.set('n', '<Space>r', builtin.registers, {})
+vim.keymap.set('n', '<Space>p', builtin.resume, {})
 vim.keymap.set('n', '<Space>l', '<CMD>Telescope coc workspace_symbols<CR>', {})
 vim.keymap.set('n', '<Space>k', '<CMD>Telescope coc references_used<CR>', {})
 
@@ -69,7 +70,6 @@ function clipboard(opts)
 end
 
 vim.keymap.set('n', '<Space>c', clipboard, {})
-vim.keymap.set('n', '<Space><Space>', builtin.resume, {})
 
 
 local default_maps = {