Replies: 1 comment 2 replies
-
|
Hi @umgpy , Thanks for your interest here. Thanks. |
Beta Was this translation helpful? Give feedback.
-
|
Hi @Nic-Ma , Yes I have it in my code. Sorry for not positing the whole thing. I have added it for reference. Could it be that the loss and metric are intertwined as mention here #3694 , since using HD or Surface distance alone with Dice loss also results into loss becoming Nan. Just to clarify again, I want to train my model with Dice loss and evaluate it with dice metric but in addition would like the other metrics for reporting. I believe Thanks again for the help and all the great work. 3 class labels as a mutli- channel image with shape : torch.Size([1, 3, 256, 512, 512]) max_epochs = 300
val_interval = 1
VAL_AMP = True
# standard PyTorch program style: create SegResNet, DiceLoss and Adam optimizer
device = torch.device("cuda:0")
model = SegResNet(
blocks_down=[1, 2, 2, 4],
blocks_up=[1, 1, 1],
init_filters=16,
in_channels=1,
out_channels=3,
dropout_prob=0.2,
).to(device)
loss_function = DiceLoss(smooth_nr=0, smooth_dr=1e-5, squared_pred=True, to_onehot_y=False, sigmoid=True)
optimizer = torch.optim.Adam(model.parameters(), 1e-4, weight_decay=1e-5)
lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=max_epochs)
dice_metric = DiceMetric(include_background=True, reduction="mean")
dice_metric_batch = DiceMetric(include_background=True, reduction="mean_batch")
hd95_metric = HausdorffDistanceMetric(include_background=True, percentile=95, reduction="mean")
hd95_metric_batch = HausdorffDistanceMetric(include_background=True, percentile=95, reduction="mean_batch")
post_trans = Compose(
[EnsureType(), Activations(sigmoid=True), AsDiscrete(threshold_values=True)]
)
# define inference method
def inference(input):
def _compute(input):
return sliding_window_inference(
inputs=input,
roi_size=(240, 240, 160),
sw_batch_size=1,
predictor=model,
overlap=0.5,
)
if VAL_AMP:
with torch.cuda.amp.autocast():
return _compute(input)
else:
return _compute(input)
# use amp to accelerate training
scaler = torch.cuda.amp.GradScaler()
# enable cuDNN benchmark
torch.backends.cudnn.benchmark = True
Train and validation best_metric = -1
best_metric_epoch = -1
best_metrics_epochs_and_time = [[], [], []]
epoch_loss_values = []
metric_values = []
metric_values_tc = []
metric_values_wt = []
metric_values_et = []
hd_metric_values = []
hd_metric_values_tc = []
hd_metric_values_wt = []
hd_metric_values_et = []
assd_metric_values = []
assd_metric_values_tc = []
assd_metric_values_wt = []
assd_metric_values_et = []
total_start = time.time()
for epoch in range(max_epochs):
epoch_start = time.time()
print("-" * 10)
print(f"epoch {epoch + 1}/{max_epochs}")
model.train()
epoch_loss = 0
step = 0
for batch_data in train_loader:
step_start = time.time()
step += 1
inputs, labels = (
batch_data["Img"].to(device),
batch_data["Seg"].to(device),
)
optimizer.zero_grad()
with torch.cuda.amp.autocast():
outputs = model(inputs)
loss = loss_function(outputs, labels)
scaler.scale(loss).backward()
scaler.step(optimizer)
scaler.update()
epoch_loss += loss.item()
print(
f"{step}/{len(train_ds) // train_loader.batch_size}"
f", train_loss: {loss.item():.4f}"
f", step time: {(time.time() - step_start):.4f}"
)
lr_scheduler.step()
epoch_loss /= step
epoch_loss_values.append(epoch_loss)
print(f"epoch {epoch + 1} average loss: {epoch_loss:.4f}")
if (epoch + 1) % val_interval == 0:
model.eval()
with torch.no_grad():
for val_data in val_loader:
val_inputs, val_labels = (
val_data["Img"].to(device),
val_data["Seg"].to(device),
)
val_outputs = inference(val_inputs)
val_outputs = [post_trans(i) for i in decollate_batch(val_outputs)]
dice_metric(y_pred=val_outputs, y=val_labels)
dice_metric_batch(y_pred=val_outputs, y=val_labels)
hd95_metric(y_pred=val_outputs, y=val_labels)
hd95_metric_batch(y_pred=val_outputs, y=val_labels)
metric = dice_metric.aggregate().item()
metric_values.append(metric)
metric_batch = dice_metric_batch.aggregate()
metric_tc = metric_batch[0].item()
metric_values_tc.append(metric_tc)
metric_wt = metric_batch[1].item()
metric_values_wt.append(metric_wt)
metric_et = metric_batch[2].item()
metric_values_et.append(metric_et)
dice_metric.reset()
dice_metric_batch.reset()
hd_metric = hd95_metric.aggregate().item()
hd_metric_values.append(hd_metric)
hd_metric_batch = hd95_metric_batch.aggregate()
hd_metric_tc = hd_metric_batch[0].item()
hd_metric_values_tc.append(hd_metric_tc)
hd_metric_wt = hd_metric_batch[1].item()
hd_metric_values_wt.append(hd_metric_wt)
hd_metric_et = hd_metric_batch[2].item()
hd_metric_values_et.append(hd_metric_et)
hd95_metric.reset()
hd95_metric_batch.reset()
if metric < best_metric:
best_metric = metric
best_metric_epoch = epoch + 1
best_metrics_epochs_and_time[0].append(best_metric)
best_metrics_epochs_and_time[1].append(best_metric_epoch)
best_metrics_epochs_and_time[2].append(time.time() - total_start)
torch.save(
model.state_dict(),
os.path.join(output_path, "best_metric_model_multiClassTest_test_delete.pth"),
)
print("saved new best metric model")
print(
f"current epoch: {epoch + 1} current mean dice: {metric:.4f}"
f" BG: {metric_tc:.4f} fem: {metric_wt:.4f} pel: {metric_et:.4f}"
f"\nbest mean dice: {best_metric:.4f}"
f" at epoch: {best_metric_epoch}"
)
print(
f"current epoch: {epoch + 1} current mean hd95: {hd_metric:.4f}"
f" BG: {hd_metric_tc:.4f} fem: {hd_metric_wt:.4f} pel: {hd_metric_et:.4f}"
f"\nbest mean dice: {best_hd_metric:.4f}"
f" at epoch: {hd_metric_epoch}"
)
print(f"time consuming of epoch {epoch + 1} is: {(time.time() - epoch_start):.4f}")
total_time = time.time() - total_start |
Beta Was this translation helpful? Give feedback.
-
|
Did you guys see anything wrong in the test program? Thanks in advance. |
Beta Was this translation helpful? Give feedback.
-
Hi,
I am training a multi-class segmentation model using the brats tutorial as base. Here i am using Dice loss and Adam optimizer. While validating I would like to evaluate my model with multiple metrics namely Dice Metric, Hausdorff Distance, Average Surface distance, Confusion metric etc. The additional metrics are for reporting purpose only.
However, while training with just DSC metric the model trains properly but with addition of any of the other metrics the loss goes to inf.
Is it possible to just compute the metric without the training to be affected. I believe TorchMetrics package with its MetricCollection class allows for metric calculation for reporting only.
My code looks like this-
Training log -
Beta Was this translation helpful? Give feedback.
All reactions