4_evaluate.Rmd

---
title: 'Evaluate Models'
author: 'Author: Aaron S Coyner, PhD'
date: 'Last update: `r Sys.Date()`'
output:
    html_notebook:
        toc: yes
        toc_float: yes
        toc_depth: 3
---


# Setup
```{r}
library(tidyverse)
library(janitor)
library(tools)
library(caret)
library(ROCR)
library(caret)
```


```{r}
compute_auroc <- function(predictions, labels, title) {
    pred <- prediction(predictions, labels)
    auroc <- performance(pred, measure = 'auc')
    auroc <- auroc@y.values[[1]]
    perf <- performance(pred, 'tpr', 'fpr')
    plot(perf,
         main = title,
         xlim=c(0,1),
         ylim=c(0,1),
         colorize = FALSE)
    abline(a = 0,
           b = 1,
           col = 'red',
           lty = 2)
    text(0.8,
         0.25,
         paste('AUROC:', sprintf('%.3f', round(auroc, 3))))
}
```


```{r}
compute_aupr <- function(predictions, labels, title) {
    n_black <- length(labels[labels == 1])
    n_total <- length(labels)
    pred <- prediction(predictions, labels)
    aupr <- performance(pred, measure = 'aucpr')
    aupr <- aupr@y.values[[1]]
    perf <- performance(pred, 'prec', 'rec')
    plot(perf,
         main = title,
         xlim=c(0,1),
         ylim=c(0,1),
         colorize = FALSE)
    abline(a = n_black/n_total,
           b = 0,
           col = 'red',
           lty = 2)
    text(0.2,
         0.2,
         paste('Null AUPR:', sprintf('%.3f', round(n_black / n_total, 3))))
    text(0.8,
         0.2,
         paste('AUPR:', sprintf('%.3f', round(aupr, 3))))
}
```


```{r}
confusion_matrix <- function(df, column, threshold=0.5) {
    preds <- as.factor(if_else(df[column] >= threshold, 1, 0))
    confusionMatrix(preds, df$label, positive = '1')
}
```



# Load data
```{r}
irop_data <- read_csv('/Volumes/External/irop_data/irop_07092020.csv') %>%
    clean_names() %>%
    distinct(subject_id, .keep_all = TRUE) %>%
    select(subject_id, race)

test_data <- read_csv('./out/datasets/test_data.csv') %>%
    select(subject_id, image_id)


image_level <- read_csv('./out/probabilities/retcam_filtered_0.csv', col_types = cols()) %>%
    mutate(label = as.factor(if_else(str_detect(img_loc, 'black'), 1, 0)),
           image_id = as.numeric(file_path_sans_ext(basename(img_loc)))) %>%
    select(img_loc, image_id, label, retcam = probability) %>%
    bind_cols(select(read_csv('./out/probabilities/retcam_filtered_0_random.csv', col_types = cols()), retcam_random = probability)) %>%
    
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_0_random.csv', col_types = cols()), filter_0_random = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_0.csv', col_types = cols()), filter_0 = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_0_binarized.csv', col_types = cols()), filter_0_binarized = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_0_skeletonized.csv', col_types = cols()), filter_0_skeletonized = probability)) %>%
    
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_50.csv', col_types = cols()), filter_50 = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_50_binarized.csv', col_types = cols()), filter_50_binarized = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_50_skeletonized.csv', col_types = cols()), filter_50_skeletonized = probability)) %>%

    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_100.csv', col_types = cols()), filter_100 = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_100_binarized.csv', col_types = cols()), filter_100_binarized = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_100_skeletonized.csv', col_types = cols()), filter_100_skeletonized = probability)) %>%

    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_150.csv', col_types = cols()), filter_150 = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_150_binarized.csv', col_types = cols()), filter_150_binarized = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_150_skeletonized.csv', col_types = cols()), filter_150_skeletonized = probability)) %>%

    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_200.csv', col_types = cols()), filter_200 = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_200_binarized.csv', col_types = cols()), filter_200_binarized = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_200_skeletonized.csv', col_types = cols()), filter_200_skeletonized = probability)) %>%

    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_210.csv', col_types = cols()), filter_210 = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_210_binarized.csv', col_types = cols()), filter_210_binarized = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_210_skeletonized.csv', col_types = cols()), filter_210_skeletonized = probability)) %>%

    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_220.csv', col_types = cols()), filter_220 = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_220_binarized.csv', col_types = cols()), filter_220_binarized = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_220_skeletonized.csv', col_types = cols()), filter_220_skeletonized = probability)) %>%

    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_230.csv', col_types = cols()), filter_230 = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_230_binarized.csv', col_types = cols()), filter_230_binarized = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_230_skeletonized.csv', col_types = cols()), filter_230_skeletonized = probability)) %>%

    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_240.csv', col_types = cols()), filter_240 = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_240_binarized.csv', col_types = cols()), filter_240_binarized = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_240_skeletonized.csv', col_types = cols()), filter_240_skeletonized = probability)) %>%

    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_250.csv', col_types = cols()), filter_250 = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_250_binarized.csv', col_types = cols()), filter_250_binarized = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_250_skeletonized.csv', col_types = cols()), filter_250_skeletonized = probability)) %>%

    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_257.csv', col_types = cols()), filter_257 = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_257_binarized.csv', col_types = cols()), filter_257_binarized = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_257_skeletonized.csv', col_types = cols()), filter_257_skeletonized = probability)) %>%

    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_0_10.csv', col_types = cols()), filter_10 = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_0_10_binarized.csv', col_types = cols()), filter_10_binarized = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_0_10_skeletonized.csv', col_types = cols()), filter_10_skeletonized = probability)) %>%

    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_75_150.csv', col_types = cols()), filter_75 = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_75_150_binarized.csv', col_types = cols()), filter_75_binarized = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_75_150_skeletonized.csv', col_types = cols()), filter_75_skeletonized = probability)) %>%
    
    inner_join(test_data, by = 'image_id') %>%
    select(subject_id, everything(), -img_loc)



subject_level <- image_level %>%
    group_by(subject_id) %>%
    mutate(across(c(-image_id, -label), median)) %>%
    ungroup() %>%
    distinct(subject_id, .keep_all = TRUE)
```


# Image-level Analysis

### RetCam Images

```{r}
compute_aupr(image_level$retcam, image_level$label, 'PR: Raw RetCam Images')
compute_auroc(image_level$retcam, image_level$label, 'ROC: Raw RetCam Images')
confusion_matrix(image_level, 'retcam')
```

```{r}
compute_aupr(image_level$retcam_random, image_level$label, 'PR: Raw RetCam Images - Shuffled Labels')
compute_auroc(image_level$retcam_random, image_level$label, 'ROC: Raw RetCam Images - Shuffled Labels')
confusion_matrix(image_level, 'retcam_random')
```



### Segmentations: zero all pixels < 0

```{r}
compute_aupr(image_level$filter_0, image_level$label, 'PR: Segmentations: zero all pixels < 0')
compute_auroc(image_level$filter_0, image_level$label, 'ROC: Segmentations: zero all pixels < 0')
confusion_matrix(image_level, 'filter_0')
```

```{r}
compute_aupr(image_level$filter_0_binarized, image_level$label, 'PR: Segmentations: zero all pixels < 0 and binarize')
compute_auroc(image_level$filter_0_binarized, image_level$label, 'ROC: Segmentations: zero all pixels < 0 and binarize')
confusion_matrix(image_level, 'filter_0_binarized')
```

```{r}
compute_aupr(image_level$filter_0_skeletonized, image_level$label, 'PR: Segmentations: zero all pixels < 0 and skeletonize')
compute_auroc(image_level$filter_0_skeletonized, image_level$label, 'ROC: Segmentations: zero all pixels < 0 and skeletonize')
confusion_matrix(image_level, 'filter_0_skeletonized')
```

```{r}
compute_aupr(image_level$filter_0_random, image_level$label, 'PR: Segmentations: zero all pixels < 0 - Shuffled Labels')
compute_auroc(image_level$filter_0_random, image_level$label, 'ROC: Segmentations: zero all pixels < 0 - Shuffled Labels')
confusion_matrix(image_level, 'filter_0_random')
```


### Segmentations: zero all pixels < 50

```{r}
compute_aupr(image_level$filter_50, image_level$label, 'PR: Segmentations: zero all pixels < 50')
compute_auroc(image_level$filter_50, image_level$label, 'ROC: Segmentations: zero all pixels < 50')
confusion_matrix(image_level, 'filter_50')
```

```{r}
compute_aupr(image_level$filter_50_binarized, image_level$label, 'PR: Segmentations: zero all pixels < 50 and binarize')
compute_auroc(image_level$filter_50_binarized, image_level$label, 'ROC: Segmentations: zero all pixels < 50 and binarize')
confusion_matrix(image_level, 'filter_0_binarized')
```

```{r}
compute_aupr(image_level$filter_50_skeletonized, image_level$label, 'PR: Segmentations: zero all pixels < 50 and skeletonize')
compute_auroc(image_level$filter_50_skeletonized, image_level$label, 'ROC: Segmentations: zero all pixels < 50 and skeletonize')
confusion_matrix(image_level, 'filter_50_skeletonized')
```



### Segmentations: zero all pixels < 100

```{r}
compute_aupr(image_level$filter_100, image_level$label, 'PR: Segmentations: zero all pixels < 100')
compute_auroc(image_level$filter_100, image_level$label, 'ROC: Segmentations: zero all pixels < 100')
confusion_matrix(image_level, 'filter_100')
```

```{r}
compute_aupr(image_level$filter_100_binarized, image_level$label, 'PR: Segmentations: zero all pixels < 100 and binarize')
compute_auroc(image_level$filter_100_binarized, image_level$label, 'ROC: Segmentations: zero all pixels < 100 and binarize')
confusion_matrix(image_level, 'filter_100_binarized')
```

```{r}
compute_aupr(image_level$filter_100_skeletonized, image_level$label, 'PR: Segmentations: zero all pixels < 100 and skeletonize')
compute_auroc(image_level$filter_100_skeletonized, image_level$label, 'ROC: Segmentations: zero all pixels < 100 and skeletonize')
confusion_matrix(image_level, 'filter_100_skeletonized')
```



### Segmentations: zero all pixels < 150

```{r}
compute_aupr(image_level$filter_150, image_level$label, 'PR: Segmentations: zero all pixels < 150')
compute_auroc(image_level$filter_150, image_level$label, 'ROC: Segmentations: zero all pixels < 150')
confusion_matrix(image_level, 'filter_150')
```

```{r}
compute_aupr(image_level$filter_150_binarized, image_level$label, 'PR: Segmentations: zero all pixels < 150 and binarize')
compute_auroc(image_level$filter_150_binarized, image_level$label, 'ROC: Segmentations: zero all pixels < 150 and binarize')
confusion_matrix(image_level, 'filter_150_binarized')
```

```{r}
compute_aupr(image_level$filter_150_skeletonized, image_level$label, 'PR: Segmentations: zero all pixels < 150 and skeletonize')
compute_auroc(image_level$filter_150_skeletonized, image_level$label, 'ROC: Segmentations: zero all pixels < 150 and skeletonize')
confusion_matrix(image_level, 'filter_150_skeletonized')
```



### Segmentations: zero all pixels < 200

```{r}
compute_aupr(image_level$filter_200, image_level$label, 'PR: Segmentations: zero all pixels < 200')
compute_auroc(image_level$filter_200, image_level$label, 'ROC: Segmentations: zero all pixels < 200')
confusion_matrix(image_level, 'filter_200')
```

```{r}
compute_aupr(image_level$filter_200_binarized, image_level$label, 'PR: Segmentations: zero all pixels < 200 and binarize')
compute_auroc(image_level$filter_200_binarized, image_level$label, 'ROC: Segmentations: zero all pixels < 200 and binarize')
confusion_matrix(image_level, 'filter_200_binarized')
```

```{r}
compute_aupr(image_level$filter_200_skeletonized, image_level$label, 'PR: Segmentations: zero all pixels < 200 and skeletonize')
compute_auroc(image_level$filter_200_skeletonized, image_level$label, 'ROC: Segmentations: zero all pixels < 200 and skeletonize')
confusion_matrix(image_level, 'filter_200_skeletonized')
```



### Segmentations: zero all pixels < 210

```{r}
compute_aupr(image_level$filter_210, image_level$label, 'PR: Segmentations: zero all pixels < 210')
compute_auroc(image_level$filter_210, image_level$label, 'ROC: Segmentations: zero all pixels < 210')
confusion_matrix(image_level, 'filter_210')
```

```{r}
compute_aupr(image_level$filter_210_binarized, image_level$label, 'PR: Segmentations: zero all pixels < 210 and binarize')
compute_auroc(image_level$filter_210_binarized, image_level$label, 'ROC: Segmentations: zero all pixels < 210 and binarize')
confusion_matrix(image_level, 'filter_210_binarized')
```

```{r}
compute_aupr(image_level$filter_210_skeletonized, image_level$label, 'PR: Segmentations: zero all pixels < 210 and skeletonize')
compute_auroc(image_level$filter_210_skeletonized, image_level$label, 'ROC: Segmentations: zero all pixels < 210 and skeletonize')
confusion_matrix(image_level, 'filter_210_skeletonized')
```



### Segmentations: zero all pixels < 220

```{r}
compute_aupr(image_level$filter_220, image_level$label, 'PR: Segmentations: zero all pixels < 220')
compute_auroc(image_level$filter_220, image_level$label, 'ROC: Segmentations: zero all pixels < 220')
confusion_matrix(image_level, 'filter_220')
```

```{r}
compute_aupr(image_level$filter_220_binarized, image_level$label, 'PR: Segmentations: zero all pixels < 220 and binarize')
compute_auroc(image_level$filter_220_binarized, image_level$label, 'ROC: Segmentations: zero all pixels < 220 and binarize')
confusion_matrix(image_level, 'filter_220_binarized')
```

```{r}
compute_aupr(image_level$filter_220_skeletonized, image_level$label, 'PR: Segmentations: zero all pixels < 220 and skeletonize')
compute_auroc(image_level$filter_220_skeletonized, image_level$label, 'ROC: Segmentations: zero all pixels < 220 and skeletonize')
confusion_matrix(image_level, 'filter_220_skeletonized')
```



### Segmentations: zero all pixels < 230

```{r}
compute_aupr(image_level$filter_230, image_level$label, 'PR: Segmentations: zero all pixels < 230')
compute_auroc(image_level$filter_230, image_level$label, 'ROC: Segmentations: zero all pixels < 230')
confusion_matrix(image_level, 'filter_230')
```

```{r}
compute_aupr(image_level$filter_230_binarized, image_level$label, 'PR: Segmentations: zero all pixels < 230 and binarize')
compute_auroc(image_level$filter_230_binarized, image_level$label, 'ROC: Segmentations: zero all pixels < 230 and binarize')
confusion_matrix(image_level, 'filter_230_binarized')
```

```{r}
compute_aupr(image_level$filter_230_skeletonized, image_level$label, 'PR: Segmentations: zero all pixels < 230 and skeletonize')
compute_auroc(image_level$filter_230_skeletonized, image_level$label, 'ROC: Segmentations: zero all pixels < 230 and skeletonize')
confusion_matrix(image_level, 'filter_230_skeletonized')
```



### Segmentations: zero all pixels < 240

```{r}
compute_aupr(image_level$filter_240, image_level$label, 'PR: Segmentations: zero all pixels < 240')
compute_auroc(image_level$filter_240, image_level$label, 'ROC: Segmentations: zero all pixels < 240')
confusion_matrix(image_level, 'filter_240')
```

```{r}
compute_aupr(image_level$filter_240_binarized, image_level$label, 'PR: Segmentations: zero all pixels < 240 and binarize')
compute_auroc(image_level$filter_240_binarized, image_level$label, 'ROC: Segmentations: zero all pixels < 240 and binarize')
confusion_matrix(image_level, 'filter_240_binarized')
```

```{r}
compute_aupr(image_level$filter_240_skeletonized, image_level$label, 'PR: Segmentations: zero all pixels < 240 and skeletonize')
compute_auroc(image_level$filter_240_skeletonized, image_level$label, 'ROC: Segmentations: zero all pixels < 240 and skeletonize')
confusion_matrix(image_level, 'filter_240_skeletonized')
```



### Segmentations: zero all pixels < 250

```{r}
compute_aupr(image_level$filter_250, image_level$label, 'PR: Segmentations: zero all pixels < 250')
compute_auroc(image_level$filter_250, image_level$label, 'ROC: Segmentations: zero all pixels < 250')
confusion_matrix(image_level, 'filter_250')
```

```{r}
compute_aupr(image_level$filter_250_binarized, image_level$label, 'PR: Segmentations: zero all pixels < 250 and binarize')
compute_auroc(image_level$filter_250_binarized, image_level$label, 'ROC: Segmentations: zero all pixels < 250 and binarize')
confusion_matrix(image_level, 'filter_250_binarized')
```

```{r}
compute_aupr(image_level$filter_250_skeletonized, image_level$label, 'PR: Segmentations: zero all pixels < 250 and skeletonize')
compute_auroc(image_level$filter_250_skeletonized, image_level$label, 'ROC: Segmentations: zero all pixels < 250 and skeletonize')
confusion_matrix(image_level, 'filter_250_skeletonized')
```



### Segmentations: zero all pixels < 257

```{r}
compute_aupr(image_level$filter_257, image_level$label, 'PR: Segmentations: zero all pixels < 257')
compute_auroc(image_level$filter_257, image_level$label, 'ROC: Segmentations: zero all pixels < 257')
confusion_matrix(image_level, 'filter_257')
```

```{r}
compute_aupr(image_level$filter_257_binarized, image_level$label, 'PR: Segmentations: zero all pixels < 257 and binarize')
compute_auroc(image_level$filter_257_binarized, image_level$label, 'ROC: Segmentations: zero all pixels < 257 and binarize')
confusion_matrix(image_level, 'filter_257_binarized')
```

```{r}
compute_aupr(image_level$filter_257_skeletonized, image_level$label, 'PR: Segmentations: zero all pixels < 257 and skeletonize')
compute_auroc(image_level$filter_257_skeletonized, image_level$label, 'ROC: Segmentations: zero all pixels < 257 and skeletonize')
confusion_matrix(image_level, 'filter_257_skeletonized')
```



### Segmentations: zero all pixels > 10

```{r}
compute_aupr(image_level$filter_10, image_level$label, 'PR: Segmentations: zero all pixels < 10')
compute_auroc(image_level$filter_10, image_level$label, 'ROC: Segmentations: zero all pixels < 10')
confusion_matrix(image_level, 'filter_10')
```

```{r}
compute_aupr(image_level$filter_10_binarized, image_level$label, 'PR: Segmentations: zero all pixels < 10 and binarize')
compute_auroc(image_level$filter_10_binarized, image_level$label, 'ROC: Segmentations: zero all pixels < 10 and binarize')
confusion_matrix(image_level, 'filter_10_binarized')
```

```{r}
compute_aupr(image_level$filter_10_skeletonized, image_level$label, 'PR: Segmentations: zero all pixels < 10 and skeletonize')
compute_auroc(image_level$filter_10_skeletonized, image_level$label, 'ROC: Segmentations: zero all pixels < 10 and skeletonize')
confusion_matrix(image_level, 'filter_10_skeletonized')
```



### Segmentations: zero all pixels < 75 and > 150

```{r}
compute_aupr(image_level$filter_75, image_level$label, 'PR: Segmentations: zero all pixels < 75 and > 150')
compute_auroc(image_level$filter_75, image_level$label, 'ROC: Segmentations: zero all pixels < 75 and > 150')
confusion_matrix(image_level, 'filter_75')
```

```{r}
compute_aupr(image_level$filter_75_binarized, image_level$label, 'PR: Segmentations: zero all pixels < 75 and > 150 and binarize')
compute_auroc(image_level$filter_75_binarized, image_level$label, 'ROC: Segmentations: zero all pixels < 75 and > 150 and binarize')
confusion_matrix(image_level, 'filter_75_binarized')
```

```{r}
compute_aupr(image_level$filter_75_skeletonized, image_level$label, 'PR: Segmentations: zero all pixels < 75 and > 150 and skeletonize')
compute_auroc(image_level$filter_75_skeletonized, image_level$label, 'ROC: Segmentations: zero all pixels < 75 and > 150 and skeletonize')
confusion_matrix(image_level, 'filter_75_skeletonized')
```











# Subject-level Analysis

### RetCam Images

```{r}
compute_aupr(subject_level$retcam, subject_level$label, 'PR: Raw RetCam Images')
compute_auroc(subject_level$retcam, subject_level$label, 'ROC: Raw RetCam Images')
confusion_matrix(subject_level, 'retcam')
```

```{r}
compute_aupr(subject_level$retcam_random, subject_level$label, 'PR: Raw RetCam Images - Shuffled Labels')
compute_auroc(subject_level$retcam_random, subject_level$label, 'ROC: Raw RetCam Images - Shuffled Labels')
confusion_matrix(subject_level, 'retcam_random')
```



### Segmentations: zero all pixels < 0

```{r}
compute_aupr(subject_level$filter_0, subject_level$label, 'PR: Segmentations: zero all pixels < 0')
compute_auroc(subject_level$filter_0, subject_level$label, 'ROC: Segmentations: zero all pixels < 0')
confusion_matrix(subject_level, 'filter_0')
```

```{r}
compute_aupr(subject_level$filter_0_binarized, subject_level$label, 'PR: Segmentations: zero all pixels < 0 and binarize')
compute_auroc(subject_level$filter_0_binarized, subject_level$label, 'ROC: Segmentations: zero all pixels < 0 and binarize')
confusion_matrix(subject_level, 'filter_0_binarized')
```

```{r}
compute_aupr(subject_level$filter_0_skeletonized, subject_level$label, 'PR: Segmentations: zero all pixels < 0 and skeletonize')
compute_auroc(subject_level$filter_0_skeletonized, subject_level$label, 'ROC: Segmentations: zero all pixels < 0 and skeletonize')
confusion_matrix(subject_level, 'filter_0_skeletonized')
```

```{r}
compute_aupr(subject_level$filter_0_random, subject_level$label, 'PR: Segmentations: zero all pixels < 0 - Shuffled Labels')
compute_auroc(subject_level$filter_0_random, subject_level$label, 'ROC: Segmentations: zero all pixels < 0 - Shuffled Labels')
confusion_matrix(subject_level, 'filter_0_random')
```


### Segmentations: zero all pixels < 50

```{r}
compute_aupr(subject_level$filter_50, subject_level$label, 'PR: Segmentations: zero all pixels < 50')
compute_auroc(subject_level$filter_50, subject_level$label, 'ROC: Segmentations: zero all pixels < 50')
confusion_matrix(subject_level, 'filter_50')
```

```{r}
compute_aupr(subject_level$filter_50_binarized, subject_level$label, 'PR: Segmentations: zero all pixels < 50 and binarize')
compute_auroc(subject_level$filter_50_binarized, subject_level$label, 'ROC: Segmentations: zero all pixels < 50 and binarize')
confusion_matrix(subject_level, 'filter_0_binarized')
```

```{r}
compute_aupr(subject_level$filter_50_skeletonized, subject_level$label, 'PR: Segmentations: zero all pixels < 50 and skeletonize')
compute_auroc(subject_level$filter_50_skeletonized, subject_level$label, 'ROC: Segmentations: zero all pixels < 50 and skeletonize')
confusion_matrix(subject_level, 'filter_50_skeletonized')
```



### Segmentations: zero all pixels < 100

```{r}
compute_aupr(subject_level$filter_100, subject_level$label, 'PR: Segmentations: zero all pixels < 100')
compute_auroc(subject_level$filter_100, subject_level$label, 'ROC: Segmentations: zero all pixels < 100')
confusion_matrix(subject_level, 'filter_100')
```

```{r}
compute_aupr(subject_level$filter_100_binarized, subject_level$label, 'PR: Segmentations: zero all pixels < 100 and binarize')
compute_auroc(subject_level$filter_100_binarized, subject_level$label, 'ROC: Segmentations: zero all pixels < 100 and binarize')
confusion_matrix(subject_level, 'filter_100_binarized')
```

```{r}
compute_aupr(subject_level$filter_100_skeletonized, subject_level$label, 'PR: Segmentations: zero all pixels < 100 and skeletonize')
compute_auroc(subject_level$filter_100_skeletonized, subject_level$label, 'ROC: Segmentations: zero all pixels < 100 and skeletonize')
confusion_matrix(subject_level, 'filter_100_skeletonized')
```



### Segmentations: zero all pixels < 150

```{r}
compute_aupr(subject_level$filter_150, subject_level$label, 'PR: Segmentations: zero all pixels < 150')
compute_auroc(subject_level$filter_150, subject_level$label, 'ROC: Segmentations: zero all pixels < 150')
confusion_matrix(subject_level, 'filter_150')
```

```{r}
compute_aupr(subject_level$filter_150_binarized, subject_level$label, 'PR: Segmentations: zero all pixels < 150 and binarize')
compute_auroc(subject_level$filter_150_binarized, subject_level$label, 'ROC: Segmentations: zero all pixels < 150 and binarize')
confusion_matrix(subject_level, 'filter_150_binarized')
```

```{r}
compute_aupr(subject_level$filter_150_skeletonized, subject_level$label, 'PR: Segmentations: zero all pixels < 150 and skeletonize')
compute_auroc(subject_level$filter_150_skeletonized, subject_level$label, 'ROC: Segmentations: zero all pixels < 150 and skeletonize')
confusion_matrix(subject_level, 'filter_150_skeletonized')
```



### Segmentations: zero all pixels < 200

```{r}
compute_aupr(subject_level$filter_200, subject_level$label, 'PR: Segmentations: zero all pixels < 200')
compute_auroc(subject_level$filter_200, subject_level$label, 'ROC: Segmentations: zero all pixels < 200')
confusion_matrix(subject_level, 'filter_200')
```

```{r}
compute_aupr(subject_level$filter_200_binarized, subject_level$label, 'PR: Segmentations: zero all pixels < 200 and binarize')
compute_auroc(subject_level$filter_200_binarized, subject_level$label, 'ROC: Segmentations: zero all pixels < 200 and binarize')
confusion_matrix(subject_level, 'filter_200_binarized')
```

```{r}
compute_aupr(subject_level$filter_200_skeletonized, subject_level$label, 'PR: Segmentations: zero all pixels < 200 and skeletonize')
compute_auroc(subject_level$filter_200_skeletonized, subject_level$label, 'ROC: Segmentations: zero all pixels < 200 and skeletonize')
confusion_matrix(subject_level, 'filter_200_skeletonized')
```



### Segmentations: zero all pixels < 210

```{r}
compute_aupr(subject_level$filter_210, subject_level$label, 'PR: Segmentations: zero all pixels < 210')
compute_auroc(subject_level$filter_210, subject_level$label, 'ROC: Segmentations: zero all pixels < 210')
confusion_matrix(subject_level, 'filter_210')
```

```{r}
compute_aupr(subject_level$filter_210_binarized, subject_level$label, 'PR: Segmentations: zero all pixels < 210 and binarize')
compute_auroc(subject_level$filter_210_binarized, subject_level$label, 'ROC: Segmentations: zero all pixels < 210 and binarize')
confusion_matrix(subject_level, 'filter_210_binarized')
```

```{r}
compute_aupr(subject_level$filter_210_skeletonized, subject_level$label, 'PR: Segmentations: zero all pixels < 210 and skeletonize')
compute_auroc(subject_level$filter_210_skeletonized, subject_level$label, 'ROC: Segmentations: zero all pixels < 210 and skeletonize')
confusion_matrix(subject_level, 'filter_210_skeletonized')
```



### Segmentations: zero all pixels < 220

```{r}
compute_aupr(subject_level$filter_220, subject_level$label, 'PR: Segmentations: zero all pixels < 220')
compute_auroc(subject_level$filter_220, subject_level$label, 'ROC: Segmentations: zero all pixels < 220')
confusion_matrix(subject_level, 'filter_220')
```

```{r}
compute_aupr(subject_level$filter_220_binarized, subject_level$label, 'PR: Segmentations: zero all pixels < 220 and binarize')
compute_auroc(subject_level$filter_220_binarized, subject_level$label, 'ROC: Segmentations: zero all pixels < 220 and binarize')
confusion_matrix(subject_level, 'filter_220_binarized')
```

```{r}
compute_aupr(subject_level$filter_220_skeletonized, subject_level$label, 'PR: Segmentations: zero all pixels < 220 and skeletonize')
compute_auroc(subject_level$filter_220_skeletonized, subject_level$label, 'ROC: Segmentations: zero all pixels < 220 and skeletonize')
confusion_matrix(subject_level, 'filter_220_skeletonized')
```



### Segmentations: zero all pixels < 230

```{r}
compute_aupr(subject_level$filter_230, subject_level$label, 'PR: Segmentations: zero all pixels < 230')
compute_auroc(subject_level$filter_230, subject_level$label, 'ROC: Segmentations: zero all pixels < 230')
confusion_matrix(subject_level, 'filter_230')
```

```{r}
compute_aupr(subject_level$filter_230_binarized, subject_level$label, 'PR: Segmentations: zero all pixels < 230 and binarize')
compute_auroc(subject_level$filter_230_binarized, subject_level$label, 'ROC: Segmentations: zero all pixels < 230 and binarize')
confusion_matrix(subject_level, 'filter_230_binarized')
```

```{r}
compute_aupr(subject_level$filter_230_skeletonized, subject_level$label, 'PR: Segmentations: zero all pixels < 230 and skeletonize')
compute_auroc(subject_level$filter_230_skeletonized, subject_level$label, 'ROC: Segmentations: zero all pixels < 230 and skeletonize')
confusion_matrix(subject_level, 'filter_230_skeletonized')
```



### Segmentations: zero all pixels < 240

```{r}
compute_aupr(subject_level$filter_240, subject_level$label, 'PR: Segmentations: zero all pixels < 240')
compute_auroc(subject_level$filter_240, subject_level$label, 'ROC: Segmentations: zero all pixels < 240')
confusion_matrix(subject_level, 'filter_240')
```

```{r}
compute_aupr(subject_level$filter_240_binarized, subject_level$label, 'PR: Segmentations: zero all pixels < 240 and binarize')
compute_auroc(subject_level$filter_240_binarized, subject_level$label, 'ROC: Segmentations: zero all pixels < 240 and binarize')
confusion_matrix(subject_level, 'filter_240_binarized')
```

```{r}
compute_aupr(subject_level$filter_240_skeletonized, subject_level$label, 'PR: Segmentations: zero all pixels < 240 and skeletonize')
compute_auroc(subject_level$filter_240_skeletonized, subject_level$label, 'ROC: Segmentations: zero all pixels < 240 and skeletonize')
confusion_matrix(subject_level, 'filter_240_skeletonized')
```



### Segmentations: zero all pixels < 250

```{r}
compute_aupr(subject_level$filter_250, subject_level$label, 'PR: Segmentations: zero all pixels < 250')
compute_auroc(subject_level$filter_250, subject_level$label, 'ROC: Segmentations: zero all pixels < 250')
confusion_matrix(subject_level, 'filter_250')
```

```{r}
compute_aupr(subject_level$filter_250_binarized, subject_level$label, 'PR: Segmentations: zero all pixels < 250 and binarize')
compute_auroc(subject_level$filter_250_binarized, subject_level$label, 'ROC: Segmentations: zero all pixels < 250 and binarize')
confusion_matrix(subject_level, 'filter_250_binarized')
```

```{r}
compute_aupr(subject_level$filter_250_skeletonized, subject_level$label, 'PR: Segmentations: zero all pixels < 250 and skeletonize')
compute_auroc(subject_level$filter_250_skeletonized, subject_level$label, 'ROC: Segmentations: zero all pixels < 250 and skeletonize')
confusion_matrix(subject_level, 'filter_250_skeletonized')
```



### Segmentations: zero all pixels < 257

```{r}
compute_aupr(subject_level$filter_257, subject_level$label, 'PR: Segmentations: zero all pixels < 257')
compute_auroc(subject_level$filter_257, subject_level$label, 'ROC: Segmentations: zero all pixels < 257')
confusion_matrix(subject_level, 'filter_257')
```

```{r}
compute_aupr(subject_level$filter_257_binarized, subject_level$label, 'PR: Segmentations: zero all pixels < 257 and binarize')
compute_auroc(subject_level$filter_257_binarized, subject_level$label, 'ROC: Segmentations: zero all pixels < 257 and binarize')
confusion_matrix(subject_level, 'filter_257_binarized')
```

```{r}
compute_aupr(subject_level$filter_257_skeletonized, subject_level$label, 'PR: Segmentations: zero all pixels < 257 and skeletonize')
compute_auroc(subject_level$filter_257_skeletonized, subject_level$label, 'ROC: Segmentations: zero all pixels < 257 and skeletonize')
confusion_matrix(subject_level, 'filter_257_skeletonized')
```



### Segmentations: zero all pixels > 10

```{r}
compute_aupr(subject_level$filter_10, subject_level$label, 'PR: Segmentations: zero all pixels < 10')
compute_auroc(subject_level$filter_10, subject_level$label, 'ROC: Segmentations: zero all pixels < 10')
confusion_matrix(subject_level, 'filter_10')
```

```{r}
compute_aupr(subject_level$filter_10_binarized, subject_level$label, 'PR: Segmentations: zero all pixels < 10 and binarize')
compute_auroc(subject_level$filter_10_binarized, subject_level$label, 'ROC: Segmentations: zero all pixels < 10 and binarize')
confusion_matrix(subject_level, 'filter_10_binarized')
```

```{r}
compute_aupr(subject_level$filter_10_skeletonized, subject_level$label, 'PR: Segmentations: zero all pixels < 10 and skeletonize')
compute_auroc(subject_level$filter_10_skeletonized, subject_level$label, 'ROC: Segmentations: zero all pixels < 10 and skeletonize')
confusion_matrix(subject_level, 'filter_10_skeletonized')
```



### Segmentations: zero all pixels < 75 and > 150

```{r}
compute_aupr(subject_level$filter_75, subject_level$label, 'PR: Segmentations: zero all pixels < 75 and > 150')
compute_auroc(subject_level$filter_75, subject_level$label, 'ROC: Segmentations: zero all pixels < 75 and > 150')
confusion_matrix(subject_level, 'filter_75')
```

```{r}
compute_aupr(subject_level$filter_75_binarized, subject_level$label, 'PR: Segmentations: zero all pixels < 75 and > 150 and binarize')
compute_auroc(subject_level$filter_75_binarized, subject_level$label, 'ROC: Segmentations: zero all pixels < 75 and > 150 and binarize')
confusion_matrix(subject_level, 'filter_75_binarized')
```

```{r}
compute_aupr(subject_level$filter_75_skeletonized, subject_level$label, 'PR: Segmentations: zero all pixels < 75 and > 150 and skeletonize')
compute_auroc(subject_level$filter_75_skeletonized, subject_level$label, 'ROC: Segmentations: zero all pixels < 75 and > 150 and skeletonize')
confusion_matrix(subject_level, 'filter_75_skeletonized')
```



```{r}
aupr_figures <- function(predictions, labels, title, names, save_name) {
    n_black <- length(labels[labels == 1])
    n_total <- length(labels)
    for (i in seq_along(predictions)) {
        preds <- prediction(predictions[i], labels)
        aupr <- performance(preds, measure = 'aucpr')
        aupr <- aupr@y.values[[1]]
        perf <- performance(preds, 'prec', 'rec')
        if (is.null(dev.list())) {
            png(save_name,
                width=7,
                height=5,
                units='in',
                res=300)
            plot(perf,
                 main = title,
                 xlim=c(0,1),
                 ylim=c(0,1),
                 col = i,
                 lwd = 2)
        } else {
            plot(perf,
                 main = title,
                 xlim=c(0,1),
                 ylim=c(0,1),
                 col = i,
                 lwd = 2,
                 add = TRUE)
        }
    }
    abline(a = n_black/n_total,
           b = 0,
           col = 'red',
           lty = 2,
           lwd = 2)
    legend('bottomleft',
           names,
           lwd = 2,
           col = seq_along(predictions),
           bty = 'n',
           inset = c(0.1, 0.07))
    legend('bottomleft',
           paste('Null AUPR:', sprintf('%.3f', round(n_black / n_total, 3))),
           lwd = 2,
           lty = 2,
           col = 'red',
           bty = 'n',
           inset = c(0.1, 0.0))
}
```


## Figures for Paper
```{r}
aupr_figures(list(image_level$filter_0, image_level$filter_50, image_level$filter_200, image_level$filter_240),
             image_level$label,
             'Precision-Recall Curves of Thresholded RVMs',
             c('Threshold 0 AUPR:', 'Threshold 50 AUPR: ', 'Threshold 150 AUPR: ', 'Threshold 220 AUPR: '),
             './out/figures/thresholded_image.png')
```

```{r}
aupr_figures(list(image_level$filter_0_binarized, image_level$filter_50_binarized, image_level$filter_200_binarized, image_level$filter_240_binarized),
             image_level$label,
             'Precision-Recall Curves of Binarized RVMs',
             c('Threshold 0 AUPR:', 'Threshold 50 AUPR: ', 'Threshold 150 AUPR: ', 'Threshold 220 AUPR: '),
             './out/figures/binarized_image.png')
```

```{r}
aupr_figures(list(image_level$filter_0_skeletonized, image_level$filter_50_skeletonized, image_level$filter_200_skeletonized, image_level$filter_240_skeletonized),
             image_level$label,
             'Precision-Recall Curves of Skeletonized RVMs',
             c('Threshold 0 AUPR:', 'Threshold 50 AUPR: ', 'Threshold 150 AUPR: ', 'Threshold 220 AUPR: '),
             './out/figures/skeletonized_image.png')
```

```{r}
aupr_figures(list(subject_level$filter_0, subject_level$filter_50, subject_level$filter_200, subject_level$filter_240),
             subject_level$label,
             'Precision-Recall Curves of Thresholded RVMs',
             c('Threshold 0 AUPR:', 'Threshold 50 AUPR: ', 'Threshold 150 AUPR: ', 'Threshold 220 AUPR: '),
             './out/figures/thresholded_subject.png')
```

```{r}
aupr_figures(list(subject_level$filter_0_binarized, subject_level$filter_50_binarized, subject_level$filter_200_binarized, subject_level$filter_240_binarized),
             subject_level$label,
             'Precision-Recall Curves of Binarized RVMs',
             c('Threshold 0 AUPR:', 'Threshold 50 AUPR: ', 'Threshold 150 AUPR: ', 'Threshold 220 AUPR: '),
             './out/figures/binarized_subject.png')
```

```{r}
aupr_figures(list(subject_level$filter_0_skeletonized, subject_level$filter_50_skeletonized, subject_level$filter_200_skeletonized, subject_level$filter_240_skeletonized),
             subject_level$label,
             'Precision-Recall Curves of Skeletonized RVMs',
             c('Threshold 0 AUPR:', 'Threshold 50 AUPR: ', 'Threshold 150 AUPR: ', 'Threshold 220 AUPR: '),
             './out/figures/skeletonized_subject.png')
```