From ae574e1dea161f375fa29e6c39e3728be6b63169 Mon Sep 17 00:00:00 2001 From: roomrys Date: Thu, 29 Aug 2024 20:54:46 -0700 Subject: [PATCH 01/13] Constrain attrs, python, opencv and bump version --- .conda/meta.yaml | 4 +- .conda_mac/meta.yaml | 12 +- .github/ISSUE_TEMPLATE/bug_report.md | 2 +- .github/workflows/build_manual.yml | 26 +- docs/conf.py | 5 +- docs/installation.md | 6 +- docs/notebooks/Data_structures.ipynb | 2 +- .../Interactive_and_realtime_inference.ipynb | 2 +- .../Interactive_and_resumable_training.ipynb | 2 +- docs/notebooks/Model_evaluation.ipynb | 2 +- docs/notebooks/Post_inference_tracking.ipynb | 2 +- ..._and_inference_on_an_example_dataset.ipynb | 2570 ++++++++--------- ...ing_and_inference_using_Google_Drive.ipynb | 726 ++--- sleap/version.py | 3 +- 14 files changed, 1684 insertions(+), 1680 deletions(-) diff --git a/.conda/meta.yaml b/.conda/meta.yaml index caffe9fcb..0751d7a69 100644 --- a/.conda/meta.yaml +++ b/.conda/meta.yaml @@ -16,7 +16,7 @@ source: path: ../ build: - number: 1 + number: 0 requirements: host: @@ -86,5 +86,7 @@ requirements: - conda-forge::tensorflow-hub <0.14.0 # Causes pynwb conflicts on linux GH-1446 test: + requires: + - libopencv *=headless* # [linux] imports: - sleap \ No newline at end of file diff --git a/.conda_mac/meta.yaml b/.conda_mac/meta.yaml index 7496f2057..80fc6f426 100644 --- a/.conda_mac/meta.yaml +++ b/.conda_mac/meta.yaml @@ -16,14 +16,14 @@ about: summary: {{ data.get('description') }} build: - number: 1 + number: 0 source: path: ../ requirements: host: - - conda-forge::python ~=3.9 + - conda-forge::python >=3.9.0,<3.10.0 - anaconda::numpy >=1.19.5,<1.23.0 - conda-forge::setuptools - conda-forge::packaging @@ -31,7 +31,7 @@ requirements: # Only the packages above are required to build, but listing them all ensures no # unnecessary pypi packages are installed via the build script (bld.bat, build.sh) - - conda-forge::attrs >=21.2.0 + - conda-forge::attrs >=21.2.0,<22.2.0 - conda-forge::cattrs ==1.1.1 - conda-forge::h5py - conda-forge::imgaug ==0.4.0 @@ -57,8 +57,8 @@ requirements: - conda-forge::seaborn run: - - conda-forge::python ~=3.9 - - conda-forge::attrs >=21.2.0 + - conda-forge::python >=3.9.0,<3.10.0 + - conda-forge::attrs >=21.2.0,<22.2.0 - conda-forge::cattrs ==1.1.1 - conda-forge::h5py - conda-forge::imgaug ==0.4.0 @@ -86,5 +86,7 @@ requirements: - conda-forge::tensorflow-hub test: + requires: + - libopencv *=headless* # [linux] imports: - sleap diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 8c95f28dc..e882bab4b 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -28,7 +28,7 @@ Please include information about how you installed. - OS: - Version(s): - + - SLEAP installation method (listed [here](https://sleap.ai/installation.html#)): - [ ] [Conda from package](https://sleap.ai/installation.html#conda-package) - [ ] [Conda from source](https://sleap.ai/installation.html#conda-from-source) diff --git a/.github/workflows/build_manual.yml b/.github/workflows/build_manual.yml index ab689342d..e30233cad 100644 --- a/.github/workflows/build_manual.yml +++ b/.github/workflows/build_manual.yml @@ -3,13 +3,13 @@ name: Build (manual) # Uncomment the following lines to trigger a build if the release build failed -on: - push: - paths: - - '.conda/meta.yaml' - - '.conda_mac/meta.yaml' - branches: - - develop +# on: +# push: +# paths: +# - '.conda/meta.yaml' +# - '.conda_mac/meta.yaml' +# branches: +# - develop jobs: build: @@ -97,7 +97,7 @@ jobs: shell: bash -l {0} run: | yes 2>/dev/null | anaconda login --username sleap --password "$ANACONDA_LOGIN" || true - + # Login to conda (Windows) - name: Login to Anaconda (Windows) if: matrix.os == 'windows-2022' @@ -106,7 +106,7 @@ jobs: shell: powershell run: | echo "yes" | anaconda login --username sleap --password "$env:ANACONDA_LOGIN" - + # Login to conda (Mac) - name: Login to Anaconda (Mac) if: matrix.os == 'macos-latest' @@ -115,28 +115,28 @@ jobs: shell: bash -l {0} run: | yes 2>/dev/null | anaconda login --username sleap --password "$ANACONDA_LOGIN" || true - + # Upload conda package (Windows) - name: Upload conda package (Windows/dev) if: matrix.os == 'windows-2022' shell: powershell run: | anaconda -v upload "build\win-64\*.tar.bz2" --label dev - + # Upload conda package (Ubuntu) - name: Upload conda package (Ubuntu/dev) if: matrix.os == 'ubuntu-22.04' shell: bash -l {0} run: | anaconda -v upload build/linux-64/*.tar.bz2 --label dev - + # Upload conda package (Mac) - name: Upload conda package (Mac/dev) if: matrix.os == 'macos-latest' shell: bash -l {0} run: | anaconda -v upload build/osx-64/*.tar.bz2 --label dev - + - name: Logout from Anaconda shell: bash -l {0} run: | diff --git a/docs/conf.py b/docs/conf.py index 572e73ea0..ef54eac31 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -28,7 +28,7 @@ copyright = f"2019–{date.today().year}, Talmo Lab" # The short X.Y version -version = "1.3.3" +version = "1.3.4" # Get the sleap version # with open("../sleap/version.py") as f: @@ -36,7 +36,7 @@ # version = re.search("\d.+(?=['\"])", version_file).group(0) # Release should be the full branch name -release = "v1.3.3" +release = "v1.3.4" html_title = f"SLEAP ({release})" html_short_title = "SLEAP" @@ -85,6 +85,7 @@ pygments_style = "sphinx" pygments_dark_style = "monokai" + # Autosummary linkcode resolution # https://www.sphinx-doc.org/en/master/usage/extensions/linkcode.html def linkcode_resolve(domain, info): diff --git a/docs/installation.md b/docs/installation.md index eea65cc31..ee2e7eec0 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -137,13 +137,13 @@ SLEAP can be installed three different ways: via {ref}`conda package=1.3.3\"\n", + "!pip install -qqq \"sleap[pypi]>=1.3.4\"\n", "\n", "# But to do it locally, we'd recommend the conda package (available on Windows + Linux):\n", "# conda create -n sleap -c sleap -c conda-forge -c nvidia sleap" diff --git a/docs/notebooks/Interactive_and_realtime_inference.ipynb b/docs/notebooks/Interactive_and_realtime_inference.ipynb index 4a3b612a2..94a20ea3b 100644 --- a/docs/notebooks/Interactive_and_realtime_inference.ipynb +++ b/docs/notebooks/Interactive_and_realtime_inference.ipynb @@ -60,7 +60,7 @@ "source": [ "# This should take care of all the dependencies on colab:\n", "!pip uninstall -qqq -y opencv-python opencv-contrib-python\n", - "!pip install -qqq \"sleap[pypi]>=1.3.3\"\n", + "!pip install -qqq \"sleap[pypi]>=1.3.4\"\n", "\n", "\n", "# But to do it locally, we'd recommend the conda package (available on Windows + Linux):\n", diff --git a/docs/notebooks/Interactive_and_resumable_training.ipynb b/docs/notebooks/Interactive_and_resumable_training.ipynb index f30f036f3..68b4f0715 100644 --- a/docs/notebooks/Interactive_and_resumable_training.ipynb +++ b/docs/notebooks/Interactive_and_resumable_training.ipynb @@ -62,7 +62,7 @@ "source": [ "# This should take care of all the dependencies on colab:\n", "!pip uninstall -qqq -y opencv-python opencv-contrib-python\n", - "!pip install -qqq \"sleap[pypi]>=1.3.3\"\n", + "!pip install -qqq \"sleap[pypi]>=1.3.4\"\n", "\n", "\n", "# But to do it locally, we'd recommend the conda package (available on Windows + Linux):\n", diff --git a/docs/notebooks/Model_evaluation.ipynb b/docs/notebooks/Model_evaluation.ipynb index 41ca6568c..af2b55d51 100644 --- a/docs/notebooks/Model_evaluation.ipynb +++ b/docs/notebooks/Model_evaluation.ipynb @@ -40,7 +40,7 @@ ], "source": [ "!pip uninstall -qqq -y opencv-python opencv-contrib-python\n", - "!pip install -qqq \"sleap[pypi]>=1.3.3\"\n", + "!pip install -qqq \"sleap[pypi]>=1.3.4\"\n", "!apt -qq install tree\n", "!wget -q https://storage.googleapis.com/sleap-data/reference/flies13/td_fast.210505_012601.centered_instance.n%3D1800.zip\n", "!unzip -qq -o -d \"td_fast.210505_012601.centered_instance.n=1800\" \"td_fast.210505_012601.centered_instance.n=1800.zip\"" diff --git a/docs/notebooks/Post_inference_tracking.ipynb b/docs/notebooks/Post_inference_tracking.ipynb index 239176bdb..e91ed002d 100644 --- a/docs/notebooks/Post_inference_tracking.ipynb +++ b/docs/notebooks/Post_inference_tracking.ipynb @@ -61,7 +61,7 @@ "source": [ "# This should take care of all the dependencies on colab:\n", "!pip uninstall -qqq -y opencv-python opencv-contrib-python\n", - "!pip install -qqq \"sleap[pypi]>=1.3.3\"\n", + "!pip install -qqq \"sleap[pypi]>=1.3.4\"\n", "\n", "# But to do it locally, we'd recommend the conda package (available on Windows + Linux):\n", "# conda create -n sleap -c sleap -c conda-forge -c nvidia sleap" diff --git a/docs/notebooks/Training_and_inference_on_an_example_dataset.ipynb b/docs/notebooks/Training_and_inference_on_an_example_dataset.ipynb index b0211bbca..df101eb19 100644 --- a/docs/notebooks/Training_and_inference_on_an_example_dataset.ipynb +++ b/docs/notebooks/Training_and_inference_on_an_example_dataset.ipynb @@ -1,1302 +1,1302 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "view-in-github" - }, - "source": [ - "\"Open" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Training and inference on an example dataset" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LlV70jDuWzea" - }, - "source": [ - "In this notebook we'll install SLEAP, download a sample dataset, run training and inference on that dataset using the SLEAP command-line interface, and then download the predictions." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "yX9noEb8m8re" - }, - "source": [ - "## Install SLEAP\n", - "Note: Before installing SLEAP check [SLEAP releases](https://github.com/talmolab/sleap/releases) page for the latest version." - ] - }, - { - "cell_type": "code", - "execution_count": 36, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 1000 + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "view-in-github" + }, + "source": [ + "\"Open" + ] }, - "id": "DUfnkxMtLcK3", - "outputId": "a6340ef1-eaac-42ef-f8d4-bcc499feb57b" - }, - "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[31mERROR: Cannot uninstall opencv-python 4.6.0, RECORD file not found. Hint: The package was installed by conda.\u001b[0m\u001b[31m\n", - "\u001b[0m\u001b[31mERROR: Cannot uninstall shiboken2 5.15.6, RECORD file not found. You might be able to recover from this via: 'pip install --force-reinstall --no-deps shiboken2==5.15.6'.\u001b[0m\u001b[31m\n", - "\u001b[0m" - ] - } - ], - "source": [ - "!pip uninstall -qqq -y opencv-python opencv-contrib-python\n", - "!pip install -qqq \"sleap[pypi]>=1.3.3\"" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "iq7jrgUksLtR" - }, - "source": [ - "## Download sample training data into Colab\n", - "Let's download a sample dataset from the SLEAP [sample datasets repository](https://github.com/talmolab/sleap-datasets) into Colab." - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Training and inference on an example dataset" + ] }, - "id": "fm3cU1Bc0tWc", - "outputId": "c0ac5677-e3c5-477c-a2f7-44d619208b22" - }, - "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "E: Could not open lock file /var/lib/dpkg/lock-frontend - open (13: Permission denied)\n", - "E: Unable to acquire the dpkg frontend lock (/var/lib/dpkg/lock-frontend), are you root?\n", - "--2023-09-01 13:30:33-- https://github.com/talmolab/sleap-datasets/releases/download/dm-courtship-v1/drosophila-melanogaster-courtship.zip\n", - "Resolving github.com (github.com)... 192.30.255.113\n", - "Connecting to github.com (github.com)|192.30.255.113|:443... connected.\n", - "HTTP request sent, awaiting response... 302 Found\n", - "Location: https://objects.githubusercontent.com/github-production-release-asset-2e65be/263375180/16df8d00-94f1-11ea-98d1-6c03a2f89e1c?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIWNJYAX4CSVEH53A%2F20230901%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20230901T203033Z&X-Amz-Expires=300&X-Amz-Signature=b9b0638744af3144affdc46668c749128bd6c4f23ca2a1313821c7bbcd54ccdd&X-Amz-SignedHeaders=host&actor_id=0&key_id=0&repo_id=263375180&response-content-disposition=attachment%3B%20filename%3Ddrosophila-melanogaster-courtship.zip&response-content-type=application%2Foctet-stream [following]\n", - "--2023-09-01 13:30:33-- https://objects.githubusercontent.com/github-production-release-asset-2e65be/263375180/16df8d00-94f1-11ea-98d1-6c03a2f89e1c?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIWNJYAX4CSVEH53A%2F20230901%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20230901T203033Z&X-Amz-Expires=300&X-Amz-Signature=b9b0638744af3144affdc46668c749128bd6c4f23ca2a1313821c7bbcd54ccdd&X-Amz-SignedHeaders=host&actor_id=0&key_id=0&repo_id=263375180&response-content-disposition=attachment%3B%20filename%3Ddrosophila-melanogaster-courtship.zip&response-content-type=application%2Foctet-stream\n", - "Resolving objects.githubusercontent.com (objects.githubusercontent.com)... 185.199.108.133, 185.199.109.133, 185.199.110.133, ...\n", - "Connecting to objects.githubusercontent.com (objects.githubusercontent.com)|185.199.108.133|:443... connected.\n", - "HTTP request sent, awaiting response... 200 OK\n", - "Length: 111973079 (107M) [application/octet-stream]\n", - "Saving to: ‘dataset.zip’\n", - "\n", - "dataset.zip 100%[===================>] 106.79M 63.0MB/s in 1.7s \n", - "\n", - "2023-09-01 13:30:35 (63.0 MB/s) - ‘dataset.zip’ saved [111973079/111973079]\n", - "\n", - "Archive: dataset.zip\n", - " creating: dataset/drosophila-melanogaster-courtship/\n", - " inflating: dataset/drosophila-melanogaster-courtship/.DS_Store \n", - " creating: dataset/__MACOSX/\n", - " creating: dataset/__MACOSX/drosophila-melanogaster-courtship/\n", - " inflating: dataset/__MACOSX/drosophila-melanogaster-courtship/._.DS_Store \n", - " inflating: dataset/drosophila-melanogaster-courtship/20190128_113421.mp4 \n", - " inflating: dataset/__MACOSX/drosophila-melanogaster-courtship/._20190128_113421.mp4 \n", - " inflating: dataset/drosophila-melanogaster-courtship/courtship_labels.slp \n", - " inflating: dataset/__MACOSX/drosophila-melanogaster-courtship/._courtship_labels.slp \n", - " inflating: dataset/drosophila-melanogaster-courtship/example.jpg \n", - " inflating: dataset/__MACOSX/drosophila-melanogaster-courtship/._example.jpg \n", - "\u001b[01;34mdataset\u001b[00m\n", - "├── \u001b[01;34mdrosophila-melanogaster-courtship\u001b[00m\n", - "│   ├── \u001b[01;32m20190128_113421.mp4\u001b[00m\n", - "│   ├── \u001b[01;32mcourtship_labels.slp\u001b[00m\n", - "│   └── \u001b[01;35mexample.jpg\u001b[00m\n", - "└── \u001b[01;34m__MACOSX\u001b[00m\n", - " └── \u001b[01;34mdrosophila-melanogaster-courtship\u001b[00m\n", - "\n", - "3 directories, 3 files\n" - ] - } - ], - "source": [ - "!apt-get install tree\n", - "!wget -O dataset.zip https://github.com/talmolab/sleap-datasets/releases/download/dm-courtship-v1/drosophila-melanogaster-courtship.zip\n", - "!mkdir dataset\n", - "!unzip dataset.zip -d dataset\n", - "!rm dataset.zip\n", - "!tree dataset" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "xZ-sr67av5uu" - }, - "source": [ - "## Train models\n", - "For the top-down pipeline, we'll need train two models: a centroid model and a centered-instance model.\n", - "\n", - "Using the command-line interface, we'll first train a model for centroids using the default **training profile**. The training profile determines the model architecture, the learning rate, and other parameters.\n", - "\n", - "When you start training, you'll first see the training parameters and then the training and validation loss for each training epoch. \n", - "\n", - "As soon as you're satisfied with the validation loss you see for an epoch during training, you're welcome to stop training by clicking the stop button. The version of the model with the lowest validation loss is saved during training, and that's what will be used for inference.\n", - "\n", - "If you don't stop training, it will run for 200 epochs or until validation loss fails to improve for some number of epochs (controlled by the `early_stopping` fields in the training profile)." - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "metadata": { - "id": "QKf6qzMqNBUi" - }, - "outputs": [ + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "LlV70jDuWzea" + }, + "source": [ + "In this notebook we'll install SLEAP, download a sample dataset, run training and inference on that dataset using the SLEAP command-line interface, and then download the predictions." + ] + }, { - "name": "stdout", - "output_type": "stream", - "text": [ - "INFO:sleap.nn.training:Versions:\n", - "SLEAP: 1.3.2\n", - "TensorFlow: 2.7.0\n", - "Numpy: 1.21.5\n", - "Python: 3.7.12\n", - "OS: Linux-5.15.0-78-generic-x86_64-with-debian-bookworm-sid\n", - "INFO:sleap.nn.training:Training labels file: dataset/drosophila-melanogaster-courtship/courtship_labels.slp\n", - "INFO:sleap.nn.training:Training profile: /home/talmolab/sleap-estimates-animal-poses/pull-requests/sleap/sleap/training_profiles/baseline.centroid.json\n", - "INFO:sleap.nn.training:\n", - "INFO:sleap.nn.training:Arguments:\n", - "INFO:sleap.nn.training:{\n", - " \"training_job_path\": \"baseline.centroid.json\",\n", - " \"labels_path\": \"dataset/drosophila-melanogaster-courtship/courtship_labels.slp\",\n", - " \"video_paths\": [\n", - " \"dataset/drosophila-melanogaster-courtship/20190128_113421.mp4\"\n", - " ],\n", - " \"val_labels\": null,\n", - " \"test_labels\": null,\n", - " \"base_checkpoint\": null,\n", - " \"tensorboard\": false,\n", - " \"save_viz\": false,\n", - " \"zmq\": false,\n", - " \"run_name\": \"courtship.centroid\",\n", - " \"prefix\": \"\",\n", - " \"suffix\": \"\",\n", - " \"cpu\": false,\n", - " \"first_gpu\": false,\n", - " \"last_gpu\": false,\n", - " \"gpu\": \"auto\"\n", - "}\n", - "INFO:sleap.nn.training:\n", - "INFO:sleap.nn.training:Training job:\n", - "INFO:sleap.nn.training:{\n", - " \"data\": {\n", - " \"labels\": {\n", - " \"training_labels\": null,\n", - " \"validation_labels\": null,\n", - " \"validation_fraction\": 0.1,\n", - " \"test_labels\": null,\n", - " \"split_by_inds\": false,\n", - " \"training_inds\": null,\n", - " \"validation_inds\": null,\n", - " \"test_inds\": null,\n", - " \"search_path_hints\": [],\n", - " \"skeletons\": []\n", - " },\n", - " \"preprocessing\": {\n", - " \"ensure_rgb\": false,\n", - " \"ensure_grayscale\": false,\n", - " \"imagenet_mode\": null,\n", - " \"input_scaling\": 0.5,\n", - " \"pad_to_stride\": null,\n", - " \"resize_and_pad_to_target\": true,\n", - " \"target_height\": null,\n", - " \"target_width\": null\n", - " },\n", - " \"instance_cropping\": {\n", - " \"center_on_part\": null,\n", - " \"crop_size\": null,\n", - " \"crop_size_detection_padding\": 16\n", - " }\n", - " },\n", - " \"model\": {\n", - " \"backbone\": {\n", - " \"leap\": null,\n", - " \"unet\": {\n", - " \"stem_stride\": null,\n", - " \"max_stride\": 16,\n", - " \"output_stride\": 2,\n", - " \"filters\": 16,\n", - " \"filters_rate\": 2.0,\n", - " \"middle_block\": true,\n", - " \"up_interpolate\": true,\n", - " \"stacks\": 1\n", - " },\n", - " \"hourglass\": null,\n", - " \"resnet\": null,\n", - " \"pretrained_encoder\": null\n", - " },\n", - " \"heads\": {\n", - " \"single_instance\": null,\n", - " \"centroid\": {\n", - " \"anchor_part\": null,\n", - " \"sigma\": 2.5,\n", - " \"output_stride\": 2,\n", - " \"loss_weight\": 1.0,\n", - " \"offset_refinement\": false\n", - " },\n", - " \"centered_instance\": null,\n", - " \"multi_instance\": null,\n", - " \"multi_class_bottomup\": null,\n", - " \"multi_class_topdown\": null\n", - " },\n", - " \"base_checkpoint\": null\n", - " },\n", - " \"optimization\": {\n", - " \"preload_data\": true,\n", - " \"augmentation_config\": {\n", - " \"rotate\": true,\n", - " \"rotation_min_angle\": -15.0,\n", - " \"rotation_max_angle\": 15.0,\n", - " \"translate\": false,\n", - " \"translate_min\": -5,\n", - " \"translate_max\": 5,\n", - " \"scale\": false,\n", - " \"scale_min\": 0.9,\n", - " \"scale_max\": 1.1,\n", - " \"uniform_noise\": false,\n", - " \"uniform_noise_min_val\": 0.0,\n", - " \"uniform_noise_max_val\": 10.0,\n", - " \"gaussian_noise\": false,\n", - " \"gaussian_noise_mean\": 5.0,\n", - " \"gaussian_noise_stddev\": 1.0,\n", - " \"contrast\": false,\n", - " \"contrast_min_gamma\": 0.5,\n", - " \"contrast_max_gamma\": 2.0,\n", - " \"brightness\": false,\n", - " \"brightness_min_val\": 0.0,\n", - " \"brightness_max_val\": 10.0,\n", - " \"random_crop\": false,\n", - " \"random_crop_height\": 256,\n", - " \"random_crop_width\": 256,\n", - " \"random_flip\": false,\n", - " \"flip_horizontal\": true\n", - " },\n", - " \"online_shuffling\": true,\n", - " \"shuffle_buffer_size\": 128,\n", - " \"prefetch\": true,\n", - " \"batch_size\": 4,\n", - " \"batches_per_epoch\": null,\n", - " \"min_batches_per_epoch\": 200,\n", - " \"val_batches_per_epoch\": null,\n", - " \"min_val_batches_per_epoch\": 10,\n", - " \"epochs\": 200,\n", - " \"optimizer\": \"adam\",\n", - " \"initial_learning_rate\": 0.0001,\n", - " \"learning_rate_schedule\": {\n", - " \"reduce_on_plateau\": true,\n", - " \"reduction_factor\": 0.5,\n", - " \"plateau_min_delta\": 1e-08,\n", - " \"plateau_patience\": 5,\n", - " \"plateau_cooldown\": 3,\n", - " \"min_learning_rate\": 1e-08\n", - " },\n", - " \"hard_keypoint_mining\": {\n", - " \"online_mining\": false,\n", - " \"hard_to_easy_ratio\": 2.0,\n", - " \"min_hard_keypoints\": 2,\n", - " \"max_hard_keypoints\": null,\n", - " \"loss_scale\": 5.0\n", - " },\n", - " \"early_stopping\": {\n", - " \"stop_training_on_plateau\": true,\n", - " \"plateau_min_delta\": 1e-08,\n", - " \"plateau_patience\": 20\n", - " }\n", - " },\n", - " \"outputs\": {\n", - " \"save_outputs\": true,\n", - " \"run_name\": \"courtship.centroid\",\n", - " \"run_name_prefix\": \"\",\n", - " \"run_name_suffix\": null,\n", - " \"runs_folder\": \"models\",\n", - " \"tags\": [],\n", - " \"save_visualizations\": true,\n", - " \"delete_viz_images\": true,\n", - " \"zip_outputs\": false,\n", - " \"log_to_csv\": true,\n", - " \"checkpointing\": {\n", - " \"initial_model\": false,\n", - " \"best_model\": true,\n", - " \"every_epoch\": false,\n", - " \"latest_model\": false,\n", - " \"final_model\": false\n", - " },\n", - " \"tensorboard\": {\n", - " \"write_logs\": false,\n", - " \"loss_frequency\": \"epoch\",\n", - " \"architecture_graph\": false,\n", - " \"profile_graph\": false,\n", - " \"visualizations\": true\n", - " },\n", - " \"zmq\": {\n", - " \"subscribe_to_controller\": false,\n", - " \"controller_address\": \"tcp://127.0.0.1:9000\",\n", - " \"controller_polling_timeout\": 10,\n", - " \"publish_updates\": false,\n", - " \"publish_address\": \"tcp://127.0.0.1:9001\"\n", - " }\n", - " },\n", - " \"name\": \"\",\n", - " \"description\": \"\",\n", - " \"sleap_version\": \"1.3.2\",\n", - " \"filename\": \"/home/talmolab/sleap-estimates-animal-poses/pull-requests/sleap/sleap/training_profiles/baseline.centroid.json\"\n", - "}\n", - "INFO:sleap.nn.training:\n", - "2023-09-01 13:30:38.827290: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:939] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n", - "2023-09-01 13:30:38.831845: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:939] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n", - "2023-09-01 13:30:38.832633: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:939] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n", - "INFO:sleap.nn.training:Auto-selected GPU 0 with 22980 MiB of free memory.\n", - "INFO:sleap.nn.training:Using GPU 0 for acceleration.\n", - "INFO:sleap.nn.training:Disabled GPU memory pre-allocation.\n", - "INFO:sleap.nn.training:System:\n", - "GPUs: 1/1 available\n", - " Device: /physical_device:GPU:0\n", - " Available: True\n", - " Initalized: False\n", - " Memory growth: True\n", - "INFO:sleap.nn.training:\n", - "INFO:sleap.nn.training:Initializing trainer...\n", - "INFO:sleap.nn.training:Loading training labels from: dataset/drosophila-melanogaster-courtship/courtship_labels.slp\n", - "INFO:sleap.nn.training:Creating training and validation splits from validation fraction: 0.1\n", - "INFO:sleap.nn.training: Splits: Training = 134 / Validation = 15.\n", - "INFO:sleap.nn.training:Setting up for training...\n", - "INFO:sleap.nn.training:Setting up pipeline builders...\n", - "INFO:sleap.nn.training:Setting up model...\n", - "INFO:sleap.nn.training:Building test pipeline...\n", - "2023-09-01 13:30:39.755154: I tensorflow/core/platform/cpu_feature_guard.cc:151] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", - "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "2023-09-01 13:30:39.756024: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:939] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n", - "2023-09-01 13:30:39.757213: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:939] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n", - "2023-09-01 13:30:39.758315: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:939] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n", - "2023-09-01 13:30:40.089801: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:939] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n", - "2023-09-01 13:30:40.090652: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:939] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n", - "2023-09-01 13:30:40.091464: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:939] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n", - "2023-09-01 13:30:40.092164: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1525] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 21084 MB memory: -> device: 0, name: NVIDIA RTX A5000, pci bus id: 0000:01:00.0, compute capability: 8.6\n", - "INFO:sleap.nn.training:Loaded test example. [1.326s]\n", - "INFO:sleap.nn.training: Input shape: (512, 512, 3)\n", - "INFO:sleap.nn.training:Created Keras model.\n", - "INFO:sleap.nn.training: Backbone: UNet(stacks=1, filters=16, filters_rate=2.0, kernel_size=3, stem_kernel_size=7, convs_per_block=2, stem_blocks=0, down_blocks=4, middle_block=True, up_blocks=3, up_interpolate=True, block_contraction=False)\n", - "INFO:sleap.nn.training: Max stride: 16\n", - "INFO:sleap.nn.training: Parameters: 1,953,393\n", - "INFO:sleap.nn.training: Heads: \n", - "INFO:sleap.nn.training: [0] = CentroidConfmapsHead(anchor_part=None, sigma=2.5, output_stride=2, loss_weight=1.0)\n", - "INFO:sleap.nn.training: Outputs: \n", - "INFO:sleap.nn.training: [0] = KerasTensor(type_spec=TensorSpec(shape=(None, 256, 256, 1), dtype=tf.float32, name=None), name='CentroidConfmapsHead/BiasAdd:0', description=\"created by layer 'CentroidConfmapsHead'\")\n", - "INFO:sleap.nn.training:Training from scratch\n", - "INFO:sleap.nn.training:Setting up data pipelines...\n", - "INFO:sleap.nn.training:Training set: n = 134\n", - "INFO:sleap.nn.training:Validation set: n = 15\n", - "INFO:sleap.nn.training:Setting up optimization...\n", - "INFO:sleap.nn.training: Learning rate schedule: LearningRateScheduleConfig(reduce_on_plateau=True, reduction_factor=0.5, plateau_min_delta=1e-08, plateau_patience=5, plateau_cooldown=3, min_learning_rate=1e-08)\n", - "INFO:sleap.nn.training: Early stopping: EarlyStoppingConfig(stop_training_on_plateau=True, plateau_min_delta=1e-08, plateau_patience=20)\n", - "INFO:sleap.nn.training:Setting up outputs...\n", - "INFO:sleap.nn.training:Created run path: models/courtship.centroid\n", - "INFO:sleap.nn.training:Setting up visualization...\n", - "INFO:sleap.nn.training:Finished trainer set up. [3.5s]\n", - "INFO:sleap.nn.training:Creating tf.data.Datasets for training data generation...\n", - "INFO:sleap.nn.training:Finished creating training datasets. [5.4s]\n", - "INFO:sleap.nn.training:Starting training loop...\n", - "Epoch 1/200\n", - "2023-09-01 13:30:49.814560: I tensorflow/stream_executor/cuda/cuda_dnn.cc:366] Loaded cuDNN version 8201\n", - "2023-09-01 13:31:07.940585: I tensorflow/stream_executor/cuda/cuda_blas.cc:1774] TensorFloat-32 will be used for the matrix multiplication. This will only be logged once.\n", - "200/200 - 20s - loss: 2.5945e-04 - val_loss: 1.5190e-04 - lr: 1.0000e-04 - 20s/epoch - 99ms/step\n", - "Epoch 2/200\n", - "200/200 - 11s - loss: 1.2513e-04 - val_loss: 9.5694e-05 - lr: 1.0000e-04 - 11s/epoch - 57ms/step\n", - "Epoch 3/200\n", - "200/200 - 11s - loss: 9.6987e-05 - val_loss: 6.8224e-05 - lr: 1.0000e-04 - 11s/epoch - 57ms/step\n", - "Epoch 4/200\n", - "200/200 - 12s - loss: 8.1486e-05 - val_loss: 5.0657e-05 - lr: 1.0000e-04 - 12s/epoch - 58ms/step\n", - "Epoch 5/200\n", - "200/200 - 11s - loss: 7.2174e-05 - val_loss: 5.3859e-05 - lr: 1.0000e-04 - 11s/epoch - 55ms/step\n", - "Epoch 6/200\n", - "200/200 - 11s - loss: 5.9181e-05 - val_loss: 7.0259e-05 - lr: 1.0000e-04 - 11s/epoch - 55ms/step\n", - "Epoch 7/200\n", - "200/200 - 11s - loss: 4.9353e-05 - val_loss: 4.9832e-05 - lr: 1.0000e-04 - 11s/epoch - 57ms/step\n", - "Epoch 8/200\n", - "200/200 - 11s - loss: 3.8997e-05 - val_loss: 4.4787e-05 - lr: 1.0000e-04 - 11s/epoch - 55ms/step\n", - "Epoch 9/200\n", - "200/200 - 11s - loss: 3.5596e-05 - val_loss: 6.5150e-05 - lr: 1.0000e-04 - 11s/epoch - 55ms/step\n", - "Epoch 10/200\n", - "200/200 - 12s - loss: 2.9256e-05 - val_loss: 3.8968e-05 - lr: 1.0000e-04 - 12s/epoch - 58ms/step\n", - "Epoch 11/200\n", - "200/200 - 11s - loss: 2.8572e-05 - val_loss: 3.5451e-05 - lr: 1.0000e-04 - 11s/epoch - 55ms/step\n", - "Epoch 12/200\n", - "200/200 - 11s - loss: 2.2156e-05 - val_loss: 4.8602e-05 - lr: 1.0000e-04 - 11s/epoch - 53ms/step\n", - "Epoch 13/200\n", - "200/200 - 11s - loss: 1.7656e-05 - val_loss: 4.1905e-05 - lr: 1.0000e-04 - 11s/epoch - 55ms/step\n", - "Epoch 14/200\n", - "200/200 - 11s - loss: 1.6440e-05 - val_loss: 3.6607e-05 - lr: 1.0000e-04 - 11s/epoch - 55ms/step\n", - "Epoch 15/200\n", - "200/200 - 11s - loss: 1.4415e-05 - val_loss: 4.1699e-05 - lr: 1.0000e-04 - 11s/epoch - 55ms/step\n", - "Epoch 16/200\n", - "200/200 - 11s - loss: 1.3589e-05 - val_loss: 3.5362e-05 - lr: 1.0000e-04 - 11s/epoch - 56ms/step\n", - "Epoch 17/200\n", - "200/200 - 11s - loss: 1.0888e-05 - val_loss: 2.1600e-05 - lr: 1.0000e-04 - 11s/epoch - 56ms/step\n", - "Epoch 18/200\n", - "200/200 - 11s - loss: 1.0426e-05 - val_loss: 3.6782e-05 - lr: 1.0000e-04 - 11s/epoch - 55ms/step\n", - "Epoch 19/200\n", - "200/200 - 11s - loss: 9.9092e-06 - val_loss: 3.8284e-05 - lr: 1.0000e-04 - 11s/epoch - 56ms/step\n", - "Epoch 20/200\n", - "200/200 - 11s - loss: 8.0018e-06 - val_loss: 2.9439e-05 - lr: 1.0000e-04 - 11s/epoch - 57ms/step\n", - "Epoch 21/200\n", - "200/200 - 11s - loss: 7.7977e-06 - val_loss: 2.8703e-05 - lr: 1.0000e-04 - 11s/epoch - 56ms/step\n", - "Epoch 22/200\n", - "\n", - "Epoch 00022: ReduceLROnPlateau reducing learning rate to 4.999999873689376e-05.\n", - "200/200 - 11s - loss: 6.5981e-06 - val_loss: 3.6030e-05 - lr: 1.0000e-04 - 11s/epoch - 55ms/step\n", - "Epoch 23/200\n", - "200/200 - 11s - loss: 4.6479e-06 - val_loss: 2.8081e-05 - lr: 5.0000e-05 - 11s/epoch - 55ms/step\n", - "Epoch 24/200\n", - "200/200 - 11s - loss: 4.2579e-06 - val_loss: 3.7954e-05 - lr: 5.0000e-05 - 11s/epoch - 55ms/step\n", - "Epoch 25/200\n", - "200/200 - 11s - loss: 3.9628e-06 - val_loss: 2.6399e-05 - lr: 5.0000e-05 - 11s/epoch - 56ms/step\n", - "Epoch 26/200\n", - "200/200 - 11s - loss: 3.6915e-06 - val_loss: 1.9973e-05 - lr: 5.0000e-05 - 11s/epoch - 56ms/step\n", - "Epoch 27/200\n", - "200/200 - 11s - loss: 3.4726e-06 - val_loss: 3.5831e-05 - lr: 5.0000e-05 - 11s/epoch - 55ms/step\n", - "Epoch 28/200\n", - "200/200 - 11s - loss: 3.2110e-06 - val_loss: 2.7290e-05 - lr: 5.0000e-05 - 11s/epoch - 56ms/step\n", - "Epoch 29/200\n", - "200/200 - 11s - loss: 3.3421e-06 - val_loss: 3.1827e-05 - lr: 5.0000e-05 - 11s/epoch - 56ms/step\n", - "Epoch 30/200\n", - "200/200 - 11s - loss: 3.3472e-06 - val_loss: 3.4653e-05 - lr: 5.0000e-05 - 11s/epoch - 56ms/step\n", - "Epoch 31/200\n", - "\n", - "Epoch 00031: ReduceLROnPlateau reducing learning rate to 2.499999936844688e-05.\n", - "200/200 - 11s - loss: 3.1221e-06 - val_loss: 2.7741e-05 - lr: 5.0000e-05 - 11s/epoch - 56ms/step\n", - "Epoch 32/200\n", - "200/200 - 11s - loss: 2.5739e-06 - val_loss: 3.2486e-05 - lr: 2.5000e-05 - 11s/epoch - 55ms/step\n", - "Epoch 33/200\n", - "200/200 - 11s - loss: 2.5589e-06 - val_loss: 3.3135e-05 - lr: 2.5000e-05 - 11s/epoch - 56ms/step\n", - "Epoch 34/200\n", - "200/200 - 11s - loss: 2.4215e-06 - val_loss: 2.8923e-05 - lr: 2.5000e-05 - 11s/epoch - 56ms/step\n", - "Epoch 35/200\n", - "200/200 - 11s - loss: 2.4033e-06 - val_loss: 2.8776e-05 - lr: 2.5000e-05 - 11s/epoch - 56ms/step\n", - "Epoch 36/200\n", - "200/200 - 11s - loss: 2.3358e-06 - val_loss: 2.5874e-05 - lr: 2.5000e-05 - 11s/epoch - 56ms/step\n", - "Epoch 37/200\n", - "200/200 - 11s - loss: 2.2922e-06 - val_loss: 3.6051e-05 - lr: 2.5000e-05 - 11s/epoch - 55ms/step\n", - "Epoch 38/200\n", - "\n", - "Epoch 00038: ReduceLROnPlateau reducing learning rate to 1.249999968422344e-05.\n", - "200/200 - 11s - loss: 2.1278e-06 - val_loss: 2.4898e-05 - lr: 2.5000e-05 - 11s/epoch - 55ms/step\n", - "Epoch 39/200\n", - "200/200 - 11s - loss: 2.0474e-06 - val_loss: 2.8901e-05 - lr: 1.2500e-05 - 11s/epoch - 56ms/step\n", - "Epoch 40/200\n", - "200/200 - 11s - loss: 2.0612e-06 - val_loss: 3.7469e-05 - lr: 1.2500e-05 - 11s/epoch - 56ms/step\n", - "Epoch 41/200\n", - "200/200 - 11s - loss: 1.8414e-06 - val_loss: 2.8496e-05 - lr: 1.2500e-05 - 11s/epoch - 56ms/step\n", - "Epoch 42/200\n", - "200/200 - 11s - loss: 2.0196e-06 - val_loss: 3.5206e-05 - lr: 1.2500e-05 - 11s/epoch - 56ms/step\n", - "Epoch 43/200\n", - "200/200 - 11s - loss: 1.8551e-06 - val_loss: 2.6483e-05 - lr: 1.2500e-05 - 11s/epoch - 56ms/step\n", - "Epoch 44/200\n", - "200/200 - 11s - loss: 1.9705e-06 - val_loss: 2.4643e-05 - lr: 1.2500e-05 - 11s/epoch - 55ms/step\n", - "Epoch 45/200\n", - "\n", - "Epoch 00045: ReduceLROnPlateau reducing learning rate to 6.24999984211172e-06.\n", - "200/200 - 11s - loss: 1.9136e-06 - val_loss: 2.8379e-05 - lr: 1.2500e-05 - 11s/epoch - 56ms/step\n", - "Epoch 46/200\n", - "200/200 - 11s - loss: 1.7911e-06 - val_loss: 4.0055e-05 - lr: 6.2500e-06 - 11s/epoch - 56ms/step\n", - "Epoch 00046: early stopping\n", - "INFO:sleap.nn.training:Finished training loop. [8.7 min]\n", - "INFO:sleap.nn.training:Deleting visualization directory: models/courtship.centroid/viz\n", - "INFO:sleap.nn.training:Saving evaluation metrics to model folder...\n", - "\u001b[2KPredicting... \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m100%\u001b[0m ETA: \u001b[36m0:00:00\u001b[0m \u001b[31m33.7 FPS\u001b[0m31m51.9 FPS\u001b[0m31m52.6 FPS\u001b[0mFPS\u001b[0m\n", - "\u001b[?25hINFO:sleap.nn.evals:Saved predictions: models/courtship.centroid/labels_pr.train.slp\n", - "INFO:sleap.nn.evals:Saved metrics: models/courtship.centroid/metrics.train.npz\n", - "INFO:sleap.nn.evals:OKS mAP: 0.725241\n", - "\u001b[2KPredicting... \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m100%\u001b[0m ETA: \u001b[36m0:00:00\u001b[0m \u001b[31m7.3 FPS\u001b[0m0:00:01\u001b[0m \u001b[31m184.6 FPS\u001b[0mm\n", - "\u001b[?25hINFO:sleap.nn.evals:Saved predictions: models/courtship.centroid/labels_pr.val.slp\n", - "INFO:sleap.nn.evals:Saved metrics: models/courtship.centroid/metrics.val.npz\n", - "INFO:sleap.nn.evals:OKS mAP: 0.870526\n" - ] - } - ], - "source": [ - "!sleap-train baseline.centroid.json \"dataset/drosophila-melanogaster-courtship/courtship_labels.slp\" --run_name \"courtship.centroid\" --video-paths \"dataset/drosophila-melanogaster-courtship/20190128_113421.mp4\"" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Vm3i0ry04IMx" - }, - "source": [ - "Let's now train a centered-instance model." - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "metadata": { - "id": "ufbULTDw4Hbh" - }, - "outputs": [ + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "yX9noEb8m8re" + }, + "source": [ + "## Install SLEAP\n", + "Note: Before installing SLEAP check [SLEAP releases](https://github.com/talmolab/sleap/releases) page for the latest version." + ] + }, { - "name": "stdout", - "output_type": "stream", - "text": [ - "INFO:sleap.nn.training:Versions:\n", - "SLEAP: 1.3.2\n", - "TensorFlow: 2.7.0\n", - "Numpy: 1.21.5\n", - "Python: 3.7.12\n", - "OS: Linux-5.15.0-78-generic-x86_64-with-debian-bookworm-sid\n", - "INFO:sleap.nn.training:Training labels file: dataset/drosophila-melanogaster-courtship/courtship_labels.slp\n", - "INFO:sleap.nn.training:Training profile: /home/talmolab/sleap-estimates-animal-poses/pull-requests/sleap/sleap/training_profiles/baseline_medium_rf.topdown.json\n", - "INFO:sleap.nn.training:\n", - "INFO:sleap.nn.training:Arguments:\n", - "INFO:sleap.nn.training:{\n", - " \"training_job_path\": \"baseline_medium_rf.topdown.json\",\n", - " \"labels_path\": \"dataset/drosophila-melanogaster-courtship/courtship_labels.slp\",\n", - " \"video_paths\": [\n", - " \"dataset/drosophila-melanogaster-courtship/20190128_113421.mp4\"\n", - " ],\n", - " \"val_labels\": null,\n", - " \"test_labels\": null,\n", - " \"base_checkpoint\": null,\n", - " \"tensorboard\": false,\n", - " \"save_viz\": false,\n", - " \"zmq\": false,\n", - " \"run_name\": \"courtship.topdown_confmaps\",\n", - " \"prefix\": \"\",\n", - " \"suffix\": \"\",\n", - " \"cpu\": false,\n", - " \"first_gpu\": false,\n", - " \"last_gpu\": false,\n", - " \"gpu\": \"auto\"\n", - "}\n", - "INFO:sleap.nn.training:\n", - "INFO:sleap.nn.training:Training job:\n", - "INFO:sleap.nn.training:{\n", - " \"data\": {\n", - " \"labels\": {\n", - " \"training_labels\": null,\n", - " \"validation_labels\": null,\n", - " \"validation_fraction\": 0.1,\n", - " \"test_labels\": null,\n", - " \"split_by_inds\": false,\n", - " \"training_inds\": null,\n", - " \"validation_inds\": null,\n", - " \"test_inds\": null,\n", - " \"search_path_hints\": [],\n", - " \"skeletons\": []\n", - " },\n", - " \"preprocessing\": {\n", - " \"ensure_rgb\": false,\n", - " \"ensure_grayscale\": false,\n", - " \"imagenet_mode\": null,\n", - " \"input_scaling\": 1.0,\n", - " \"pad_to_stride\": null,\n", - " \"resize_and_pad_to_target\": true,\n", - " \"target_height\": null,\n", - " \"target_width\": null\n", - " },\n", - " \"instance_cropping\": {\n", - " \"center_on_part\": null,\n", - " \"crop_size\": null,\n", - " \"crop_size_detection_padding\": 16\n", - " }\n", - " },\n", - " \"model\": {\n", - " \"backbone\": {\n", - " \"leap\": null,\n", - " \"unet\": {\n", - " \"stem_stride\": null,\n", - " \"max_stride\": 16,\n", - " \"output_stride\": 4,\n", - " \"filters\": 24,\n", - " \"filters_rate\": 2.0,\n", - " \"middle_block\": true,\n", - " \"up_interpolate\": true,\n", - " \"stacks\": 1\n", - " },\n", - " \"hourglass\": null,\n", - " \"resnet\": null,\n", - " \"pretrained_encoder\": null\n", - " },\n", - " \"heads\": {\n", - " \"single_instance\": null,\n", - " \"centroid\": null,\n", - " \"centered_instance\": {\n", - " \"anchor_part\": null,\n", - " \"part_names\": null,\n", - " \"sigma\": 2.5,\n", - " \"output_stride\": 4,\n", - " \"loss_weight\": 1.0,\n", - " \"offset_refinement\": false\n", - " },\n", - " \"multi_instance\": null,\n", - " \"multi_class_bottomup\": null,\n", - " \"multi_class_topdown\": null\n", - " },\n", - " \"base_checkpoint\": null\n", - " },\n", - " \"optimization\": {\n", - " \"preload_data\": true,\n", - " \"augmentation_config\": {\n", - " \"rotate\": true,\n", - " \"rotation_min_angle\": -15.0,\n", - " \"rotation_max_angle\": 15.0,\n", - " \"translate\": false,\n", - " \"translate_min\": -5,\n", - " \"translate_max\": 5,\n", - " \"scale\": false,\n", - " \"scale_min\": 0.9,\n", - " \"scale_max\": 1.1,\n", - " \"uniform_noise\": false,\n", - " \"uniform_noise_min_val\": 0.0,\n", - " \"uniform_noise_max_val\": 10.0,\n", - " \"gaussian_noise\": false,\n", - " \"gaussian_noise_mean\": 5.0,\n", - " \"gaussian_noise_stddev\": 1.0,\n", - " \"contrast\": false,\n", - " \"contrast_min_gamma\": 0.5,\n", - " \"contrast_max_gamma\": 2.0,\n", - " \"brightness\": false,\n", - " \"brightness_min_val\": 0.0,\n", - " \"brightness_max_val\": 10.0,\n", - " \"random_crop\": false,\n", - " \"random_crop_height\": 256,\n", - " \"random_crop_width\": 256,\n", - " \"random_flip\": false,\n", - " \"flip_horizontal\": true\n", - " },\n", - " \"online_shuffling\": true,\n", - " \"shuffle_buffer_size\": 128,\n", - " \"prefetch\": true,\n", - " \"batch_size\": 4,\n", - " \"batches_per_epoch\": null,\n", - " \"min_batches_per_epoch\": 200,\n", - " \"val_batches_per_epoch\": null,\n", - " \"min_val_batches_per_epoch\": 10,\n", - " \"epochs\": 200,\n", - " \"optimizer\": \"adam\",\n", - " \"initial_learning_rate\": 0.0001,\n", - " \"learning_rate_schedule\": {\n", - " \"reduce_on_plateau\": true,\n", - " \"reduction_factor\": 0.5,\n", - " \"plateau_min_delta\": 1e-08,\n", - " \"plateau_patience\": 5,\n", - " \"plateau_cooldown\": 3,\n", - " \"min_learning_rate\": 1e-08\n", - " },\n", - " \"hard_keypoint_mining\": {\n", - " \"online_mining\": false,\n", - " \"hard_to_easy_ratio\": 2.0,\n", - " \"min_hard_keypoints\": 2,\n", - " \"max_hard_keypoints\": null,\n", - " \"loss_scale\": 5.0\n", - " },\n", - " \"early_stopping\": {\n", - " \"stop_training_on_plateau\": true,\n", - " \"plateau_min_delta\": 1e-08,\n", - " \"plateau_patience\": 10\n", - " }\n", - " },\n", - " \"outputs\": {\n", - " \"save_outputs\": true,\n", - " \"run_name\": \"courtship.topdown_confmaps\",\n", - " \"run_name_prefix\": \"\",\n", - " \"run_name_suffix\": null,\n", - " \"runs_folder\": \"models\",\n", - " \"tags\": [],\n", - " \"save_visualizations\": true,\n", - " \"delete_viz_images\": true,\n", - " \"zip_outputs\": false,\n", - " \"log_to_csv\": true,\n", - " \"checkpointing\": {\n", - " \"initial_model\": false,\n", - " \"best_model\": true,\n", - " \"every_epoch\": false,\n", - " \"latest_model\": false,\n", - " \"final_model\": false\n", - " },\n", - " \"tensorboard\": {\n", - " \"write_logs\": false,\n", - " \"loss_frequency\": \"epoch\",\n", - " \"architecture_graph\": true,\n", - " \"profile_graph\": false,\n", - " \"visualizations\": true\n", - " },\n", - " \"zmq\": {\n", - " \"subscribe_to_controller\": false,\n", - " \"controller_address\": \"tcp://127.0.0.1:9000\",\n", - " \"controller_polling_timeout\": 10,\n", - " \"publish_updates\": false,\n", - " \"publish_address\": \"tcp://127.0.0.1:9001\"\n", - " }\n", - " },\n", - " \"name\": \"\",\n", - " \"description\": \"\",\n", - " \"sleap_version\": \"1.3.2\",\n", - " \"filename\": \"/home/talmolab/sleap-estimates-animal-poses/pull-requests/sleap/sleap/training_profiles/baseline_medium_rf.topdown.json\"\n", - "}\n", - "INFO:sleap.nn.training:\n", - "2023-09-01 13:39:43.324520: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:939] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n", - "2023-09-01 13:39:43.329181: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:939] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n", - "2023-09-01 13:39:43.329961: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:939] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n", - "INFO:sleap.nn.training:Auto-selected GPU 0 with 23056 MiB of free memory.\n", - "INFO:sleap.nn.training:Using GPU 0 for acceleration.\n", - "INFO:sleap.nn.training:Disabled GPU memory pre-allocation.\n", - "INFO:sleap.nn.training:System:\n", - "GPUs: 1/1 available\n", - " Device: /physical_device:GPU:0\n", - " Available: True\n", - " Initalized: False\n", - " Memory growth: True\n", - "INFO:sleap.nn.training:\n", - "INFO:sleap.nn.training:Initializing trainer...\n", - "INFO:sleap.nn.training:Loading training labels from: dataset/drosophila-melanogaster-courtship/courtship_labels.slp\n", - "INFO:sleap.nn.training:Creating training and validation splits from validation fraction: 0.1\n", - "INFO:sleap.nn.training: Splits: Training = 134 / Validation = 15.\n", - "INFO:sleap.nn.training:Setting up for training...\n", - "INFO:sleap.nn.training:Setting up pipeline builders...\n", - "INFO:sleap.nn.training:Setting up model...\n", - "INFO:sleap.nn.training:Building test pipeline...\n", - "2023-09-01 13:39:44.254912: I tensorflow/core/platform/cpu_feature_guard.cc:151] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", - "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "2023-09-01 13:39:44.255468: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:939] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n", - "2023-09-01 13:39:44.256291: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:939] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n", - "2023-09-01 13:39:44.257158: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:939] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n", - "2023-09-01 13:39:44.546117: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:939] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n", - "2023-09-01 13:39:44.546866: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:939] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n", - "2023-09-01 13:39:44.547533: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:939] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n", - "2023-09-01 13:39:44.548184: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1525] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 21151 MB memory: -> device: 0, name: NVIDIA RTX A5000, pci bus id: 0000:01:00.0, compute capability: 8.6\n", - "INFO:sleap.nn.training:Loaded test example. [1.684s]\n", - "INFO:sleap.nn.training: Input shape: (144, 144, 3)\n", - "INFO:sleap.nn.training:Created Keras model.\n", - "INFO:sleap.nn.training: Backbone: UNet(stacks=1, filters=24, filters_rate=2.0, kernel_size=3, stem_kernel_size=7, convs_per_block=2, stem_blocks=0, down_blocks=4, middle_block=True, up_blocks=2, up_interpolate=True, block_contraction=False)\n", - "INFO:sleap.nn.training: Max stride: 16\n", - "INFO:sleap.nn.training: Parameters: 4,311,877\n", - "INFO:sleap.nn.training: Heads: \n", - "INFO:sleap.nn.training: [0] = CenteredInstanceConfmapsHead(part_names=['head', 'thorax', 'abdomen', 'wingL', 'wingR', 'forelegL4', 'forelegR4', 'midlegL4', 'midlegR4', 'hindlegL4', 'hindlegR4', 'eyeL', 'eyeR'], anchor_part=None, sigma=2.5, output_stride=4, loss_weight=1.0)\n", - "INFO:sleap.nn.training: Outputs: \n", - "INFO:sleap.nn.training: [0] = KerasTensor(type_spec=TensorSpec(shape=(None, 36, 36, 13), dtype=tf.float32, name=None), name='CenteredInstanceConfmapsHead/BiasAdd:0', description=\"created by layer 'CenteredInstanceConfmapsHead'\")\n", - "INFO:sleap.nn.training:Training from scratch\n", - "INFO:sleap.nn.training:Setting up data pipelines...\n", - "INFO:sleap.nn.training:Training set: n = 134\n", - "INFO:sleap.nn.training:Validation set: n = 15\n", - "INFO:sleap.nn.training:Setting up optimization...\n", - "INFO:sleap.nn.training: Learning rate schedule: LearningRateScheduleConfig(reduce_on_plateau=True, reduction_factor=0.5, plateau_min_delta=1e-08, plateau_patience=5, plateau_cooldown=3, min_learning_rate=1e-08)\n", - "INFO:sleap.nn.training: Early stopping: EarlyStoppingConfig(stop_training_on_plateau=True, plateau_min_delta=1e-08, plateau_patience=10)\n", - "INFO:sleap.nn.training:Setting up outputs...\n", - "INFO:sleap.nn.training:Created run path: models/courtship.topdown_confmaps\n", - "INFO:sleap.nn.training:Setting up visualization...\n", - "INFO:sleap.nn.training:Finished trainer set up. [3.2s]\n", - "INFO:sleap.nn.training:Creating tf.data.Datasets for training data generation...\n", - "INFO:sleap.nn.training:Finished creating training datasets. [5.9s]\n", - "INFO:sleap.nn.training:Starting training loop...\n", - "Epoch 1/200\n", - "2023-09-01 13:39:54.940083: I tensorflow/stream_executor/cuda/cuda_dnn.cc:366] Loaded cuDNN version 8201\n", - "2023-09-01 13:40:00.337645: I tensorflow/stream_executor/cuda/cuda_blas.cc:1774] TensorFloat-32 will be used for the matrix multiplication. This will only be logged once.\n", - "200/200 - 8s - loss: 0.0108 - head: 0.0073 - thorax: 0.0067 - abdomen: 0.0111 - wingL: 0.0125 - wingR: 0.0126 - forelegL4: 0.0111 - forelegR4: 0.0108 - midlegL4: 0.0127 - midlegR4: 0.0128 - hindlegL4: 0.0131 - hindlegR4: 0.0131 - eyeL: 0.0082 - eyeR: 0.0083 - val_loss: 0.0087 - val_head: 0.0033 - val_thorax: 0.0039 - val_abdomen: 0.0089 - val_wingL: 0.0105 - val_wingR: 0.0106 - val_forelegL4: 0.0091 - val_forelegR4: 0.0091 - val_midlegL4: 0.0123 - val_midlegR4: 0.0116 - val_hindlegL4: 0.0128 - val_hindlegR4: 0.0116 - val_eyeL: 0.0045 - val_eyeR: 0.0045 - lr: 1.0000e-04 - 8s/epoch - 38ms/step\n", - "Epoch 2/200\n", - "200/200 - 4s - loss: 0.0064 - head: 0.0019 - thorax: 0.0029 - abdomen: 0.0057 - wingL: 0.0061 - wingR: 0.0073 - forelegL4: 0.0075 - forelegR4: 0.0078 - midlegL4: 0.0092 - midlegR4: 0.0092 - hindlegL4: 0.0099 - hindlegR4: 0.0102 - eyeL: 0.0025 - eyeR: 0.0025 - val_loss: 0.0061 - val_head: 0.0015 - val_thorax: 0.0024 - val_abdomen: 0.0049 - val_wingL: 0.0056 - val_wingR: 0.0078 - val_forelegL4: 0.0079 - val_forelegR4: 0.0067 - val_midlegL4: 0.0086 - val_midlegR4: 0.0089 - val_hindlegL4: 0.0093 - val_hindlegR4: 0.0081 - val_eyeL: 0.0037 - val_eyeR: 0.0032 - lr: 1.0000e-04 - 4s/epoch - 19ms/step\n", - "Epoch 3/200\n", - "200/200 - 3s - loss: 0.0048 - head: 8.9048e-04 - thorax: 0.0019 - abdomen: 0.0036 - wingL: 0.0041 - wingR: 0.0051 - forelegL4: 0.0063 - forelegR4: 0.0066 - midlegL4: 0.0076 - midlegR4: 0.0076 - hindlegL4: 0.0076 - hindlegR4: 0.0080 - eyeL: 0.0015 - eyeR: 0.0015 - val_loss: 0.0058 - val_head: 0.0014 - val_thorax: 0.0021 - val_abdomen: 0.0044 - val_wingL: 0.0051 - val_wingR: 0.0070 - val_forelegL4: 0.0072 - val_forelegR4: 0.0063 - val_midlegL4: 0.0088 - val_midlegR4: 0.0085 - val_hindlegL4: 0.0097 - val_hindlegR4: 0.0079 - val_eyeL: 0.0038 - val_eyeR: 0.0032 - lr: 1.0000e-04 - 3s/epoch - 16ms/step\n", - "Epoch 4/200\n", - "200/200 - 3s - loss: 0.0041 - head: 7.6417e-04 - thorax: 0.0015 - abdomen: 0.0028 - wingL: 0.0035 - wingR: 0.0041 - forelegL4: 0.0058 - forelegR4: 0.0060 - midlegL4: 0.0066 - midlegR4: 0.0064 - hindlegL4: 0.0066 - hindlegR4: 0.0070 - eyeL: 0.0013 - eyeR: 0.0012 - val_loss: 0.0048 - val_head: 7.6555e-04 - val_thorax: 0.0013 - val_abdomen: 0.0034 - val_wingL: 0.0042 - val_wingR: 0.0065 - val_forelegL4: 0.0063 - val_forelegR4: 0.0064 - val_midlegL4: 0.0069 - val_midlegR4: 0.0071 - val_hindlegL4: 0.0080 - val_hindlegR4: 0.0062 - val_eyeL: 0.0028 - val_eyeR: 0.0026 - lr: 1.0000e-04 - 3s/epoch - 15ms/step\n", - "Epoch 5/200\n", - "200/200 - 3s - loss: 0.0034 - head: 6.1233e-04 - thorax: 0.0012 - abdomen: 0.0023 - wingL: 0.0028 - wingR: 0.0032 - forelegL4: 0.0052 - forelegR4: 0.0054 - midlegL4: 0.0052 - midlegR4: 0.0051 - hindlegL4: 0.0057 - hindlegR4: 0.0058 - eyeL: 0.0011 - eyeR: 0.0011 - val_loss: 0.0044 - val_head: 9.3809e-04 - val_thorax: 0.0012 - val_abdomen: 0.0027 - val_wingL: 0.0032 - val_wingR: 0.0048 - val_forelegL4: 0.0062 - val_forelegR4: 0.0053 - val_midlegL4: 0.0068 - val_midlegR4: 0.0063 - val_hindlegL4: 0.0067 - val_hindlegR4: 0.0065 - val_eyeL: 0.0035 - val_eyeR: 0.0032 - lr: 1.0000e-04 - 3s/epoch - 15ms/step\n", - "Epoch 6/200\n", - "200/200 - 3s - loss: 0.0028 - head: 5.5957e-04 - thorax: 9.3519e-04 - abdomen: 0.0019 - wingL: 0.0023 - wingR: 0.0025 - forelegL4: 0.0045 - forelegR4: 0.0045 - midlegL4: 0.0040 - midlegR4: 0.0040 - hindlegL4: 0.0047 - hindlegR4: 0.0048 - eyeL: 0.0010 - eyeR: 9.7287e-04 - val_loss: 0.0038 - val_head: 7.6837e-04 - val_thorax: 9.9723e-04 - val_abdomen: 0.0027 - val_wingL: 0.0025 - val_wingR: 0.0046 - val_forelegL4: 0.0058 - val_forelegR4: 0.0049 - val_midlegL4: 0.0054 - val_midlegR4: 0.0058 - val_hindlegL4: 0.0057 - val_hindlegR4: 0.0065 - val_eyeL: 0.0023 - val_eyeR: 0.0022 - lr: 1.0000e-04 - 3s/epoch - 15ms/step\n", - "Epoch 7/200\n", - "200/200 - 3s - loss: 0.0024 - head: 4.7941e-04 - thorax: 7.5772e-04 - abdomen: 0.0017 - wingL: 0.0020 - wingR: 0.0022 - forelegL4: 0.0039 - forelegR4: 0.0041 - midlegL4: 0.0033 - midlegR4: 0.0033 - hindlegL4: 0.0039 - hindlegR4: 0.0040 - eyeL: 9.3055e-04 - eyeR: 8.9191e-04 - val_loss: 0.0036 - val_head: 6.1078e-04 - val_thorax: 0.0010 - val_abdomen: 0.0023 - val_wingL: 0.0025 - val_wingR: 0.0039 - val_forelegL4: 0.0053 - val_forelegR4: 0.0058 - val_midlegL4: 0.0049 - val_midlegR4: 0.0056 - val_hindlegL4: 0.0054 - val_hindlegR4: 0.0049 - val_eyeL: 0.0026 - val_eyeR: 0.0024 - lr: 1.0000e-04 - 3s/epoch - 15ms/step\n", - "Epoch 8/200\n", - "200/200 - 3s - loss: 0.0020 - head: 4.4425e-04 - thorax: 6.8283e-04 - abdomen: 0.0014 - wingL: 0.0015 - wingR: 0.0017 - forelegL4: 0.0035 - forelegR4: 0.0035 - midlegL4: 0.0027 - midlegR4: 0.0026 - hindlegL4: 0.0033 - hindlegR4: 0.0033 - eyeL: 7.7111e-04 - eyeR: 7.2022e-04 - val_loss: 0.0035 - val_head: 7.1555e-04 - val_thorax: 9.1508e-04 - val_abdomen: 0.0022 - val_wingL: 0.0023 - val_wingR: 0.0033 - val_forelegL4: 0.0054 - val_forelegR4: 0.0049 - val_midlegL4: 0.0049 - val_midlegR4: 0.0052 - val_hindlegL4: 0.0052 - val_hindlegR4: 0.0051 - val_eyeL: 0.0025 - val_eyeR: 0.0025 - lr: 1.0000e-04 - 3s/epoch - 15ms/step\n", - "Epoch 9/200\n", - "200/200 - 3s - loss: 0.0017 - head: 3.8990e-04 - thorax: 5.4963e-04 - abdomen: 0.0012 - wingL: 0.0012 - wingR: 0.0014 - forelegL4: 0.0030 - forelegR4: 0.0031 - midlegL4: 0.0022 - midlegR4: 0.0022 - hindlegL4: 0.0027 - hindlegR4: 0.0027 - eyeL: 6.9041e-04 - eyeR: 6.7679e-04 - val_loss: 0.0034 - val_head: 5.6666e-04 - val_thorax: 7.9156e-04 - val_abdomen: 0.0023 - val_wingL: 0.0020 - val_wingR: 0.0041 - val_forelegL4: 0.0043 - val_forelegR4: 0.0048 - val_midlegL4: 0.0041 - val_midlegR4: 0.0051 - val_hindlegL4: 0.0053 - val_hindlegR4: 0.0052 - val_eyeL: 0.0024 - val_eyeR: 0.0026 - lr: 1.0000e-04 - 3s/epoch - 15ms/step\n", - "Epoch 10/200\n", - "200/200 - 3s - loss: 0.0015 - head: 3.6281e-04 - thorax: 5.2471e-04 - abdomen: 0.0010 - wingL: 0.0011 - wingR: 0.0012 - forelegL4: 0.0027 - forelegR4: 0.0028 - midlegL4: 0.0019 - midlegR4: 0.0019 - hindlegL4: 0.0023 - hindlegR4: 0.0024 - eyeL: 7.0986e-04 - eyeR: 6.9581e-04 - val_loss: 0.0024 - val_head: 4.8376e-04 - val_thorax: 6.2502e-04 - val_abdomen: 0.0016 - val_wingL: 0.0014 - val_wingR: 0.0027 - val_forelegL4: 0.0035 - val_forelegR4: 0.0033 - val_midlegL4: 0.0028 - val_midlegR4: 0.0041 - val_hindlegL4: 0.0036 - val_hindlegR4: 0.0038 - val_eyeL: 0.0015 - val_eyeR: 0.0016 - lr: 1.0000e-04 - 3s/epoch - 16ms/step\n", - "Epoch 11/200\n", - "200/200 - 3s - loss: 0.0013 - head: 3.1183e-04 - thorax: 4.7891e-04 - abdomen: 9.4567e-04 - wingL: 9.6811e-04 - wingR: 0.0011 - forelegL4: 0.0023 - forelegR4: 0.0025 - midlegL4: 0.0016 - midlegR4: 0.0016 - hindlegL4: 0.0020 - hindlegR4: 0.0021 - eyeL: 5.7635e-04 - eyeR: 5.3648e-04 - val_loss: 0.0028 - val_head: 5.2940e-04 - val_thorax: 6.6554e-04 - val_abdomen: 0.0020 - val_wingL: 0.0013 - val_wingR: 0.0024 - val_forelegL4: 0.0041 - val_forelegR4: 0.0041 - val_midlegL4: 0.0034 - val_midlegR4: 0.0042 - val_hindlegL4: 0.0047 - val_hindlegR4: 0.0040 - val_eyeL: 0.0025 - val_eyeR: 0.0022 - lr: 1.0000e-04 - 3s/epoch - 15ms/step\n", - "Epoch 12/200\n", - "200/200 - 3s - loss: 0.0011 - head: 2.8863e-04 - thorax: 4.2604e-04 - abdomen: 8.0488e-04 - wingL: 8.1238e-04 - wingR: 8.5798e-04 - forelegL4: 0.0021 - forelegR4: 0.0021 - midlegL4: 0.0014 - midlegR4: 0.0014 - hindlegL4: 0.0017 - hindlegR4: 0.0018 - eyeL: 5.1007e-04 - eyeR: 4.5654e-04 - val_loss: 0.0031 - val_head: 8.1802e-04 - val_thorax: 7.9789e-04 - val_abdomen: 0.0018 - val_wingL: 0.0014 - val_wingR: 0.0028 - val_forelegL4: 0.0040 - val_forelegR4: 0.0048 - val_midlegL4: 0.0057 - val_midlegR4: 0.0037 - val_hindlegL4: 0.0053 - val_hindlegR4: 0.0050 - val_eyeL: 0.0020 - val_eyeR: 0.0018 - lr: 1.0000e-04 - 3s/epoch - 14ms/step\n", - "Epoch 13/200\n", - "200/200 - 3s - loss: 0.0010 - head: 2.8818e-04 - thorax: 4.1018e-04 - abdomen: 7.8027e-04 - wingL: 7.8017e-04 - wingR: 8.4529e-04 - forelegL4: 0.0019 - forelegR4: 0.0019 - midlegL4: 0.0013 - midlegR4: 0.0013 - hindlegL4: 0.0015 - hindlegR4: 0.0016 - eyeL: 4.6272e-04 - eyeR: 4.3265e-04 - val_loss: 0.0026 - val_head: 3.5806e-04 - val_thorax: 6.6352e-04 - val_abdomen: 0.0017 - val_wingL: 0.0015 - val_wingR: 0.0037 - val_forelegL4: 0.0036 - val_forelegR4: 0.0042 - val_midlegL4: 0.0034 - val_midlegR4: 0.0032 - val_hindlegL4: 0.0041 - val_hindlegR4: 0.0047 - val_eyeL: 0.0013 - val_eyeR: 0.0013 - lr: 1.0000e-04 - 3s/epoch - 15ms/step\n", - "Epoch 14/200\n", - "200/200 - 3s - loss: 9.4029e-04 - head: 2.8339e-04 - thorax: 3.6739e-04 - abdomen: 7.0118e-04 - wingL: 7.4831e-04 - wingR: 7.1158e-04 - forelegL4: 0.0017 - forelegR4: 0.0017 - midlegL4: 0.0012 - midlegR4: 0.0011 - hindlegL4: 0.0014 - hindlegR4: 0.0015 - eyeL: 4.2793e-04 - eyeR: 4.1400e-04 - val_loss: 0.0024 - val_head: 3.4292e-04 - val_thorax: 7.1119e-04 - val_abdomen: 0.0014 - val_wingL: 0.0013 - val_wingR: 0.0028 - val_forelegL4: 0.0030 - val_forelegR4: 0.0043 - val_midlegL4: 0.0031 - val_midlegR4: 0.0030 - val_hindlegL4: 0.0039 - val_hindlegR4: 0.0038 - val_eyeL: 0.0017 - val_eyeR: 0.0015 - lr: 1.0000e-04 - 3s/epoch - 15ms/step\n", - "Epoch 15/200\n", - "200/200 - 3s - loss: 7.8295e-04 - head: 2.3028e-04 - thorax: 3.3006e-04 - abdomen: 5.9391e-04 - wingL: 5.8825e-04 - wingR: 6.0989e-04 - forelegL4: 0.0015 - forelegR4: 0.0015 - midlegL4: 9.6945e-04 - midlegR4: 9.3611e-04 - hindlegL4: 0.0011 - hindlegR4: 0.0012 - eyeL: 3.4493e-04 - eyeR: 3.1164e-04 - val_loss: 0.0019 - val_head: 4.4152e-04 - val_thorax: 5.4500e-04 - val_abdomen: 0.0013 - val_wingL: 0.0012 - val_wingR: 0.0026 - val_forelegL4: 0.0024 - val_forelegR4: 0.0037 - val_midlegL4: 0.0024 - val_midlegR4: 0.0024 - val_hindlegL4: 0.0030 - val_hindlegR4: 0.0030 - val_eyeL: 0.0011 - val_eyeR: 0.0011 - lr: 1.0000e-04 - 3s/epoch - 15ms/step\n", - "Epoch 16/200\n", - "200/200 - 3s - loss: 7.3208e-04 - head: 2.3573e-04 - thorax: 3.0631e-04 - abdomen: 5.5007e-04 - wingL: 5.3431e-04 - wingR: 5.9773e-04 - forelegL4: 0.0013 - forelegR4: 0.0014 - midlegL4: 9.1004e-04 - midlegR4: 8.7803e-04 - hindlegL4: 0.0010 - hindlegR4: 0.0011 - eyeL: 3.3279e-04 - eyeR: 2.9841e-04 - val_loss: 0.0023 - val_head: 3.5381e-04 - val_thorax: 7.0128e-04 - val_abdomen: 0.0015 - val_wingL: 0.0013 - val_wingR: 0.0022 - val_forelegL4: 0.0031 - val_forelegR4: 0.0041 - val_midlegL4: 0.0033 - val_midlegR4: 0.0028 - val_hindlegL4: 0.0036 - val_hindlegR4: 0.0033 - val_eyeL: 0.0017 - val_eyeR: 0.0014 - lr: 1.0000e-04 - 3s/epoch - 14ms/step\n", - "Epoch 17/200\n", - "200/200 - 3s - loss: 6.3161e-04 - head: 2.0100e-04 - thorax: 2.8088e-04 - abdomen: 4.9153e-04 - wingL: 4.7586e-04 - wingR: 4.9866e-04 - forelegL4: 0.0011 - forelegR4: 0.0012 - midlegL4: 7.6100e-04 - midlegR4: 8.0266e-04 - hindlegL4: 8.9697e-04 - hindlegR4: 8.9149e-04 - eyeL: 2.8189e-04 - eyeR: 2.7208e-04 - val_loss: 0.0018 - val_head: 2.8070e-04 - val_thorax: 5.1903e-04 - val_abdomen: 0.0011 - val_wingL: 9.8509e-04 - val_wingR: 0.0025 - val_forelegL4: 0.0022 - val_forelegR4: 0.0026 - val_midlegL4: 0.0025 - val_midlegR4: 0.0021 - val_hindlegL4: 0.0031 - val_hindlegR4: 0.0031 - val_eyeL: 0.0011 - val_eyeR: 9.7838e-04 - lr: 1.0000e-04 - 3s/epoch - 15ms/step\n", - "Epoch 18/200\n", - "200/200 - 3s - loss: 5.7844e-04 - head: 1.9896e-04 - thorax: 2.9112e-04 - abdomen: 4.7495e-04 - wingL: 4.5591e-04 - wingR: 4.5877e-04 - forelegL4: 0.0011 - forelegR4: 0.0012 - midlegL4: 6.9042e-04 - midlegR4: 6.6195e-04 - hindlegL4: 7.9452e-04 - hindlegR4: 7.6819e-04 - eyeL: 2.5989e-04 - eyeR: 2.4763e-04 - val_loss: 0.0018 - val_head: 3.1925e-04 - val_thorax: 6.0394e-04 - val_abdomen: 0.0012 - val_wingL: 9.0835e-04 - val_wingR: 0.0019 - val_forelegL4: 0.0022 - val_forelegR4: 0.0029 - val_midlegL4: 0.0026 - val_midlegR4: 0.0024 - val_hindlegL4: 0.0033 - val_hindlegR4: 0.0022 - val_eyeL: 0.0015 - val_eyeR: 0.0011 - lr: 1.0000e-04 - 3s/epoch - 15ms/step\n", - "Epoch 19/200\n", - "200/200 - 3s - loss: 5.1323e-04 - head: 1.8346e-04 - thorax: 2.5475e-04 - abdomen: 4.2159e-04 - wingL: 4.3027e-04 - wingR: 3.9814e-04 - forelegL4: 9.5814e-04 - forelegR4: 9.9765e-04 - midlegL4: 5.9968e-04 - midlegR4: 5.8423e-04 - hindlegL4: 6.7869e-04 - hindlegR4: 6.9121e-04 - eyeL: 2.4343e-04 - eyeR: 2.3077e-04 - val_loss: 0.0021 - val_head: 3.3346e-04 - val_thorax: 5.9007e-04 - val_abdomen: 0.0014 - val_wingL: 0.0013 - val_wingR: 0.0031 - val_forelegL4: 0.0026 - val_forelegR4: 0.0036 - val_midlegL4: 0.0029 - val_midlegR4: 0.0021 - val_hindlegL4: 0.0037 - val_hindlegR4: 0.0036 - val_eyeL: 0.0011 - val_eyeR: 9.4254e-04 - lr: 1.0000e-04 - 3s/epoch - 14ms/step\n", - "Epoch 20/200\n", - "200/200 - 3s - loss: 4.7991e-04 - head: 1.7328e-04 - thorax: 2.2397e-04 - abdomen: 4.2417e-04 - wingL: 3.9313e-04 - wingR: 3.9871e-04 - forelegL4: 8.8547e-04 - forelegR4: 8.9704e-04 - midlegL4: 5.3515e-04 - midlegR4: 5.8294e-04 - hindlegL4: 6.5212e-04 - hindlegR4: 6.2828e-04 - eyeL: 2.2438e-04 - eyeR: 2.2012e-04 - val_loss: 0.0014 - val_head: 2.7034e-04 - val_thorax: 4.7978e-04 - val_abdomen: 9.7903e-04 - val_wingL: 8.6477e-04 - val_wingR: 0.0020 - val_forelegL4: 0.0018 - val_forelegR4: 0.0024 - val_midlegL4: 0.0019 - val_midlegR4: 0.0018 - val_hindlegL4: 0.0024 - val_hindlegR4: 0.0022 - val_eyeL: 9.9423e-04 - val_eyeR: 8.4541e-04 - lr: 1.0000e-04 - 3s/epoch - 15ms/step\n", - "Epoch 21/200\n", - "200/200 - 3s - loss: 4.4100e-04 - head: 1.6076e-04 - thorax: 2.4080e-04 - abdomen: 3.8343e-04 - wingL: 3.6759e-04 - wingR: 3.7489e-04 - forelegL4: 8.1060e-04 - forelegR4: 8.1600e-04 - midlegL4: 4.7288e-04 - midlegR4: 5.2695e-04 - hindlegL4: 5.6401e-04 - hindlegR4: 6.3519e-04 - eyeL: 1.9033e-04 - eyeR: 1.8954e-04 - val_loss: 0.0018 - val_head: 2.5764e-04 - val_thorax: 5.8718e-04 - val_abdomen: 0.0011 - val_wingL: 9.6939e-04 - val_wingR: 0.0019 - val_forelegL4: 0.0022 - val_forelegR4: 0.0026 - val_midlegL4: 0.0025 - val_midlegR4: 0.0026 - val_hindlegL4: 0.0032 - val_hindlegR4: 0.0028 - val_eyeL: 0.0014 - val_eyeR: 0.0011 - lr: 1.0000e-04 - 3s/epoch - 15ms/step\n", - "Epoch 22/200\n", - "200/200 - 3s - loss: 3.7738e-04 - head: 1.4725e-04 - thorax: 2.0905e-04 - abdomen: 3.2447e-04 - wingL: 3.2224e-04 - wingR: 3.0585e-04 - forelegL4: 6.2169e-04 - forelegR4: 6.7379e-04 - midlegL4: 4.5061e-04 - midlegR4: 4.3931e-04 - hindlegL4: 5.1129e-04 - hindlegR4: 5.2449e-04 - eyeL: 1.9372e-04 - eyeR: 1.8213e-04 - val_loss: 0.0015 - val_head: 2.2947e-04 - val_thorax: 5.4640e-04 - val_abdomen: 9.8293e-04 - val_wingL: 8.6663e-04 - val_wingR: 0.0013 - val_forelegL4: 0.0018 - val_forelegR4: 0.0027 - val_midlegL4: 0.0021 - val_midlegR4: 0.0019 - val_hindlegL4: 0.0027 - val_hindlegR4: 0.0022 - val_eyeL: 0.0013 - val_eyeR: 0.0010 - lr: 1.0000e-04 - 3s/epoch - 15ms/step\n", - "Epoch 23/200\n", - "200/200 - 3s - loss: 3.6084e-04 - head: 1.4440e-04 - thorax: 2.0277e-04 - abdomen: 3.0561e-04 - wingL: 3.0192e-04 - wingR: 2.8845e-04 - forelegL4: 6.3221e-04 - forelegR4: 6.7722e-04 - midlegL4: 3.9143e-04 - midlegR4: 4.3545e-04 - hindlegL4: 5.1985e-04 - hindlegR4: 4.5058e-04 - eyeL: 1.7636e-04 - eyeR: 1.6468e-04 - val_loss: 0.0015 - val_head: 2.9639e-04 - val_thorax: 4.6412e-04 - val_abdomen: 0.0011 - val_wingL: 9.0466e-04 - val_wingR: 0.0021 - val_forelegL4: 0.0015 - val_forelegR4: 0.0025 - val_midlegL4: 0.0018 - val_midlegR4: 0.0016 - val_hindlegL4: 0.0029 - val_hindlegR4: 0.0022 - val_eyeL: 8.7357e-04 - val_eyeR: 7.0067e-04 - lr: 1.0000e-04 - 3s/epoch - 15ms/step\n", - "Epoch 24/200\n", - "200/200 - 3s - loss: 3.4886e-04 - head: 1.4382e-04 - thorax: 1.9157e-04 - abdomen: 3.2551e-04 - wingL: 3.0634e-04 - wingR: 3.0727e-04 - forelegL4: 6.3863e-04 - forelegR4: 6.0904e-04 - midlegL4: 3.5949e-04 - midlegR4: 4.1201e-04 - hindlegL4: 4.2893e-04 - hindlegR4: 4.8121e-04 - eyeL: 1.6669e-04 - eyeR: 1.6464e-04 - val_loss: 0.0022 - val_head: 3.2159e-04 - val_thorax: 7.2743e-04 - val_abdomen: 0.0014 - val_wingL: 0.0011 - val_wingR: 0.0027 - val_forelegL4: 0.0025 - val_forelegR4: 0.0037 - val_midlegL4: 0.0033 - val_midlegR4: 0.0020 - val_hindlegL4: 0.0043 - val_hindlegR4: 0.0031 - val_eyeL: 0.0017 - val_eyeR: 0.0012 - lr: 1.0000e-04 - 3s/epoch - 14ms/step\n", - "Epoch 25/200\n", - "\n", - "Epoch 00025: ReduceLROnPlateau reducing learning rate to 4.999999873689376e-05.\n", - "200/200 - 3s - loss: 3.0444e-04 - head: 1.2563e-04 - thorax: 1.7247e-04 - abdomen: 2.6934e-04 - wingL: 2.5754e-04 - wingR: 2.4728e-04 - forelegL4: 5.8390e-04 - forelegR4: 5.3959e-04 - midlegL4: 3.3003e-04 - midlegR4: 3.6432e-04 - hindlegL4: 4.0270e-04 - hindlegR4: 3.5518e-04 - eyeL: 1.5609e-04 - eyeR: 1.5365e-04 - val_loss: 0.0017 - val_head: 2.5420e-04 - val_thorax: 5.5809e-04 - val_abdomen: 0.0011 - val_wingL: 9.6708e-04 - val_wingR: 0.0022 - val_forelegL4: 0.0018 - val_forelegR4: 0.0033 - val_midlegL4: 0.0025 - val_midlegR4: 0.0017 - val_hindlegL4: 0.0031 - val_hindlegR4: 0.0031 - val_eyeL: 9.8718e-04 - val_eyeR: 8.0263e-04 - lr: 1.0000e-04 - 3s/epoch - 15ms/step\n", - "Epoch 26/200\n", - "200/200 - 3s - loss: 2.3368e-04 - head: 1.1149e-04 - thorax: 1.5177e-04 - abdomen: 2.1763e-04 - wingL: 2.2159e-04 - wingR: 1.9396e-04 - forelegL4: 3.8234e-04 - forelegR4: 3.8248e-04 - midlegL4: 2.7555e-04 - midlegR4: 2.8653e-04 - hindlegL4: 2.7842e-04 - hindlegR4: 2.8074e-04 - eyeL: 1.3157e-04 - eyeR: 1.2374e-04 - val_loss: 0.0017 - val_head: 2.1815e-04 - val_thorax: 5.0063e-04 - val_abdomen: 0.0011 - val_wingL: 8.2248e-04 - val_wingR: 0.0020 - val_forelegL4: 0.0019 - val_forelegR4: 0.0035 - val_midlegL4: 0.0022 - val_midlegR4: 0.0016 - val_hindlegL4: 0.0031 - val_hindlegR4: 0.0022 - val_eyeL: 0.0013 - val_eyeR: 9.8071e-04 - lr: 5.0000e-05 - 3s/epoch - 14ms/step\n", - "Epoch 27/200\n", - "200/200 - 3s - loss: 2.0711e-04 - head: 9.7513e-05 - thorax: 1.4018e-04 - abdomen: 2.0210e-04 - wingL: 1.8693e-04 - wingR: 1.7399e-04 - forelegL4: 3.1753e-04 - forelegR4: 3.7613e-04 - midlegL4: 2.2838e-04 - midlegR4: 2.4643e-04 - hindlegL4: 2.4471e-04 - hindlegR4: 2.4706e-04 - eyeL: 1.1696e-04 - eyeR: 1.1452e-04 - val_loss: 0.0011 - val_head: 1.7855e-04 - val_thorax: 3.7885e-04 - val_abdomen: 7.0074e-04 - val_wingL: 6.4821e-04 - val_wingR: 0.0012 - val_forelegL4: 0.0012 - val_forelegR4: 0.0017 - val_midlegL4: 0.0014 - val_midlegR4: 0.0013 - val_hindlegL4: 0.0019 - val_hindlegR4: 0.0018 - val_eyeL: 8.8941e-04 - val_eyeR: 7.0606e-04 - lr: 5.0000e-05 - 3s/epoch - 15ms/step\n", - "Epoch 28/200\n", - "200/200 - 3s - loss: 1.9539e-04 - head: 9.4716e-05 - thorax: 1.3617e-04 - abdomen: 1.8547e-04 - wingL: 1.8173e-04 - wingR: 1.6716e-04 - forelegL4: 3.2783e-04 - forelegR4: 3.1060e-04 - midlegL4: 2.2172e-04 - midlegR4: 2.2648e-04 - hindlegL4: 2.3846e-04 - hindlegR4: 2.2823e-04 - eyeL: 1.1204e-04 - eyeR: 1.0944e-04 - val_loss: 0.0012 - val_head: 1.9505e-04 - val_thorax: 3.8105e-04 - val_abdomen: 7.7888e-04 - val_wingL: 6.8985e-04 - val_wingR: 0.0016 - val_forelegL4: 0.0015 - val_forelegR4: 0.0020 - val_midlegL4: 0.0017 - val_midlegR4: 0.0011 - val_hindlegL4: 0.0022 - val_hindlegR4: 0.0019 - val_eyeL: 9.1223e-04 - val_eyeR: 7.0778e-04 - lr: 5.0000e-05 - 3s/epoch - 15ms/step\n", - "Epoch 29/200\n", - "200/200 - 3s - loss: 1.8262e-04 - head: 9.2364e-05 - thorax: 1.3126e-04 - abdomen: 1.7625e-04 - wingL: 1.7494e-04 - wingR: 1.5998e-04 - forelegL4: 3.0159e-04 - forelegR4: 2.9470e-04 - midlegL4: 1.9773e-04 - midlegR4: 2.0446e-04 - hindlegL4: 2.0576e-04 - hindlegR4: 2.1560e-04 - eyeL: 1.1218e-04 - eyeR: 1.0720e-04 - val_loss: 0.0015 - val_head: 2.2535e-04 - val_thorax: 4.8031e-04 - val_abdomen: 9.5428e-04 - val_wingL: 7.7468e-04 - val_wingR: 0.0016 - val_forelegL4: 0.0017 - val_forelegR4: 0.0025 - val_midlegL4: 0.0021 - val_midlegR4: 0.0018 - val_hindlegL4: 0.0029 - val_hindlegR4: 0.0019 - val_eyeL: 0.0013 - val_eyeR: 9.6936e-04 - lr: 5.0000e-05 - 3s/epoch - 15ms/step\n", - "Epoch 30/200\n", - "200/200 - 3s - loss: 1.7461e-04 - head: 8.9617e-05 - thorax: 1.2428e-04 - abdomen: 1.7234e-04 - wingL: 1.6780e-04 - wingR: 1.5580e-04 - forelegL4: 2.7324e-04 - forelegR4: 2.8042e-04 - midlegL4: 1.9090e-04 - midlegR4: 2.0420e-04 - hindlegL4: 1.9914e-04 - hindlegR4: 2.0318e-04 - eyeL: 1.0518e-04 - eyeR: 1.0386e-04 - val_loss: 0.0015 - val_head: 1.9058e-04 - val_thorax: 4.9603e-04 - val_abdomen: 0.0011 - val_wingL: 9.7566e-04 - val_wingR: 0.0018 - val_forelegL4: 0.0016 - val_forelegR4: 0.0028 - val_midlegL4: 0.0022 - val_midlegR4: 0.0015 - val_hindlegL4: 0.0028 - val_hindlegR4: 0.0028 - val_eyeL: 9.9699e-04 - val_eyeR: 8.3721e-04 - lr: 5.0000e-05 - 3s/epoch - 15ms/step\n", - "Epoch 31/200\n", - "200/200 - 3s - loss: 1.7064e-04 - head: 8.7373e-05 - thorax: 1.2365e-04 - abdomen: 1.6765e-04 - wingL: 1.5656e-04 - wingR: 1.4505e-04 - forelegL4: 2.7352e-04 - forelegR4: 2.6274e-04 - midlegL4: 1.9639e-04 - midlegR4: 1.9628e-04 - hindlegL4: 2.0323e-04 - hindlegR4: 1.9917e-04 - eyeL: 1.0639e-04 - eyeR: 1.0032e-04 - val_loss: 0.0011 - val_head: 1.7938e-04 - val_thorax: 3.6727e-04 - val_abdomen: 7.7820e-04 - val_wingL: 6.4437e-04 - val_wingR: 0.0014 - val_forelegL4: 0.0014 - val_forelegR4: 0.0020 - val_midlegL4: 0.0016 - val_midlegR4: 0.0010 - val_hindlegL4: 0.0021 - val_hindlegR4: 0.0016 - val_eyeL: 8.0607e-04 - val_eyeR: 6.6172e-04 - lr: 5.0000e-05 - 3s/epoch - 16ms/step\n", - "Epoch 32/200\n", - "\n", - "Epoch 00032: ReduceLROnPlateau reducing learning rate to 2.499999936844688e-05.\n", - "200/200 - 4s - loss: 1.6547e-04 - head: 8.6407e-05 - thorax: 1.1578e-04 - abdomen: 1.6160e-04 - wingL: 1.5752e-04 - wingR: 1.4326e-04 - forelegL4: 2.5855e-04 - forelegR4: 2.8317e-04 - midlegL4: 1.7880e-04 - midlegR4: 1.8021e-04 - hindlegL4: 1.9743e-04 - hindlegR4: 1.8831e-04 - eyeL: 1.0074e-04 - eyeR: 9.9381e-05 - val_loss: 0.0012 - val_head: 1.9257e-04 - val_thorax: 3.7361e-04 - val_abdomen: 7.0451e-04 - val_wingL: 7.8240e-04 - val_wingR: 0.0015 - val_forelegL4: 0.0014 - val_forelegR4: 0.0020 - val_midlegL4: 0.0016 - val_midlegR4: 0.0011 - val_hindlegL4: 0.0020 - val_hindlegR4: 0.0019 - val_eyeL: 8.9328e-04 - val_eyeR: 7.3886e-04 - lr: 5.0000e-05 - 4s/epoch - 18ms/step\n", - "Epoch 33/200\n", - "200/200 - 3s - loss: 1.4767e-04 - head: 8.0575e-05 - thorax: 1.1097e-04 - abdomen: 1.4927e-04 - wingL: 1.4112e-04 - wingR: 1.3113e-04 - forelegL4: 2.1913e-04 - forelegR4: 2.1998e-04 - midlegL4: 1.6045e-04 - midlegR4: 1.6535e-04 - hindlegL4: 1.8091e-04 - hindlegR4: 1.7343e-04 - eyeL: 9.5387e-05 - eyeR: 9.2035e-05 - val_loss: 0.0014 - val_head: 1.9046e-04 - val_thorax: 4.6921e-04 - val_abdomen: 9.4087e-04 - val_wingL: 7.5647e-04 - val_wingR: 0.0015 - val_forelegL4: 0.0015 - val_forelegR4: 0.0025 - val_midlegL4: 0.0020 - val_midlegR4: 0.0015 - val_hindlegL4: 0.0026 - val_hindlegR4: 0.0021 - val_eyeL: 0.0013 - val_eyeR: 0.0010 - lr: 2.5000e-05 - 3s/epoch - 16ms/step\n", - "Epoch 34/200\n", - "200/200 - 3s - loss: 1.4506e-04 - head: 7.9790e-05 - thorax: 1.0771e-04 - abdomen: 1.5052e-04 - wingL: 1.4143e-04 - wingR: 1.2485e-04 - forelegL4: 2.2486e-04 - forelegR4: 2.1619e-04 - midlegL4: 1.6584e-04 - midlegR4: 1.6250e-04 - hindlegL4: 1.6521e-04 - hindlegR4: 1.6717e-04 - eyeL: 9.1550e-05 - eyeR: 8.8112e-05 - val_loss: 0.0013 - val_head: 1.8689e-04 - val_thorax: 3.7203e-04 - val_abdomen: 9.3770e-04 - val_wingL: 7.0190e-04 - val_wingR: 0.0019 - val_forelegL4: 0.0015 - val_forelegR4: 0.0023 - val_midlegL4: 0.0016 - val_midlegR4: 0.0012 - val_hindlegL4: 0.0025 - val_hindlegR4: 0.0022 - val_eyeL: 8.0213e-04 - val_eyeR: 6.5036e-04 - lr: 2.5000e-05 - 3s/epoch - 15ms/step\n", - "Epoch 35/200\n", - "200/200 - 3s - loss: 1.3911e-04 - head: 7.9674e-05 - thorax: 1.0668e-04 - abdomen: 1.4330e-04 - wingL: 1.3906e-04 - wingR: 1.2752e-04 - forelegL4: 1.9657e-04 - forelegR4: 1.9577e-04 - midlegL4: 1.5228e-04 - midlegR4: 1.5642e-04 - hindlegL4: 1.6610e-04 - hindlegR4: 1.6394e-04 - eyeL: 9.1523e-05 - eyeR: 8.9620e-05 - val_loss: 0.0013 - val_head: 1.7511e-04 - val_thorax: 4.2162e-04 - val_abdomen: 9.5009e-04 - val_wingL: 6.7908e-04 - val_wingR: 0.0013 - val_forelegL4: 0.0015 - val_forelegR4: 0.0023 - val_midlegL4: 0.0018 - val_midlegR4: 0.0014 - val_hindlegL4: 0.0027 - val_hindlegR4: 0.0019 - val_eyeL: 0.0012 - val_eyeR: 9.8818e-04 - lr: 2.5000e-05 - 3s/epoch - 16ms/step\n", - "Epoch 36/200\n", - "200/200 - 3s - loss: 1.3697e-04 - head: 7.5207e-05 - thorax: 1.0507e-04 - abdomen: 1.3913e-04 - wingL: 1.3497e-04 - wingR: 1.2511e-04 - forelegL4: 1.9152e-04 - forelegR4: 2.0264e-04 - midlegL4: 1.5207e-04 - midlegR4: 1.5519e-04 - hindlegL4: 1.6368e-04 - hindlegR4: 1.5869e-04 - eyeL: 9.0233e-05 - eyeR: 8.7055e-05 - val_loss: 0.0013 - val_head: 1.8066e-04 - val_thorax: 4.6591e-04 - val_abdomen: 9.9582e-04 - val_wingL: 7.2600e-04 - val_wingR: 0.0012 - val_forelegL4: 0.0015 - val_forelegR4: 0.0022 - val_midlegL4: 0.0019 - val_midlegR4: 0.0015 - val_hindlegL4: 0.0028 - val_hindlegR4: 0.0018 - val_eyeL: 0.0012 - val_eyeR: 9.6224e-04 - lr: 2.5000e-05 - 3s/epoch - 15ms/step\n", - "Epoch 37/200\n", - "200/200 - 3s - loss: 1.3638e-04 - head: 7.6822e-05 - thorax: 1.0531e-04 - abdomen: 1.4107e-04 - wingL: 1.4047e-04 - wingR: 1.2177e-04 - forelegL4: 1.9564e-04 - forelegR4: 1.7970e-04 - midlegL4: 1.5364e-04 - midlegR4: 1.5089e-04 - hindlegL4: 1.6647e-04 - hindlegR4: 1.6322e-04 - eyeL: 9.0198e-05 - eyeR: 8.7722e-05 - val_loss: 0.0017 - val_head: 2.3218e-04 - val_thorax: 5.3881e-04 - val_abdomen: 0.0011 - val_wingL: 0.0010 - val_wingR: 0.0019 - val_forelegL4: 0.0021 - val_forelegR4: 0.0028 - val_midlegL4: 0.0025 - val_midlegR4: 0.0016 - val_hindlegL4: 0.0033 - val_hindlegR4: 0.0029 - val_eyeL: 0.0015 - val_eyeR: 0.0012 - lr: 2.5000e-05 - 3s/epoch - 16ms/step\n", - "Epoch 00037: early stopping\n", - "INFO:sleap.nn.training:Finished training loop. [2.0 min]\n", - "INFO:sleap.nn.training:Deleting visualization directory: models/courtship.topdown_confmaps/viz\n", - "INFO:sleap.nn.training:Saving evaluation metrics to model folder...\n", - "\u001b[2KPredicting... \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m100%\u001b[0m ETA: \u001b[36m0:00:00\u001b[0m \u001b[31m39.3 FPS\u001b[0m31m48.8 FPS\u001b[0m31m49.5 FPS\u001b[0mFPS\u001b[0m\n", - "\u001b[?25hINFO:sleap.nn.evals:Saved predictions: models/courtship.topdown_confmaps/labels_pr.train.slp\n", - "INFO:sleap.nn.evals:Saved metrics: models/courtship.topdown_confmaps/metrics.train.npz\n", - "INFO:sleap.nn.evals:OKS mAP: 0.899237\n", - "\u001b[2KPredicting... \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m100%\u001b[0m ETA: \u001b[36m0:00:00\u001b[0m \u001b[31m14.2 FPS\u001b[0m0:00:01\u001b[0m \u001b[31m270.2 FPS\u001b[0mm\n", - "\u001b[?25hINFO:sleap.nn.evals:Saved predictions: models/courtship.topdown_confmaps/labels_pr.val.slp\n", - "INFO:sleap.nn.evals:Saved metrics: models/courtship.topdown_confmaps/metrics.val.npz\n", - "INFO:sleap.nn.evals:OKS mAP: 0.691378\n" - ] - } - ], - "source": [ - "!sleap-train baseline_medium_rf.topdown.json \"dataset/drosophila-melanogaster-courtship/courtship_labels.slp\" --run_name \"courtship.topdown_confmaps\" --video-paths \"dataset/drosophila-melanogaster-courtship/20190128_113421.mp4\"" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "whOf8PaFxYbt" - }, - "source": [ - "The models (along with the profiles and ground truth data used to train and validate the model) are saved in the `models/` directory:" - ] - }, - { - "cell_type": "code", - "execution_count": 27, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 306 + "cell_type": "code", + "execution_count": 36, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "id": "DUfnkxMtLcK3", + "outputId": "a6340ef1-eaac-42ef-f8d4-bcc499feb57b" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[31mERROR: Cannot uninstall opencv-python 4.6.0, RECORD file not found. Hint: The package was installed by conda.\u001b[0m\u001b[31m\n", + "\u001b[0m\u001b[31mERROR: Cannot uninstall shiboken2 5.15.6, RECORD file not found. You might be able to recover from this via: 'pip install --force-reinstall --no-deps shiboken2==5.15.6'.\u001b[0m\u001b[31m\n", + "\u001b[0m" + ] + } + ], + "source": [ + "!pip uninstall -qqq -y opencv-python opencv-contrib-python\n", + "!pip install -qqq \"sleap[pypi]>=1.3.4\"" + ] }, - "id": "GBUTQ2Cm44En", - "outputId": "ca298981-af65-43b3-f0f6-573f423acba8" - }, - "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[01;34mmodels/\u001b[00m\n", - "├── \u001b[01;34mcourtship.centroid\u001b[00m\n", - "│   ├── best_model.h5\n", - "│   ├── initial_config.json\n", - "│   ├── labels_gt.train.slp\n", - "│   ├── labels_gt.val.slp\n", - "│   ├── labels_pr.train.slp\n", - "│   ├── labels_pr.val.slp\n", - "│   ├── metrics.train.npz\n", - "│   ├── metrics.val.npz\n", - "│   ├── training_config.json\n", - "│   └── training_log.csv\n", - "└── \u001b[01;34mcourtship.topdown_confmaps\u001b[00m\n", - " ├── best_model.h5\n", - " ├── initial_config.json\n", - " ├── labels_gt.train.slp\n", - " ├── labels_gt.val.slp\n", - " ├── labels_pr.train.slp\n", - " ├── labels_pr.val.slp\n", - " ├── metrics.train.npz\n", - " ├── metrics.val.npz\n", - " ├── training_config.json\n", - " └── training_log.csv\n", - "\n", - "2 directories, 20 files\n" - ] - } - ], - "source": [ - "!tree models/" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "nIsKUX661xFK" - }, - "source": [ - "## Inference\n", - "Let's run inference with our trained models for centroids and centered instances." - ] - }, - { - "cell_type": "code", - "execution_count": 28, - "metadata": { - "id": "CLtjtq9E1Znr" - }, - "outputs": [ + "cell_type": "markdown", + "metadata": { + "id": "iq7jrgUksLtR" + }, + "source": [ + "## Download sample training data into Colab\n", + "Let's download a sample dataset from the SLEAP [sample datasets repository](https://github.com/talmolab/sleap-datasets) into Colab." + ] + }, { - "name": "stdout", - "output_type": "stream", - "text": [ - "Started inference at: 2023-09-01 13:42:03.066840\n", - "Args:\n", - "\u001b[1m{\u001b[0m\n", - "\u001b[2;32m│ \u001b[0m\u001b[32m'data_path'\u001b[0m: \u001b[32m'dataset/drosophila-melanogaster-courtship/20190128_113421.mp4'\u001b[0m,\n", - "\u001b[2;32m│ \u001b[0m\u001b[32m'models'\u001b[0m: \u001b[1m[\u001b[0m\n", - "\u001b[2;32m│ │ \u001b[0m\u001b[32m'models/courtship.centroid'\u001b[0m,\n", - "\u001b[2;32m│ │ \u001b[0m\u001b[32m'models/courtship.topdown_confmaps'\u001b[0m\n", - "\u001b[2;32m│ \u001b[0m\u001b[1m]\u001b[0m,\n", - "\u001b[2;32m│ \u001b[0m\u001b[32m'frames'\u001b[0m: \u001b[32m'0-100'\u001b[0m,\n", - "\u001b[2;32m│ \u001b[0m\u001b[32m'only_labeled_frames'\u001b[0m: \u001b[3;91mFalse\u001b[0m,\n", - "\u001b[2;32m│ \u001b[0m\u001b[32m'only_suggested_frames'\u001b[0m: \u001b[3;91mFalse\u001b[0m,\n", - "\u001b[2;32m│ \u001b[0m\u001b[32m'output'\u001b[0m: \u001b[3;35mNone\u001b[0m,\n", - "\u001b[2;32m│ \u001b[0m\u001b[32m'no_empty_frames'\u001b[0m: \u001b[3;91mFalse\u001b[0m,\n", - "\u001b[2;32m│ \u001b[0m\u001b[32m'verbosity'\u001b[0m: \u001b[32m'rich'\u001b[0m,\n", - "\u001b[2;32m│ \u001b[0m\u001b[32m'video.dataset'\u001b[0m: \u001b[3;35mNone\u001b[0m,\n", - "\u001b[2;32m│ \u001b[0m\u001b[32m'video.input_format'\u001b[0m: \u001b[32m'channels_last'\u001b[0m,\n", - "\u001b[2;32m│ \u001b[0m\u001b[32m'video.index'\u001b[0m: \u001b[32m''\u001b[0m,\n", - "\u001b[2;32m│ \u001b[0m\u001b[32m'cpu'\u001b[0m: \u001b[3;91mFalse\u001b[0m,\n", - "\u001b[2;32m│ \u001b[0m\u001b[32m'first_gpu'\u001b[0m: \u001b[3;91mFalse\u001b[0m,\n", - "\u001b[2;32m│ \u001b[0m\u001b[32m'last_gpu'\u001b[0m: \u001b[3;91mFalse\u001b[0m,\n", - "\u001b[2;32m│ \u001b[0m\u001b[32m'gpu'\u001b[0m: \u001b[32m'auto'\u001b[0m,\n", - "\u001b[2;32m│ \u001b[0m\u001b[32m'max_edge_length_ratio'\u001b[0m: \u001b[1;36m0.25\u001b[0m,\n", - "\u001b[2;32m│ \u001b[0m\u001b[32m'dist_penalty_weight'\u001b[0m: \u001b[1;36m1.0\u001b[0m,\n", - "\u001b[2;32m│ \u001b[0m\u001b[32m'batch_size'\u001b[0m: \u001b[1;36m4\u001b[0m,\n", - "\u001b[2;32m│ \u001b[0m\u001b[32m'open_in_gui'\u001b[0m: \u001b[3;91mFalse\u001b[0m,\n", - "\u001b[2;32m│ \u001b[0m\u001b[32m'peak_threshold'\u001b[0m: \u001b[1;36m0.2\u001b[0m,\n", - "\u001b[2;32m│ \u001b[0m\u001b[32m'max_instances'\u001b[0m: \u001b[3;35mNone\u001b[0m,\n", - "\u001b[2;32m│ \u001b[0m\u001b[32m'tracking.tracker'\u001b[0m: \u001b[3;35mNone\u001b[0m,\n", - "\u001b[2;32m│ \u001b[0m\u001b[32m'tracking.target_instance_count'\u001b[0m: \u001b[3;35mNone\u001b[0m,\n", - "\u001b[2;32m│ \u001b[0m\u001b[32m'tracking.pre_cull_to_target'\u001b[0m: \u001b[3;35mNone\u001b[0m,\n", - "\u001b[2;32m│ \u001b[0m\u001b[32m'tracking.pre_cull_iou_threshold'\u001b[0m: \u001b[3;35mNone\u001b[0m,\n", - "\u001b[2;32m│ \u001b[0m\u001b[32m'tracking.post_connect_single_breaks'\u001b[0m: \u001b[3;35mNone\u001b[0m,\n", - "\u001b[2;32m│ \u001b[0m\u001b[32m'tracking.clean_instance_count'\u001b[0m: \u001b[3;35mNone\u001b[0m,\n", - "\u001b[2;32m│ \u001b[0m\u001b[32m'tracking.clean_iou_threshold'\u001b[0m: \u001b[3;35mNone\u001b[0m,\n", - "\u001b[2;32m│ \u001b[0m\u001b[32m'tracking.similarity'\u001b[0m: \u001b[3;35mNone\u001b[0m,\n", - "\u001b[2;32m│ \u001b[0m\u001b[32m'tracking.match'\u001b[0m: \u001b[3;35mNone\u001b[0m,\n", - "\u001b[2;32m│ \u001b[0m\u001b[32m'tracking.robust'\u001b[0m: \u001b[3;35mNone\u001b[0m,\n", - "\u001b[2;32m│ \u001b[0m\u001b[32m'tracking.track_window'\u001b[0m: \u001b[3;35mNone\u001b[0m,\n", - "\u001b[2;32m│ \u001b[0m\u001b[32m'tracking.min_new_track_points'\u001b[0m: \u001b[3;35mNone\u001b[0m,\n", - "\u001b[2;32m│ \u001b[0m\u001b[32m'tracking.min_match_points'\u001b[0m: \u001b[3;35mNone\u001b[0m,\n", - "\u001b[2;32m│ \u001b[0m\u001b[32m'tracking.img_scale'\u001b[0m: \u001b[3;35mNone\u001b[0m,\n", - "\u001b[2;32m│ \u001b[0m\u001b[32m'tracking.of_window_size'\u001b[0m: \u001b[3;35mNone\u001b[0m,\n", - "\u001b[2;32m│ \u001b[0m\u001b[32m'tracking.of_max_levels'\u001b[0m: \u001b[3;35mNone\u001b[0m,\n", - "\u001b[2;32m│ \u001b[0m\u001b[32m'tracking.save_shifted_instances'\u001b[0m: \u001b[3;35mNone\u001b[0m,\n", - "\u001b[2;32m│ \u001b[0m\u001b[32m'tracking.kf_node_indices'\u001b[0m: \u001b[3;35mNone\u001b[0m,\n", - "\u001b[2;32m│ \u001b[0m\u001b[32m'tracking.kf_init_frame_count'\u001b[0m: \u001b[3;35mNone\u001b[0m\n", - "\u001b[1m}\u001b[0m\n", - "\n", - "2023-09-01 13:42:03.098811: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:939] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n", - "2023-09-01 13:42:03.103255: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:939] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n", - "2023-09-01 13:42:03.103982: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:939] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n", - "INFO:sleap.nn.inference:Auto-selected GPU 0 with 23050 MiB of free memory.\n", - "Versions:\n", - "SLEAP: 1.3.2\n", - "TensorFlow: 2.7.0\n", - "Numpy: 1.21.5\n", - "Python: 3.7.12\n", - "OS: Linux-5.15.0-78-generic-x86_64-with-debian-bookworm-sid\n", - "\n", - "System:\n", - "GPUs: 1/1 available\n", - " Device: /physical_device:GPU:0\n", - " Available: True\n", - " Initalized: False\n", - " Memory growth: True\n", - "\n", - "Video: dataset/drosophila-melanogaster-courtship/20190128_113421.mp4\n", - "2023-09-01 13:42:03.157392: I tensorflow/core/platform/cpu_feature_guard.cc:151] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", - "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "2023-09-01 13:42:03.158019: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:939] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n", - "2023-09-01 13:42:03.158864: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:939] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n", - "2023-09-01 13:42:03.159656: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:939] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n", - "2023-09-01 13:42:03.455402: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:939] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n", - "2023-09-01 13:42:03.456138: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:939] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n", - "2023-09-01 13:42:03.456803: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:939] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n", - "2023-09-01 13:42:03.457464: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1525] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 21145 MB memory: -> device: 0, name: NVIDIA RTX A5000, pci bus id: 0000:01:00.0, compute capability: 8.6\n", - "\u001b[2KPredicting... \u001b[38;5;237m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m ETA: \u001b[36m-:--:--\u001b[0m \u001b[31m?\u001b[0m2023-09-01 13:42:07.038687: I tensorflow/stream_executor/cuda/cuda_dnn.cc:366] Loaded cuDNN version 8201\n", - "\u001b[2KPredicting... \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m100%\u001b[0m ETA: \u001b[36m0:00:00\u001b[0m \u001b[31m51.9 FPS\u001b[0m[0m \u001b[31m126.4 FPS\u001b[0m FPS\u001b[0mFPS\u001b[0m\n", - "\u001b[?25hFinished inference at: 2023-09-01 13:42:10.842469\n", - "Total runtime: 7.775644779205322 secs\n", - "Predicted frames: 101/101\n", - "Provenance:\n", - "\u001b[1m{\u001b[0m\n", - "\u001b[2;32m│ \u001b[0m\u001b[32m'model_paths'\u001b[0m: \u001b[1m[\u001b[0m\n", - "\u001b[2;32m│ │ \u001b[0m\u001b[32m'models/courtship.centroid/training_config.json'\u001b[0m,\n", - "\u001b[2;32m│ │ \u001b[0m\u001b[32m'models/courtship.topdown_confmaps/training_config.json'\u001b[0m\n", - "\u001b[2;32m│ \u001b[0m\u001b[1m]\u001b[0m,\n", - "\u001b[2;32m│ \u001b[0m\u001b[32m'predictor'\u001b[0m: \u001b[32m'TopDownPredictor'\u001b[0m,\n", - "\u001b[2;32m│ \u001b[0m\u001b[32m'sleap_version'\u001b[0m: \u001b[32m'1.3.2'\u001b[0m,\n", - "\u001b[2;32m│ \u001b[0m\u001b[32m'platform'\u001b[0m: \u001b[32m'Linux-5.15.0-78-generic-x86_64-with-debian-bookworm-sid'\u001b[0m,\n", - "\u001b[2;32m│ \u001b[0m\u001b[32m'command'\u001b[0m: \u001b[32m'/home/talmolab/micromamba/envs/s0/bin/sleap-track dataset/drosophila-melanogaster-courtship/20190128_113421.mp4 --frames 0-100 -m models/courtship.centroid -m models/courtship.topdown_confmaps'\u001b[0m,\n", - "\u001b[2;32m│ \u001b[0m\u001b[32m'data_path'\u001b[0m: \u001b[32m'dataset/drosophila-melanogaster-courtship/20190128_113421.mp4'\u001b[0m,\n", - "\u001b[2;32m│ \u001b[0m\u001b[32m'output_path'\u001b[0m: \u001b[32m'dataset/drosophila-melanogaster-courtship/20190128_113421.mp4.predictions.slp'\u001b[0m,\n", - "\u001b[2;32m│ \u001b[0m\u001b[32m'total_elapsed'\u001b[0m: \u001b[1;36m7.775644779205322\u001b[0m,\n", - "\u001b[2;32m│ \u001b[0m\u001b[32m'start_timestamp'\u001b[0m: \u001b[32m'2023-09-01 13:42:03.066840'\u001b[0m,\n", - "\u001b[2;32m│ \u001b[0m\u001b[32m'finish_timestamp'\u001b[0m: \u001b[32m'2023-09-01 13:42:10.842469'\u001b[0m\n", - "\u001b[1m}\u001b[0m\n", - "\n", - "Saved output: dataset/drosophila-melanogaster-courtship/20190128_113421.mp4.predictions.slp\n" - ] - } - ], - "source": [ - "!sleap-track \"dataset/drosophila-melanogaster-courtship/20190128_113421.mp4\" --frames 0-100 -m \"models/courtship.centroid\" -m \"models/courtship.topdown_confmaps\"" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "nzObCUToEqwA" - }, - "source": [ - "When inference is finished, predictions are saved in a file. Since we didn't specify a path, it will be saved as `