solving merge conflicts

.devcontainer/Dockerfile

@@ -0,0 +1,25 @@
+FROM python:3.12-bookworm
+
+ARG USERNAME=flygym
+ARG USER_UID=1000
+ARG USER_GID=$USER_UID
+
+# Create the user
+RUN groupadd --gid $USER_GID $USERNAME \
+    && useradd --uid $USER_UID --gid $USER_GID -m $USERNAME \
+    # [Optional] Add sudo support. Omit if you don't need to install software after connecting.
+    && apt-get update \
+    && apt-get install -y sudo \
+    && echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME \
+    && chmod 0440 /etc/sudoers.d/$USERNAME \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install system dependencies
+RUN apt update && \
+    apt-get install -y libegl1-mesa-dev ffmpeg && \
+    rm -rf /var/lib/apt/lists/*
+
+# Set renderer
+ENV MUJOCO_GL=egl
+ENV PYOPENGL_PLATFORM=egl
+USER $USERNAME

.devcontainer/devcontainer.json

@@ -0,0 +1,32 @@
+{
+    "name": "flygym_environment",
+    "build": {
+        "dockerfile": "Dockerfile",
+        "context": ".."
+    },
+    "customizations": {
+        "vscode": {
+            "extensions": [
+                "donjayamanne.python-extension-pack"
+            ],
+            "settings": {
+                "terminal.integrated.defaultProfile.linux": "bash",
+                "terminal.integrated.profiles.linux": {
+                    "bash": {
+                        "path": "bash"
+                    }
+                }
+            }
+        }
+    },
+    "mounts": [
+        "source=/tmp/.X11-unix,target=/tmp/.X11-unix,type=bind,consistency=cached"
+    ],
+    "containerEnv": {
+        "DISPLAY": "${localEnv:DISPLAY}"
+    },
+    "postCreateCommand": "pip install -e \".[examples,dev]\" --no-warn-script-location",
+    "runArgs": [
+        "--device=/dev/dri"
+    ]
+}

.github/workflows/documentation.yaml

@@ -15,7 +15,6 @@ jobs:
           pip install --upgrade pip
           pip cache purge
           pip install sphinxcontrib-googleanalytics
-          pip install "flyvision @ https://github.com/Nely-EPFL/flyvis/archive/refs/heads/main.zip"
           pip cache purge
           pip install -e ."[dev,examples]"
           pip install toolz

.github/workflows/tests_full.yaml

@@ -25,8 +25,6 @@ jobs:
           sudo apt-get install ffmpeg
           pip install --upgrade pip
           pip cache purge
-          pip install "flyvision @ https://github.com/Nely-EPFL/flyvis/archive/refs/heads/main.zip"
-          pip cache purge
           pip install -e ".[dev,examples]"
       - name: Lint with ruff
         run: |

.github/workflows/tests_macos.yaml

@@ -22,8 +22,6 @@ jobs:
         run: |
           pip install --upgrade pip
           pip cache purge
-          pip install "flyvision @ https://github.com/Nely-EPFL/flyvis/archive/refs/heads/main.zip"
-          pip cache purge
           pip install -e ".[dev,examples]"
       - name: Test with pytest
         run: |

.github/workflows/tests_ubuntu22.yaml

@@ -25,8 +25,6 @@ jobs:
           sudo apt-get install ffmpeg
           pip install --upgrade pip
           pip cache purge
-          pip install "flyvision @ https://github.com/Nely-EPFL/flyvis/archive/refs/heads/main.zip"
-          pip cache purge
           pip install -e ".[dev,examples]"
       - name: Test with pytest
         run: |

.github/workflows/tests_windows.yaml

@@ -21,8 +21,6 @@ jobs:
         run: |
           pip install --upgrade pip
           pip cache purge
-          pip install "flyvision @ https://github.com/nely-epfl/flyvis/archive/refs/heads/main.zip"
-          pip cache purge
           pip install -e .[dev,examples]
       - name: Test with pytest
         run: |

Dockerfile

@@ -19,8 +19,7 @@ ENV PYOPENGL_PLATFORM=egl
 ADD . $HOME/flygym/
 
 # Set up virtual environment and install dependencies
-RUN pip install "flyvision @ https://github.com/Nely-EPFL/flyvis/archive/refs/heads/main.zip" && \
-    pip install -e ".[examples,dev]"
+RUN pip install -e ".[examples,dev]"
 
 # Set entrypoint
 ENTRYPOINT ["/bin/bash"]

README.md

@@ -2,7 +2,7 @@
 
 ![](https://github.com/NeLy-EPFL/_media/blob/main/flygym/banner_large.jpg?raw=true)
 
-[**Documentation**](https://neuromechfly.org/) | [**Preprint**](https://www.biorxiv.org/content/10.1101/2023.09.18.556649) | [**Discussion Board**](https://github.com/NeLy-EPFL/flygym/discussions)
+[**Documentation**](https://neuromechfly.org/) | [**Paper**](https://www.nature.com/articles/s41592-024-02497-y.epdf?sharing_token=jK2FbKWL99-O28WNqrpXWNRgN0jAjWel9jnR3ZoTv0MjiFZczOI3_5wYVxbEbClrTuJzjKyEfhm2kIwso489-ypEsSqlyasWAEsBCvR9WU5poT-q2bblI6hCc7Zji6wb_jZjfXl7KWLbd2pgZTmWvk_ADQ6RuzlnHwvQyipMJzg%3D) | [**Discussion Board**](https://github.com/NeLy-EPFL/flygym/discussions)
 
 ![Python: 3.9–3.12](https://img.shields.io/badge/python-3.9%E2%80%933.12-blue)
 [![License: MPL 2.0](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://github.com/NeLy-EPFL/flygym/blob/main/LICENSE)
@@ -16,13 +16,13 @@
 
 ![overview_video](https://github.com/NeLy-EPFL/_media/blob/main/flygym/overview_video.gif?raw=true)
 
-This repository contains the source code for FlyGym, the Python library for NeuroMechFly v2, a digital twin of the adult fruit fly *Drosophila* melanogaster that can see, smell, walk over challenging terrain, and interact with the environment (see our [NeuroMechFly v2 paper](https://www.biorxiv.org/content/10.1101/2023.09.18.556649)).
+This repository contains the source code for FlyGym, the Python library for NeuroMechFly v2, a digital twin of the adult fruit fly *Drosophila* melanogaster that can see, smell, walk over challenging terrain, and interact with the environment (see our [NeuroMechFly v2 paper](https://www.nature.com/articles/s41592-024-02497-y.epdf?sharing_token=jK2FbKWL99-O28WNqrpXWNRgN0jAjWel9jnR3ZoTv0MjiFZczOI3_5wYVxbEbClrTuJzjKyEfhm2kIwso489-ypEsSqlyasWAEsBCvR9WU5poT-q2bblI6hCc7Zji6wb_jZjfXl7KWLbd2pgZTmWvk_ADQ6RuzlnHwvQyipMJzg%3D)).
 
 NeuroMechFly consists of the following components:
 - **Biomechanical model:** The biomechanical model is based on a micro-CT scan of a real adult female fly (see our original NeuroMechFly publication). We have adjusted several body segments (in particular in the antennae) to better reflect the biological reality.
 - **Vision:** The fly has compound eyes consisting of individual units called ommatidia arranged on a hexagonal lattice. We have simulated the visual inputs on the fly’s retinas.
 - **Olfaction:** The fly has odor receptors in the antennae and the maxillary palps. We have simulated the odor inputs experienced by the fly by computing the odor/chemical intensity at these locations.
-- **Hierarchical control:** The fly’s Central Nervous System consists of the brain and the Ventral Nerve Cord (VNC), a hierarchy analogous to our brain-spinal cord organization. The user can build a two-part model — one handling brain-level sensory integration and decision making and one handling VNC-level motor control — with a interface between the two consisting of descending (brain-to-VNC) and ascending (VNC-to-brain) representations.
+- **Hierarchical control:** The fly’s Central Nervous System consists of the brain and the Ventral Nerve Cord (VNC), a hierarchy analogous to our brain-spinal cord organization. The user can build a two-part model — one handling brain-level sensory integration and decision making and one handling VNC-level motor control — with an interface between the two consisting of descending (brain-to-VNC) and ascending (VNC-to-brain) representations.
 - **Leg adhesion:** Insects have evolved specialized adhesive structures at the tips of the legs that enable locomotion vertical walls and overhanging ceilings. We have simulated these structures in our model. The mechanism by which the fly lifts the legs during locomotion despite adhesive forces is not well understood; to abstract this, adhesion can be turned on/off during leg stance/swing.
 - **Mechanosensory feedback:** The user has access to the joint angles, forces, and contact forces experienced by the fly.
 
@@ -70,4 +70,4 @@ If you use NeuroMechFly in your work, please cite the following papers:
   title = {{NeuroMechFly}, a neuromechanical model of adult {Drosophila} melanogaster},
   journal = {Nature Methods}
 }
-```
+```

doc/source/api_ref/mdp_specs.rst

@@ -17,9 +17,10 @@ Default ``Simulation``
 * "fly": The fly state as a NumPy array of shape (4, 3). 0th row: x, y, z position of the fly in arena. 1st row: x, y, z velocity of the fly in arena. 2nd row: orientation of fly around x, y, z axes. 3rd row: rate of change of fly orientation.
 * "contact_forces": Readings of the touch contact sensors, one placed for each of the body segments specified in ``Fly.contact_sensor_placements``. This is a NumPy array of shape (num_contact_sensor_placements, 3).
 * "end_effectors": The positions of the end effectors (most distal tarsus link) of the legs as a NumPy array of shape (6, 3). The order of the legs is: LF, LM, LH, RF, RM, RH (L/R = left/right, F/M/H = front/middle/hind).
-* "fly_orientation": NumPy array of shape (3,). This is the vector (x, y, z) pointing toward the direction that the fly is facing.
+* "fly_orientation": [Deprecated] this entry in the observation space is deprecated and will be removed in future releases. Use the "forward" vector from "cardinal_vectors" instead. Previously, this variable is a NumPy array of shape (3,). This is the vector (x, y, z) pointing toward the direction that the fly is facing.
 * "vision" (if ``Fly.enable_vision`` is True): The light intensities sensed by the ommatidia on the compound eyes. This is a NumPy array of shape (2, num_ommatidia_per_eye, 2), where the zeroth dimension is the side (left, right in that order); the second dimension specifies the ommatidium, and the last column is for the spectral channel (yellow-type, pale-type in that order). Each ommatidium only has one channel with nonzero reading. The intensities are given on a [0, 1] scale.
 * "odor_intensity" (if ``Fly.enable_olfaction`` is True): The odor intensities sensed by the odor sensors (by default 2 antennae and 2 maxillary palps). This is a NumPy array of shape (odor_space_dimension, num_sensors).
+* "cardinal_vectors": The cardinal vectors (forward, left, up) of the fly's spatial orientation in the global frame. This is a NumPy array of shape (3, 3) where the 0th dimension specifies forward/left/up in that order, and the 1st dimension specifies the x/y/z components of the vector. Note that the forward vector is a bit tiled up; therefore if the fly is walking perfectly forward on the floor, one should expect a non-negligible positive z component in the forward vector (and a non-negligible negative value in the x component of the up vector).
 
 **Info:** The info dictionary contains the following:
 
@@ -80,15 +81,19 @@ Plume tracking task (``PlumeNavigationTask``)
 
 
 
-NeuroMechFly with connectome-constrained vision network (``RealisticVisionController``)
+NeuroMechFly with connectome-constrained vision network (``RealisticVisionFly``)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-**Action, reward, termination, and truncation:** The ``flygym.examples.realistic_vision.RealisticVisionController`` class expects the same action and returns the same reward, "terminated" flag, and "truncated" flags as ``HybridTurningController``.
+.. important::
 
-**Observation:** In addition to what is returned by the ``HybridTurningController``, the ``flygym.examples.realistic_vision.RealisticVisionController`` class also provides the following in the observation dictionary:
+   Note that ``RealisticVisionFly`` is an extension of ``Fly``, not ``Simulation``. The action and observation spaces of the simulation are modified accordingly nonetheless at the level of each flies.
+
+**Action, reward, termination, and truncation:** The ``flygym.examples.vision.RealisticVisionFly`` class expects the same action and returns the same reward, "terminated" flag, and "truncated" flags as ``HybridTurningController``.
+
+**Observation:** In addition to what is returned by the ``HybridTurningController``, the ``flygym.examples.vision.RealisticVisionFly`` class also provides the following in the observation dictionary:
 
 * "nn_activities_arr": The activities of the visual system neurons, represented as a NumPy array of shape (2, num_cells_per_eye). The 0th dimension corresponds to the eyes in the order (left, right).
 
-**Info:** In addition to what is returned by the ``HybridTurningController``, the ``flygym.examples.realistic_vision.RealisticVisionController`` class also provides the following in the "info" dictionary:
+**Info:** In addition to what is returned by the ``HybridTurningController``, the ``flygym.examples.vision.RealisticVisionFly`` class also provides the following in the "info" dictionary:
 
-* "nn_activities": Activities of the visual system neurons as a ``flyvision.LayerActivity`` object. This is similar to ``obs["nn_activities_arr"]`` but in the form of a ``flyvision.LayerActivity`` object rather than a plain array.
+* "nn_activities": Activities of the visual system neurons as a ``flyvis.LayerActivity`` object. This is similar to ``obs["nn_activities_arr"]`` but in the form of a ``flyvis.LayerActivity`` object rather than a plain array.

doc/source/api_ref/vision.rst

@@ -4,7 +4,7 @@ Vision
 This page documents the implementation of the visual input received by the simulated fly. Note that in the typical use case, the user should **not** have to access most of the functions described here. Instead, the visual inputs are given as a part of the *observation* returned by ``NeuroMechFly`` at each time step. Nonetheless, the full API reference is provided here for greater transparency.
 
 .. note::
-   For API references of NeuroMechFly simulation with the connectome-constrained model proposed in `Lappalainen et al., 2023, <https://www.biorxiv.org/content/10.1101/2023.03.11.532232>`_, see the `Advanced Vision <api_ref/examples/vision.html>`_ page.
+   For API references of NeuroMechFly simulation with the connectome-constrained model proposed in `Lappalainen et al., 2024, <https://doi.org/10.1038/s41586-024-07939-3>`_, see the `Advanced Vision <api_ref/examples/vision.html>`_ page.
 
 Retina simulation
 -----------------

doc/source/changelog.rst

@@ -1,6 +1,25 @@
 Change Log
 ==========
 
+.. note:: 
+
+   FlyGym uses `EffVer <https://jacobtomlinson.dev/effver/>`_ as its versioning policy. The version number will communicate how much effort we expect a user to spend to adopt the new version. 
+
+* **1.2.0:**
+
+  * TODO
+
+* **1.1.0:**
+
+  * Added cardinal direction sensing (vectors describing +x, +y, +z of the fly) to the observation space.
+  * Removed legacy spawn orientation preprocessing: Previously, pi/2 was subtracted from the user-specified spawn orientation on the x-y plane. This was to make the behavior consistent with a legacy version of NeuroMechFly. This behavior is no longer desired; from this version onwards, the spawn orientation is used as is.
+  * Strictly fixed the required MuJoCo version to 3.2.3, and dm_control version to 1.0.23. This is to prevent API-breaking changes in future versions of these libraries from affecting FlyGym. FlyGym maintainers will periodically check for compatibility with newer versions of these libraries.
+  * Changed flip detection method: Previously, flips are reported when all legs reliably lose contact with the ground. Now, we simply check if the z component of the "up" cardinal vector is negative. Additionally, the ``detect_flip`` parameter of ``Fly`` is now deprecated; flips are always detect and reported.
+  * Allowed different sets of DoFs to be monitored vs. actuated. Previously, the two sets are always the same.
+  * From this version onwards, we will use `EffVer <https://jacobtomlinson.dev/effver/>`_ as the versioning policy. The version number will communicate how much effort we expect a user will need to spend to adopt the new version. While we previously tried to adhere to the stricter `SemVer <https://semver.org/>`_, we found that it was not effective because many core dependencies of FlyGym (e.g., MuJoCo, NumPy, and Python itself) do not use SemVer.
+
+* **1.0.1:** Fixed minor bugs related to the set of DoFs in the predefined poses, and to rendering at extremely high frequencies. Fixed outdated class names and links in the docs. In addition, contact sensor placements used by the hybrid turning controller are now added to the ``preprogrammed`` module.
+
 * **1.0.0:** In spring 2024, NeuroMechFly was used, for the second time, in a course titled "`Controlling behavior in animals and robots <https://edu.epfl.ch/coursebook/en/controlling-behavior-in-animals-and-robots-BIOENG-456>`_" at EPFL. At the same time, we revised the NeuroMechFly v2 manuscript. In these processes, we significantly improved the FlyGym package, added new functionalities, and incorporated changes as we received feedback from the students. These enhancements are released as FlyGym version 1.0.0. This release is not backward compatible; please refer to the `tutorials <https://neuromechfly.org/tutorials/index.html>`_ and `API references <https://neuromechfly.org/api_ref/index.html>`_ for more information. The main changes are:
 
   * Major API changes:

doc/source/contributing.rst

@@ -8,6 +8,9 @@ Code of conduct & licensing
 ---------------------------
 Please respect the `Contributor Covenant Code of Conduct <https://www.contributor-covenant.org/version/2/1/code_of_conduct/code_of_conduct.txt>`_. FlyGym is made open source under `Apache License 2.0 <https://github.com/NeLy-EPFL/flygym/blob/main/LICENSE>`_. By contributing to this package (including any issue, pull request, and discussion), you agree that your content will be shared under the same license.
 
+Versioning
+----------
+FlyGym uses `EffVer <https://jacobtomlinson.dev/effver/>`_ versioning system, which is based on the amount of effort that users are expected to spend to adopt the new version. The version number is in the format of ``X.Y.Z``, where ``X`` is the macro version, ``Y`` is the meso version, and ``Z`` is the micro version. When a macro version is updated, it means that a large effort is required to adopt the new version. When a meso version is updated, it means that a some effort is required to adopt the new version. When a micro version is updated, it means that no effort is required at all (e.g. bug fixes or optimizations that are not exposed to the API).
 
 Branches
 --------
@@ -25,7 +28,7 @@ Documentation
 -------------
 We use the  `NumPy Docstring Style <https://numpydoc.readthedocs.io/en/latest/format.html>`_. We use a line length limit of 75 characters for docstrings. Please stick with the NumPy style so the API reference can be generated automatically.
 
-The source files (in RST) of the documentation website are located in the ``doc/source`` folder. The API reference is generated automatically using `Sphinx <https://www.sphinx-doc.org/en/master/>`_. The documentation is written in `reStructuredText <https://sphinx-tutorial.readthedocs.io/step-1/>`_ (RST). When you merge a pull request into the main branch, the documentation is automatically built and deployed on `neuromechfly.org <https://neuromechfly.org/>`_. If you want to check the documentation on a branch (that is not `main`) locally, you can run `make html` under the `doc` folder. The generated HTML files will be placed under `doc/build/html`. You can open `doc/build/html/index.html` in your browser to view the documentation.
+The source files (in RST) of the documentation website are located in the ``doc/source`` folder. The API reference is generated automatically using `Sphinx <https://www.sphinx-doc.org/en/master/>`_. The documentation is written in `reStructuredText <https://sphinx-tutorial.readthedocs.io/step-1/>`_ (RST). When you merge a pull request into the main branch, the documentation is automatically built and deployed on `neuromechfly.org <https://neuromechfly.org/>`_. If you want to check the documentation on a branch (that is not ``main``) locally, you can run ``make html`` under the ``doc`` folder. The generated HTML files will be placed under ``doc/build/html``. You can open ``doc/build/html/index.html`` in your browser to view the documentation.
 
 API changes / migration guide
 -----------------------------