Merge pull request #328 from NextCenturyCorporation/development

[Python] Update master to 0.4.2

.gitignore

@@ -38,3 +38,7 @@ scripts/**/*.json
 scripts/**/*.mp4
 scripts/**/*.png
 scripts/**/*.zip
+
+# Logging output and user config
+logs/*.log
+scripts/log.config.user.py

CPU_Container.dockerfile

@@ -9,7 +9,11 @@ ARG DEBIAN_FRONTEND=noninteractive
 ENV TZ=America/New_York
 
 # --build-arg mcsversion=0.0.x to override default in docker build command
-ARG mcsversion=0.4.1.1
+ARG mcsversion=0.4.2
+ARG mcs_library_version=master
+
+WORKDIR /mcs
+
 
 ENV DEBIAN_FRONTEND=noninteractive
 ENV MCS_EXECUTABLE_PATH="/MCS-AI2-THOR-Unity-App-v${mcsversion}.x86_64"
@@ -50,16 +54,19 @@ RUN apt-get update -qq \
         cmake pkg-config libgtk-3-dev libavcodec-dev libavformat-dev \
         libswscale-dev libv4l-dev libxvidcore-dev libx264-dev libjpeg-dev libpng-dev libtiff-dev gfortran openexr \
         libatlas-base-dev libtbb2 libtbb-dev libdc1394-22-dev libopenexr-dev libgstreamer-plugins-base1.0-dev \
-        libgstreamer1.0-dev \
-        # mesa-utils libgl1-mesa-glx \
-    && rm -rf /var/lib/apt/lists/* && \
+        libgstreamer1.0-dev && \
+    python3 -m pip install --upgrade pip setuptools wheel && \
+    python3 -m pip install git+https://github.com/NextCenturyCorporation/MCS@${mcs_library_version}#egg=machine_common_sense && \
+    rm -rf /var/lib/apt/lists/* && \
     ln -s /usr/bin/python3 /usr/bin/python
 
+WORKDIR /mcs
 
-RUN wget https://github.com/NextCenturyCorporation/MCS/releases/download/${mcsversion}/MCS-AI2-THOR-Unity-App-v${mcsversion}.x86_64 && \
-    wget https://github.com/NextCenturyCorporation/MCS/releases/download/${mcsversion}/MCS-AI2-THOR-Unity-App-v${mcsversion}_Data.tar.gz && \
-    python3 -m pip install git+https://github.com/NextCenturyCorporation/MCS@${mcsversion}#egg=machine_common_sense && \
-    tar -xzvf MCS-AI2-THOR-Unity-App-v${mcsversion}_Data.tar.gz && \
-    chmod a+x MCS-AI2-THOR-Unity-App-v${mcsversion}.x86_64 && \
-    rm MCS-AI2-THOR-Unity-App-v${mcsversion}_Data.tar.gz && echo "[MCS]\nmetadata: oracle" > /mcs_config_oracle.yaml && \
-    echo "[MCS]\nmetadata:  level1" > /mcs_config_level1.yaml && echo "[MCS]\nmetadata: level2" > /mcs_config_level2.yaml
+# Add ai2thor/Unity resources
+ADD https://github.com/NextCenturyCorporation/MCS/releases/download/${mcsversion}/MCS-AI2-THOR-Unity-App-v${mcsversion}.x86_64 /mcs
+ADD https://github.com/NextCenturyCorporation/MCS/releases/download/${mcsversion}/MCS-AI2-THOR-Unity-App-v${mcsversion}_Data.tar.gz /mcs
+ADD https://github.com/NextCenturyCorporation/MCS/releases/download/${mcsversion}/UnityPlayer.so /mcs
+RUN tar -xzvf /mcs/MCS-AI2-THOR-Unity-App-v${mcsversion}_Data.tar.gz -C /mcs && \
+    chmod a+x /mcs/MCS-AI2-THOR-Unity-App-v${mcsversion}.x86_64 && \
+    rm /mcs/MCS-AI2-THOR-Unity-App-v${mcsversion}_Data.tar.gz && echo "[MCS]\nmetadata: oracle" > /mcs/mcs_config_oracle.yaml && \
+    echo "[MCS]\nmetadata:  level1" > /mcs/mcs_config_level1.yaml && echo "[MCS]\nmetadata: level2" > /mcs/mcs_config_level2.yaml

Dockerfile

@@ -19,20 +19,22 @@ ENV NVIDIA_DRIVER_CAPABILITIES ${NVIDIA_DRIVER_CAPABILITIES},display
 # ENV LANG C.UTF-8
 
 # --build-arg mcsversion=0.0.x to override default in docker build command
-ARG mcsversion=0.4.1.1
+ARG mcsversion=0.4.2
+ARG mcs_library_version=master
 
 WORKDIR /mcs
 
 RUN apt-get update && \
     apt-get install -y git python3 python3-pip mesa-utils && \
     python3 -m pip install --upgrade pip setuptools wheel && \
-    python3 -m pip install git+https://github.com/NextCenturyCorporation/MCS@${mcsversion}#egg=machine_common_sense && \
+    python3 -m pip install git+https://github.com/NextCenturyCorporation/MCS@${mcs_library_version}#egg=machine_common_sense && \
     rm -rf /var/lib/apt/lists/* && \
     ln -s /usr/bin/python3 /usr/bin/python
 
-# add ai2thor/unity resources
+# Add ai2thor/Unity resources
 ADD https://github.com/NextCenturyCorporation/MCS/releases/download/${mcsversion}/MCS-AI2-THOR-Unity-App-v${mcsversion}.x86_64 /mcs
 ADD https://github.com/NextCenturyCorporation/MCS/releases/download/${mcsversion}/MCS-AI2-THOR-Unity-App-v${mcsversion}_Data.tar.gz /mcs
+ADD https://github.com/NextCenturyCorporation/MCS/releases/download/${mcsversion}/UnityPlayer.so /mcs
 RUN tar -xzvf /mcs/MCS-AI2-THOR-Unity-App-v${mcsversion}_Data.tar.gz -C /mcs && \
     chmod a+x /mcs/MCS-AI2-THOR-Unity-App-v${mcsversion}.x86_64 && \
     rm /mcs/MCS-AI2-THOR-Unity-App-v${mcsversion}_Data.tar.gz

README.md

@@ -4,6 +4,7 @@
 - [Download](#download)
 - [Training Datasets](#training-datasets)
 - [Usage](#usage)
+- [Logging](#logging)
 - [Run with Human Input](#run-with-human-input)
 - [Run with Scene Timer](#run-with-scene-timer)
 - [Config File](#config-file)
@@ -16,7 +17,7 @@
 
 ## Installation
 
-The latest release of the MCS Python library is `0.4.1.1`.
+The latest release of the MCS Python library is `0.4.2`.
 
 ### Virtual Environments
 
@@ -65,35 +66,35 @@ Here are the instructions for downloading and installing our latest Unity releas
 
 ### Unity Application
 
-The latest release of the MCS Unity app is `0.4.1.1`.
+The latest release of the MCS Unity app is `0.4.2`.
 
 Please note that our Unity App is built on Linux or Mac.
 
 Linux Version:
 
-1. [Download the latest MCS Unity App](https://github.com/NextCenturyCorporation/MCS/releases/download/0.4.1-1/MCS-AI2-THOR-Unity-App-v0.4.1.1.x86_64)
+1. [Download the latest MCS Unity App](https://github.com/NextCenturyCorporation/MCS/releases/download/0.4.2/MCS-AI2-THOR-Unity-App-v0.4.2.x86_64)
 
-2. [Download the latest MCS Unity Data Directory TAR](https://github.com/NextCenturyCorporation/MCS/releases/download/0.4.1-1/MCS-AI2-THOR-Unity-App-v0.4.1.1_Data.tar.gz)
+2. [Download the latest MCS Unity Data Directory TAR](https://github.com/NextCenturyCorporation/MCS/releases/download/0.4.2/MCS-AI2-THOR-Unity-App-v0.4.2_Data.tar.gz)
 
-3. [Download the latest UnityPlayer.so file](https://github.com/NextCenturyCorporation/MCS/releases/download/0.4.1-1/UnityPlayer.so)
+3. [Download the latest UnityPlayer.so file](https://github.com/NextCenturyCorporation/MCS/releases/download/0.4.2/UnityPlayer.so)
 
 4. Ensure that both the Unity App and the TAR are in the same directory.
 
 5. Untar the Data Directory:
 
 ```
-tar -xzvf MCS-AI2-THOR-Unity-App-v0.4.1.1_Data.tar.gz
+tar -xzvf MCS-AI2-THOR-Unity-App-v0.4.2_Data.tar.gz
 ```
 
 6. Mark the Unity App as executable:
 
 ```
-chmod a+x MCS-AI2-THOR-Unity-App-v0.4.1.1.x86_64
+chmod a+x MCS-AI2-THOR-Unity-App-v0.4.2.x86_64
 ```
 
 Mac Version:
 
-[Download the Mac ZIP](https://github.com/NextCenturyCorporation/MCS/releases/download/0.4.1-1/MCS-AI2-THOR-Unity-App-v0.4.1.1-mac.zip)
+[Download the Mac ZIP](https://github.com/NextCenturyCorporation/MCS/releases/download/0.4.2/MCS-AI2-THOR-Unity-App-v0.4.2-mac.zip)
 
 
 ## Training Datasets
@@ -212,6 +213,43 @@ for scene_json_file_path in scene_json_file_list:
     controller.end_scene()
 ```
 
+### Logging
+#### Initialize development logging config settings
+In startup script make sure you include:
+```python
+import machine_common_sense as mcs
+
+#This code initializes logging using config files
+mcs.init_logging()
+```
+#### Logging configuration files
+This logging configuration system uses a file at [scripts/log.config.py](./scripts/log.config.py) as a default configuration file.  See that file for links to documentation and some descriptions.
+
+Users can copy that file to scripts/log.config.user.py to create their own configuration.  This file is gitignored so it will not be shared.
+
+#### Why use logging and configuration files
+
+We use logging configuration files so users have a reasonable default configuration but also have the ability to override when circumstances demand.  This eliminates the need to alter code when changing what logging is necessary.  In the future, additional features to the initialization function to support difficult overrides if necessary (i.e. different logging override files for dev, test, eval, etc).
+
+#### Example with logging config file:
+
+Sample of how to log in all files:
+```python
+import logging
+
+# logging by name is useful to turn certain logs on/off and
+# also to help determine where an error log might have originated from
+logger = logging.getLogger(__name__)
+
+def any_function(self):
+    #basic log functions in order of level.  If logger or handler is set to a level, that level and above will be logged.
+    logger.critical("This message is see as long as the logger is enabled")
+    logger.error("This message reports an error")
+    logger.info("This message is general info")
+    logger.debug("This message is mainly for debugging and shouldn't cause usage problems if missing")
+    # Unlike many common loggers in other languages, python logging doesn't seem to have TRACE
+```
+
 ## Run with Human Input
 
 To start the Unity application and enter your actions and parameters from the terminal, you can run the `run_in_human_input_mode` script that was installed in the package with the MCS Python Library (the `mcs_unity_build_file` is the Unity executable downloaded previously):
@@ -239,18 +277,6 @@ To use an MCS configuration file, you can either pass in a file path via the `co
 
 ### Config File Properties
 
-#### debug
-
-(boolean, optional)
-
-Whether to save MCS output debug files in this folder and print debug output to terminal. Will default to `False`.
-
-#### debug_output
-
-(string, optional)
-
-Alternatively to the `debug` property, `debug_output` can be used to either print debug info to the terminal or to debug files only. This should either be set to `file` or `terminal`, and will default to None. Will be ignored if `debug` is set.
-
 #### history_enabled
 
 (boolean, optional)
@@ -268,14 +294,27 @@ The `metadata` property describes what metadata will be returned by the MCS Pyth
 - `level1`: Only returns the images (with depth maps but NOT object masks), camera info, and properties corresponding to the player themself (like head tilt or pose). No information about specific objects will be included.
 - `none`: Only returns the images (but no depth maps or object masks), camera info, and properties corresponding to the player themself (like head tilt or pose). No information about specific objects will be included.
 
-Otherwise, return the metadata for the visible and held objects.
+If no metadata level is set:
+- `default`: Fallback if no metadata level is specified. Only meant for use during development (evaluations will never be run this way). Includes metadata for visible and held objects in the scene, as well as camera info and properties corresponding to the player. Does not include depth maps or object masks.
 
 #### noise_enabled
 
 (boolean, optional)
 
 Whether to add random noise to the numerical amounts in movement and object interaction action parameters. Will default to `False`.
 
+#### save_debug_images
+
+(boolean, optional)
+
+Save RGB frames, depth masks, and object instance segmentation masks (if returned in the output by the chosen metadata tier) to image files on each step. Default: False
+
+#### save_debug_json
+
+(boolean, optional)
+
+Save AI2-THOR/Unity input, AI2-THOR/Unity output, and MCS StepMetadata output to JSON file on each step. Default: False
+
 #### seed
 
 (int, optional)
@@ -292,7 +331,7 @@ Desired screen width. If value given, it must be more than `450`. If none given,
 
 (boolean, optional)
 
-Save videos of the RGB frames, depth masks, object instance segmentation masks (if returned in the output by the chosen metadata tier), 2D top-down scene views, and the heatmap images given to us in `make_step_prediction` by the AI performer.
+Create and save videos of the RGB frames, depth masks, object instance segmentation masks (if returned in the output by the chosen metadata tier), 2D top-down scene views, and the heatmap images given to us in `make_step_prediction` by the AI performer. Default: False
 
 ### Using the Config File to Generate Scene Graphs or Maps
 

integration_tests/config_level1.ini

@@ -1,4 +1,3 @@
 [MCS]
-debug: false
 history_enabled: false
 metadata: level1

integration_tests/config_level2.ini

@@ -1,4 +1,3 @@
 [MCS]
-debug: false
 history_enabled: false
 metadata: level2

integration_tests/config_oracle.ini

@@ -1,4 +1,3 @@
 [MCS]
-debug: false
 history_enabled: false
 metadata: oracle