Merge pull request #216 from NextCenturyCorporation/development

0.3.5 Release

Dockerfile

@@ -15,16 +15,17 @@ FROM nvidia/cudagl:10.1-base-ubuntu18.04
 ARG DEBIAN_FRONTEND=noninteractive
 
 ENV NVIDIA_DRIVER_CAPABILITIES ${NVIDIA_DRIVER_CAPABILITIES},display
+#ENV LANG C.UTF-8
 
 # --build-arg mcsversion=0.0.x to override default in docker build command
-ARG mcsversion=0.3.3
+ARG mcsversion=0.3.5
 
 WORKDIR /mcs
 
 RUN apt-get update -y && \
     apt-get install -y git python3.6 python3-pip mesa-utils && \
     python3.6 -m pip install --upgrade pip setuptools wheel && \
-    python3.6 -m pip install git+https://github.com/NextCenturyCorporation/MCS@${mcsversion}
+    python3.6 -m pip install git+https://github.com/NextCenturyCorporation/MCS@${mcsversion}#egg=machine_common_sense
 
 # add ai2thor/unity resources
 ADD https://github.com/NextCenturyCorporation/MCS/releases/download/${mcsversion}/MCS-AI2-THOR-Unity-App-v${mcsversion}.x86_64 /mcs

README.md

@@ -2,7 +2,7 @@
 
 ## Installation
 
-The latest release of the MCS Python library is `0.3.3`.
+The latest release of the MCS Python library is `0.3.5`.
 
 ### Virtual Environments
 
@@ -34,30 +34,44 @@ Here are the instructions for downloading and installing our latest Unity releas
 
 ### Unity Application
 
-The latest release of the MCS Unity app is `0.3.3`.
+The latest release of the MCS Unity app is `0.3.5`.
 
 Please note that our Unity App is built on Linux. If you need a Mac or Windows version, please [contact us](#troubleshooting) directly.
 
-1. [Download the Latest MCS Unity App](https://github.com/NextCenturyCorporation/MCS/releases/download/0.3.3/MCS-AI2-THOR-Unity-App-v0.3.3.x86_64)
+1. [Download the Latest MCS Unity App](https://github.com/NextCenturyCorporation/MCS/releases/download/0.3.5/MCS-AI2-THOR-Unity-App-v0.3.5.x86_64)
 
-2. [Download the Latest MCS Unity Data Directory TAR](https://github.com/NextCenturyCorporation/MCS/releases/download/0.3.3/MCS-AI2-THOR-Unity-App-v0.3.3_Data.tar.gz)
+2. [Download the Latest MCS Unity Data Directory TAR](https://github.com/NextCenturyCorporation/MCS/releases/download/0.3.5/MCS-AI2-THOR-Unity-App-v0.3.5_Data.tar.gz)
 
 3. Ensure that both the Unity App and the TAR are in the same directory.
 
 4. Untar the Data Directory:
 
 ```
-tar -xzvf MCS-AI2-THOR-Unity-App-v0.3.3_Data.tar.gz
+tar -xzvf MCS-AI2-THOR-Unity-App-v0.3.5_Data.tar.gz
 ```
 
 5. Mark the Unity App as executable:
 
 ```
-chmod a+x MCS-AI2-THOR-Unity-App-v0.3.3.x86_64
+chmod a+x MCS-AI2-THOR-Unity-App-v0.3.5.x86_64
 ```
 
 ### Training Datasets
 
+#### Winter 2020
+
+*Please use the most recent 0.3.X release version*
+
+Passive Agent (only expected scenes):
+- https://evaluation-training-scenes.s3.amazonaws.com/eval3/training-single-object.zip
+- https://evaluation-training-scenes.s3.amazonaws.com/eval3/training-object-preference.zip
+
+Passive Intuitive Physics (only plausible scenes):
+- https://evaluation-training-scenes.s3.amazonaws.com/eval3/training-passive-physics.zip
+
+Example Scenes:
+- https://github.com/NextCenturyCorporation/MCS/tree/master/machine_common_sense/scenes
+
 #### Summer 2020
 
 *Use a release version between 0.0.9 and 0.1.0*
@@ -162,8 +176,8 @@ To use an MCS configuration file, set the `MCS_CONFIG_FILE_PATH` environment var
 The `metadata` property describes what metadata will be returned by the MCS Python library. The `metadata` property is available so that users can run baseline or ablation studies during training. It can be set to one of the following strings:
 
 - `oracle`: Returns the metadata for all the objects in the scene, including visible, held, and hidden objects. Object masks will have consistent colors throughout all steps for a scene.
-- `level2`: Only returns the images (with depth masks AND object masks), camera info, and properties corresponding to the player themself (like head tilt or pose). No information about specific objects will be included. Note that here, object masks will have randomized colors per step.
-- `level1`: Only returns the images (with depth masks but NOT object masks), camera info, and properties corresponding to the player themself (like head tilt or pose). No information about specific objects will be included.
+- `level2`: Only returns the images (with depth maps AND object masks), camera info, and properties corresponding to the player themself (like head tilt or pose). No information about specific objects will be included. Note that here, object masks will have randomized colors per step.
+- `level1`: Only returns the images (with depth maps but NOT object masks), camera info, and properties corresponding to the player themself (like head tilt or pose). No information about specific objects will be included.
 
 Otherwise, return the metadata for the visible and held objects.
 

machine_common_sense/API.md

@@ -85,7 +85,7 @@ Make a prediction on the previously taken step/action.
 * **Parameters**
 
 
-    * **choice** (*string**, **optional*) – The selected choice required by the end of scenes with
+    * **choice** (*string**, **optional*) – The selected choice for per frame prediction with
     violation-of-expectation or classification goals.
     Is not required for other goals. (default None)
 

machine_common_sense/controller.py

@@ -161,6 +161,7 @@ class Controller():
     CONFIG_S3_BUCKET = 's3_bucket'
     CONFIG_S3_FOLDER = 's3_folder'
     CONFIG_TEAM = 'team'
+    CONFIG_EVALUATION_NAME = 'evaluation_name'
 
     def __init__(self, unity_app_file_path, debug=False,
                  enable_noise=False, seed=None, size=None,
@@ -305,6 +306,7 @@ def _create_video_recorders(self):
         '''Create video recorders used to capture evaluation scenes for review
         '''
         output_folder = pathlib.Path(self.__output_folder)
+        eval_name = self._config.get(self.CONFIG_EVALUATION_NAME, '')
         team = self._config.get(self.CONFIG_TEAM, '')
         scene_name = self.__scene_configuration.get(
             'name', '').replace('json', '')
@@ -314,7 +316,8 @@ def _create_video_recorders(self):
 
         timestamp = self.generate_time()
         basename_template = '_'.join(
-            [team, scene_name, self.PLACEHOLDER, timestamp]) + '.mp4'
+            [eval_name, self._metadata_tier, team, scene_name,
+             self.PLACEHOLDER, timestamp]) + '.mp4'
 
         visual_video_filename = basename_template.replace(
             self.PLACEHOLDER, self.VISUAL)
@@ -393,7 +396,11 @@ def end_scene(self, choice, confidence=1.0):
                 self.__uploader.upload_history(
                     history_path=self.__history_writer.scene_history_file,
                     s3_filename=(folder_prefix + '/' +
-                                 self._config[self.CONFIG_TEAM] +
+                                 self._config.get(
+                                     self.CONFIG_EVALUATION_NAME, ''
+                                 ) +
+                                 '_' + self._metadata_tier +
+                                 '_' + self._config.get(self.CONFIG_TEAM, '') +
                                  '_' + history_filename)
                 )
 
@@ -459,9 +466,21 @@ def start_scene(self, config_data):
             # Ensure the previous scene history writer has saved its file.
             if self.__history_writer:
                 self.__history_writer.check_file_written()
+
+            hist_info = {}
+            hist_info[self.CONFIG_EVALUATION_NAME] = self._config.get(
+                self.CONFIG_EVALUATION_NAME, ''
+            )
+            hist_info[self.CONFIG_EVALUATION] = self._config.get(
+                self.CONFIG_EVALUATION, False
+            )
+            hist_info[self.CONFIG_METADATA_TIER] = self._metadata_tier
+            hist_info[self.CONFIG_TEAM] = self._config.get(
+                self.CONFIG_TEAM, ''
+            )
             # Create a new scene history writer with each new scene (config
             # data) so we always create a new, separate scene history file.
-            self.__history_writer = HistoryWriter(config_data)
+            self.__history_writer = HistoryWriter(config_data, hist_info)
 
         skip_preview_phase = (True if 'goal' in config_data and
                               'skip_preview_phase' in config_data['goal']
@@ -500,6 +519,8 @@ def start_scene(self, config_data):
 
         output = self.restrict_step_output_metadata(pre_restrict_output)
 
+        self.write_debug_output(output)
+
         if not skip_preview_phase:
             if (self._goal is not None and
                     self._goal.last_preview_phase_step > 0):
@@ -740,6 +761,8 @@ def step(self, action: str, **kwargs) -> StepMetadata:
 
         output = self.restrict_step_output_metadata(pre_restrict_output)
 
+        self.write_debug_output(output)
+
         return output
 
     def make_step_prediction(self, choice: str = None,
@@ -752,7 +775,7 @@ def make_step_prediction(self, choice: str = None,
         Parameters
         ----------
         choice : string, optional
-            The selected choice required by the end of scenes with
+            The selected choice for per frame prediction with
             violation-of-expectation or classification goals.
             Is not required for other goals. (default None)
         confidence : float, optional
@@ -1191,6 +1214,9 @@ def wrap_output(self, scene_event):
 
         self.__head_tilt = step_output.head_tilt
 
+        return step_output
+
+    def write_debug_output(self, step_output):
         if self.__debug_to_terminal:
             print("RETURN STATUS: " + step_output.return_status)
             print("REWARD: " + str(step_output.reward))
@@ -1210,8 +1236,6 @@ def wrap_output(self, scene_event):
                       str(self.__step_number) + '.json', 'w') as json_file:
                 json_file.write(str(step_output))
 
-        return step_output
-
     def wrap_step(self, **kwargs):
         # whether or not to randomize segmentation mask colors
         consistentColors = False

machine_common_sense/history_writer.py

@@ -7,8 +7,8 @@
 
 
 class HistoryWriter(object):
-    def __init__(self, scene_config_data=None):
-        self.info_obj = {}
+    def __init__(self, scene_config_data=None, hist_info={}):
+        self.info_obj = hist_info
         self.current_steps = []
         self.end_score = {}
         self.scene_history_file = None
@@ -27,14 +27,17 @@ def __init__(self, scene_config_data=None):
             if not os.path.exists(prefix_directory):
                 os.makedirs(prefix_directory)
 
+        timestamp = self.generate_time()
+
         if ('screenshot' not in scene_config_data or
                 not scene_config_data['screenshot']):
             self.scene_history_file = os.path.join(
                 self.HISTORY_DIRECTORY, scene_config_data['name'].replace(
-                    '.json', '') + "-" + self.generate_time() + ".json")
+                    '.json', '') + "-" + timestamp + ".json")
 
         self.info_obj['name'] = scene_config_data['name'].replace(
             '.json', '')
+        self.info_obj['timestamp'] = timestamp
 
     def generate_time(self):
         return datetime.datetime.now().strftime("%Y%m%d-%H%M%S")

machine_common_sense/scenes/ball_close.json

@@ -1,13 +1,23 @@
 {
     "name": "ball_close",
     "version": 2,
-    "ceilingMaterial": "Walls/WallDrywallWhite",
-    "floorMaterial": "Fabrics/RUG4",
-    "wallMaterial": "Walls/YellowDrywall",
+    "ceilingMaterial": "AI2-THOR/Materials/Walls/WallDrywallWhite",
+    "floorMaterial": "AI2-THOR/Materials/Fabrics/RUG4",
+    "wallMaterial": "AI2-THOR/Materials/Walls/YellowDrywall",
+    "performerStart": {
+        "position": {
+            "x": 0,
+            "z": 0
+        },
+        "rotation": {
+            "x": 0,
+            "y": 0
+        }
+    },
     "objects": [{
         "id": "testBall",
         "type": "sphere",
-        "materialFile": "Plastics/BlueRubber",
+        "materials": ["AI2-THOR/Materials/Plastics/BlueRubber"],
         "pickupable": true,
         "shows": [{
             "stepBegin": 0,

machine_common_sense/scenes/ball_far.json

@@ -1,13 +1,23 @@
 {
     "name": "ball_far",
     "version": 2,
-    "ceilingMaterial": "Walls/WallDrywallWhite",
-    "floorMaterial": "Fabrics/RUG4",
-    "wallMaterial": "Walls/YellowDrywall",
+    "ceilingMaterial": "AI2-THOR/Materials/Walls/WallDrywallWhite",
+    "floorMaterial": "AI2-THOR/Materials/Fabrics/RUG4",
+    "wallMaterial": "AI2-THOR/Materials/Walls/YellowDrywall",
+    "performerStart": {
+        "position": {
+            "x": 0,
+            "z": 0
+        },
+        "rotation": {
+            "x": 0,
+            "y": 0
+        }
+    },
     "objects": [{
         "id": "testBall",
         "type": "sphere",
-        "materialFile": "Plastics/BlueRubber",
+        "materials": ["AI2-THOR/Materials/Plastics/BlueRubber"],
         "pickupable": true,
         "shows": [{
             "stepBegin": 0,

machine_common_sense/scenes/ball_obstructed.json

@@ -1,14 +1,24 @@
 {
     "name": "ball_obstructed",
     "version": 2,
-    "ceilingMaterial": "Walls/WallDrywallWhite",
-    "floorMaterial": "Fabrics/RUG4",
-    "wallMaterial": "Walls/YellowDrywall",
+    "ceilingMaterial": "AI2-THOR/Materials/Walls/WallDrywallWhite",
+    "floorMaterial": "AI2-THOR/Materials/Fabrics/RUG4",
+    "wallMaterial": "AI2-THOR/Materials/Walls/YellowDrywall",
+    "performerStart": {
+        "position": {
+            "x": 0,
+            "z": 0
+        },
+        "rotation": {
+            "x": 0,
+            "y": 0
+        }
+    },
     "objects": [{
         "id": "testWall",
         "type": "cube",
         "structure": true,
-        "materialFile": "SourceTextures/Materials/RedBricks",
+        "materials": ["AI2-THOR/Materials/Ceramics/RedBricks"],
         "shows": [{
             "stepBegin": 0,
             "position": {
@@ -25,7 +35,7 @@
     }, {
         "id": "testBall",
         "type": "sphere",
-        "materialFile": "Plastics/BlueRubber",
+        "materials": ["AI2-THOR/Materials/Plastics/BlueRubber"],
         "pickupable": true,
         "shows": [{
             "stepBegin": 0,

machine_common_sense/scenes/block_close.json

@@ -1,13 +1,23 @@
 {
     "name": "block_close",
     "version": 2,
-    "ceilingMaterial": "Walls/WallDrywallWhite",
-    "floorMaterial": "Fabrics/RUG4",
-    "wallMaterial": "Walls/YellowDrywall",
+    "ceilingMaterial": "AI2-THOR/Materials/Walls/WallDrywallWhite",
+    "floorMaterial": "AI2-THOR/Materials/Fabrics/RUG4",
+    "wallMaterial": "AI2-THOR/Materials/Walls/YellowDrywall",
+    "performerStart": {
+        "position": {
+            "x": 0,
+            "z": 0
+        },
+        "rotation": {
+            "x": 0,
+            "y": 0
+        }
+    },
     "objects": [{
         "id": "testBlock",
         "type": "cube",
-        "materialFile": "Plastics/GreenPlastic",
+        "materials": ["AI2-THOR/Materials/Wood/WornWood"],
         "physics": true,
         "shows": [{
             "stepBegin": 0,

machine_common_sense/scenes/empty.json

@@ -0,0 +1,18 @@
+{
+    "name": "empty",
+    "version": 2,
+    "ceilingMaterial": "AI2-THOR/Materials/Walls/WallDrywallWhite",
+    "floorMaterial": "AI2-THOR/Materials/Fabrics/RUG4",
+    "wallMaterial": "AI2-THOR/Materials/Walls/YellowDrywall",
+    "performerStart": {
+        "position": {
+            "x": 0,
+            "z": 0
+        },
+        "rotation": {
+            "x": 0,
+            "y": 0
+        }
+    },
+    "objects": []
+}

setup.py

@@ -5,7 +5,7 @@
 
 setuptools.setup(
     name='machine_common_sense',
-    version='0.3.3',
+    version='0.3.5',
     maintainer='Next Century, a wholly owned subsidiary of CACI',
     maintainer_email='[email protected]',
     url='https://github.com/NextCenturyCorporation/MCS/',
@@ -28,7 +28,7 @@
         'matplotlib>=3.3',
         ('ai2thor @ '
          'git+https://github.com/NextCenturyCorporation/ai2thor'
-         '@0.3.3#egg=ai2thor')
+         '@0.3.5#egg=ai2thor')
     ],
     entry_points={
         'console_scripts': [