diff --git a/Dockerfile b/Dockerfile index 58e19e3ae..5c7a5770e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,16 +15,17 @@ FROM nvidia/cudagl:10.1-base-ubuntu18.04 ARG DEBIAN_FRONTEND=noninteractive ENV NVIDIA_DRIVER_CAPABILITIES ${NVIDIA_DRIVER_CAPABILITIES},display +#ENV LANG C.UTF-8 # --build-arg mcsversion=0.0.x to override default in docker build command -ARG mcsversion=0.3.3 +ARG mcsversion=0.3.5 WORKDIR /mcs RUN apt-get update -y && \ apt-get install -y git python3.6 python3-pip mesa-utils && \ python3.6 -m pip install --upgrade pip setuptools wheel && \ - python3.6 -m pip install git+https://github.com/NextCenturyCorporation/MCS@${mcsversion} + python3.6 -m pip install git+https://github.com/NextCenturyCorporation/MCS@${mcsversion}#egg=machine_common_sense # add ai2thor/unity resources ADD https://github.com/NextCenturyCorporation/MCS/releases/download/${mcsversion}/MCS-AI2-THOR-Unity-App-v${mcsversion}.x86_64 /mcs diff --git a/README.md b/README.md index b1c06c2ba..58e6d5f4f 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ ## Installation -The latest release of the MCS Python library is `0.3.3`. +The latest release of the MCS Python library is `0.3.5`. ### Virtual Environments @@ -34,30 +34,44 @@ Here are the instructions for downloading and installing our latest Unity releas ### Unity Application -The latest release of the MCS Unity app is `0.3.3`. +The latest release of the MCS Unity app is `0.3.5`. Please note that our Unity App is built on Linux. If you need a Mac or Windows version, please [contact us](#troubleshooting) directly. -1. [Download the Latest MCS Unity App](https://github.com/NextCenturyCorporation/MCS/releases/download/0.3.3/MCS-AI2-THOR-Unity-App-v0.3.3.x86_64) +1. [Download the Latest MCS Unity App](https://github.com/NextCenturyCorporation/MCS/releases/download/0.3.5/MCS-AI2-THOR-Unity-App-v0.3.5.x86_64) -2. [Download the Latest MCS Unity Data Directory TAR](https://github.com/NextCenturyCorporation/MCS/releases/download/0.3.3/MCS-AI2-THOR-Unity-App-v0.3.3_Data.tar.gz) +2. [Download the Latest MCS Unity Data Directory TAR](https://github.com/NextCenturyCorporation/MCS/releases/download/0.3.5/MCS-AI2-THOR-Unity-App-v0.3.5_Data.tar.gz) 3. Ensure that both the Unity App and the TAR are in the same directory. 4. Untar the Data Directory: ``` -tar -xzvf MCS-AI2-THOR-Unity-App-v0.3.3_Data.tar.gz +tar -xzvf MCS-AI2-THOR-Unity-App-v0.3.5_Data.tar.gz ``` 5. Mark the Unity App as executable: ``` -chmod a+x MCS-AI2-THOR-Unity-App-v0.3.3.x86_64 +chmod a+x MCS-AI2-THOR-Unity-App-v0.3.5.x86_64 ``` ### Training Datasets +#### Winter 2020 + +*Please use the most recent 0.3.X release version* + +Passive Agent (only expected scenes): +- https://evaluation-training-scenes.s3.amazonaws.com/eval3/training-single-object.zip +- https://evaluation-training-scenes.s3.amazonaws.com/eval3/training-object-preference.zip + +Passive Intuitive Physics (only plausible scenes): +- https://evaluation-training-scenes.s3.amazonaws.com/eval3/training-passive-physics.zip + +Example Scenes: +- https://github.com/NextCenturyCorporation/MCS/tree/master/machine_common_sense/scenes + #### Summer 2020 *Use a release version between 0.0.9 and 0.1.0* @@ -162,8 +176,8 @@ To use an MCS configuration file, set the `MCS_CONFIG_FILE_PATH` environment var The `metadata` property describes what metadata will be returned by the MCS Python library. The `metadata` property is available so that users can run baseline or ablation studies during training. It can be set to one of the following strings: - `oracle`: Returns the metadata for all the objects in the scene, including visible, held, and hidden objects. Object masks will have consistent colors throughout all steps for a scene. -- `level2`: Only returns the images (with depth masks AND object masks), camera info, and properties corresponding to the player themself (like head tilt or pose). No information about specific objects will be included. Note that here, object masks will have randomized colors per step. -- `level1`: Only returns the images (with depth masks but NOT object masks), camera info, and properties corresponding to the player themself (like head tilt or pose). No information about specific objects will be included. +- `level2`: Only returns the images (with depth maps AND object masks), camera info, and properties corresponding to the player themself (like head tilt or pose). No information about specific objects will be included. Note that here, object masks will have randomized colors per step. +- `level1`: Only returns the images (with depth maps but NOT object masks), camera info, and properties corresponding to the player themself (like head tilt or pose). No information about specific objects will be included. Otherwise, return the metadata for the visible and held objects. diff --git a/machine_common_sense/API.md b/machine_common_sense/API.md index 782e76c52..bafa0e097 100644 --- a/machine_common_sense/API.md +++ b/machine_common_sense/API.md @@ -85,7 +85,7 @@ Make a prediction on the previously taken step/action. * **Parameters** - * **choice** (*string**, **optional*) – The selected choice required by the end of scenes with + * **choice** (*string**, **optional*) – The selected choice for per frame prediction with violation-of-expectation or classification goals. Is not required for other goals. (default None) diff --git a/machine_common_sense/controller.py b/machine_common_sense/controller.py index 1a025285a..55b6ddd77 100644 --- a/machine_common_sense/controller.py +++ b/machine_common_sense/controller.py @@ -161,6 +161,7 @@ class Controller(): CONFIG_S3_BUCKET = 's3_bucket' CONFIG_S3_FOLDER = 's3_folder' CONFIG_TEAM = 'team' + CONFIG_EVALUATION_NAME = 'evaluation_name' def __init__(self, unity_app_file_path, debug=False, enable_noise=False, seed=None, size=None, @@ -305,6 +306,7 @@ def _create_video_recorders(self): '''Create video recorders used to capture evaluation scenes for review ''' output_folder = pathlib.Path(self.__output_folder) + eval_name = self._config.get(self.CONFIG_EVALUATION_NAME, '') team = self._config.get(self.CONFIG_TEAM, '') scene_name = self.__scene_configuration.get( 'name', '').replace('json', '') @@ -314,7 +316,8 @@ def _create_video_recorders(self): timestamp = self.generate_time() basename_template = '_'.join( - [team, scene_name, self.PLACEHOLDER, timestamp]) + '.mp4' + [eval_name, self._metadata_tier, team, scene_name, + self.PLACEHOLDER, timestamp]) + '.mp4' visual_video_filename = basename_template.replace( self.PLACEHOLDER, self.VISUAL) @@ -393,7 +396,11 @@ def end_scene(self, choice, confidence=1.0): self.__uploader.upload_history( history_path=self.__history_writer.scene_history_file, s3_filename=(folder_prefix + '/' + - self._config[self.CONFIG_TEAM] + + self._config.get( + self.CONFIG_EVALUATION_NAME, '' + ) + + '_' + self._metadata_tier + + '_' + self._config.get(self.CONFIG_TEAM, '') + '_' + history_filename) ) @@ -459,9 +466,21 @@ def start_scene(self, config_data): # Ensure the previous scene history writer has saved its file. if self.__history_writer: self.__history_writer.check_file_written() + + hist_info = {} + hist_info[self.CONFIG_EVALUATION_NAME] = self._config.get( + self.CONFIG_EVALUATION_NAME, '' + ) + hist_info[self.CONFIG_EVALUATION] = self._config.get( + self.CONFIG_EVALUATION, False + ) + hist_info[self.CONFIG_METADATA_TIER] = self._metadata_tier + hist_info[self.CONFIG_TEAM] = self._config.get( + self.CONFIG_TEAM, '' + ) # Create a new scene history writer with each new scene (config # data) so we always create a new, separate scene history file. - self.__history_writer = HistoryWriter(config_data) + self.__history_writer = HistoryWriter(config_data, hist_info) skip_preview_phase = (True if 'goal' in config_data and 'skip_preview_phase' in config_data['goal'] @@ -500,6 +519,8 @@ def start_scene(self, config_data): output = self.restrict_step_output_metadata(pre_restrict_output) + self.write_debug_output(output) + if not skip_preview_phase: if (self._goal is not None and self._goal.last_preview_phase_step > 0): @@ -740,6 +761,8 @@ def step(self, action: str, **kwargs) -> StepMetadata: output = self.restrict_step_output_metadata(pre_restrict_output) + self.write_debug_output(output) + return output def make_step_prediction(self, choice: str = None, @@ -752,7 +775,7 @@ def make_step_prediction(self, choice: str = None, Parameters ---------- choice : string, optional - The selected choice required by the end of scenes with + The selected choice for per frame prediction with violation-of-expectation or classification goals. Is not required for other goals. (default None) confidence : float, optional @@ -1191,6 +1214,9 @@ def wrap_output(self, scene_event): self.__head_tilt = step_output.head_tilt + return step_output + + def write_debug_output(self, step_output): if self.__debug_to_terminal: print("RETURN STATUS: " + step_output.return_status) print("REWARD: " + str(step_output.reward)) @@ -1210,8 +1236,6 @@ def wrap_output(self, scene_event): str(self.__step_number) + '.json', 'w') as json_file: json_file.write(str(step_output)) - return step_output - def wrap_step(self, **kwargs): # whether or not to randomize segmentation mask colors consistentColors = False diff --git a/machine_common_sense/history_writer.py b/machine_common_sense/history_writer.py index 039b8a705..fa1f30b7b 100644 --- a/machine_common_sense/history_writer.py +++ b/machine_common_sense/history_writer.py @@ -7,8 +7,8 @@ class HistoryWriter(object): - def __init__(self, scene_config_data=None): - self.info_obj = {} + def __init__(self, scene_config_data=None, hist_info={}): + self.info_obj = hist_info self.current_steps = [] self.end_score = {} self.scene_history_file = None @@ -27,14 +27,17 @@ def __init__(self, scene_config_data=None): if not os.path.exists(prefix_directory): os.makedirs(prefix_directory) + timestamp = self.generate_time() + if ('screenshot' not in scene_config_data or not scene_config_data['screenshot']): self.scene_history_file = os.path.join( self.HISTORY_DIRECTORY, scene_config_data['name'].replace( - '.json', '') + "-" + self.generate_time() + ".json") + '.json', '') + "-" + timestamp + ".json") self.info_obj['name'] = scene_config_data['name'].replace( '.json', '') + self.info_obj['timestamp'] = timestamp def generate_time(self): return datetime.datetime.now().strftime("%Y%m%d-%H%M%S") diff --git a/machine_common_sense/scenes/ball_close.json b/machine_common_sense/scenes/ball_close.json index 386c0d93c..41db81e2b 100644 --- a/machine_common_sense/scenes/ball_close.json +++ b/machine_common_sense/scenes/ball_close.json @@ -1,13 +1,23 @@ { "name": "ball_close", "version": 2, - "ceilingMaterial": "Walls/WallDrywallWhite", - "floorMaterial": "Fabrics/RUG4", - "wallMaterial": "Walls/YellowDrywall", + "ceilingMaterial": "AI2-THOR/Materials/Walls/WallDrywallWhite", + "floorMaterial": "AI2-THOR/Materials/Fabrics/RUG4", + "wallMaterial": "AI2-THOR/Materials/Walls/YellowDrywall", + "performerStart": { + "position": { + "x": 0, + "z": 0 + }, + "rotation": { + "x": 0, + "y": 0 + } + }, "objects": [{ "id": "testBall", "type": "sphere", - "materialFile": "Plastics/BlueRubber", + "materials": ["AI2-THOR/Materials/Plastics/BlueRubber"], "pickupable": true, "shows": [{ "stepBegin": 0, diff --git a/machine_common_sense/scenes/ball_far.json b/machine_common_sense/scenes/ball_far.json index 35f0cffc9..42fb41bb0 100644 --- a/machine_common_sense/scenes/ball_far.json +++ b/machine_common_sense/scenes/ball_far.json @@ -1,13 +1,23 @@ { "name": "ball_far", "version": 2, - "ceilingMaterial": "Walls/WallDrywallWhite", - "floorMaterial": "Fabrics/RUG4", - "wallMaterial": "Walls/YellowDrywall", + "ceilingMaterial": "AI2-THOR/Materials/Walls/WallDrywallWhite", + "floorMaterial": "AI2-THOR/Materials/Fabrics/RUG4", + "wallMaterial": "AI2-THOR/Materials/Walls/YellowDrywall", + "performerStart": { + "position": { + "x": 0, + "z": 0 + }, + "rotation": { + "x": 0, + "y": 0 + } + }, "objects": [{ "id": "testBall", "type": "sphere", - "materialFile": "Plastics/BlueRubber", + "materials": ["AI2-THOR/Materials/Plastics/BlueRubber"], "pickupable": true, "shows": [{ "stepBegin": 0, diff --git a/machine_common_sense/scenes/ball_obstructed.json b/machine_common_sense/scenes/ball_obstructed.json index 5a41ec895..4eff97110 100644 --- a/machine_common_sense/scenes/ball_obstructed.json +++ b/machine_common_sense/scenes/ball_obstructed.json @@ -1,14 +1,24 @@ { "name": "ball_obstructed", "version": 2, - "ceilingMaterial": "Walls/WallDrywallWhite", - "floorMaterial": "Fabrics/RUG4", - "wallMaterial": "Walls/YellowDrywall", + "ceilingMaterial": "AI2-THOR/Materials/Walls/WallDrywallWhite", + "floorMaterial": "AI2-THOR/Materials/Fabrics/RUG4", + "wallMaterial": "AI2-THOR/Materials/Walls/YellowDrywall", + "performerStart": { + "position": { + "x": 0, + "z": 0 + }, + "rotation": { + "x": 0, + "y": 0 + } + }, "objects": [{ "id": "testWall", "type": "cube", "structure": true, - "materialFile": "SourceTextures/Materials/RedBricks", + "materials": ["AI2-THOR/Materials/Ceramics/RedBricks"], "shows": [{ "stepBegin": 0, "position": { @@ -25,7 +35,7 @@ }, { "id": "testBall", "type": "sphere", - "materialFile": "Plastics/BlueRubber", + "materials": ["AI2-THOR/Materials/Plastics/BlueRubber"], "pickupable": true, "shows": [{ "stepBegin": 0, diff --git a/machine_common_sense/scenes/block_close.json b/machine_common_sense/scenes/block_close.json index c073adf35..d10349b2a 100644 --- a/machine_common_sense/scenes/block_close.json +++ b/machine_common_sense/scenes/block_close.json @@ -1,13 +1,23 @@ { "name": "block_close", "version": 2, - "ceilingMaterial": "Walls/WallDrywallWhite", - "floorMaterial": "Fabrics/RUG4", - "wallMaterial": "Walls/YellowDrywall", + "ceilingMaterial": "AI2-THOR/Materials/Walls/WallDrywallWhite", + "floorMaterial": "AI2-THOR/Materials/Fabrics/RUG4", + "wallMaterial": "AI2-THOR/Materials/Walls/YellowDrywall", + "performerStart": { + "position": { + "x": 0, + "z": 0 + }, + "rotation": { + "x": 0, + "y": 0 + } + }, "objects": [{ "id": "testBlock", "type": "cube", - "materialFile": "Plastics/GreenPlastic", + "materials": ["AI2-THOR/Materials/Wood/WornWood"], "physics": true, "shows": [{ "stepBegin": 0, diff --git a/machine_common_sense/scenes/empty.json b/machine_common_sense/scenes/empty.json new file mode 100644 index 000000000..6f8b46df4 --- /dev/null +++ b/machine_common_sense/scenes/empty.json @@ -0,0 +1,18 @@ +{ + "name": "empty", + "version": 2, + "ceilingMaterial": "AI2-THOR/Materials/Walls/WallDrywallWhite", + "floorMaterial": "AI2-THOR/Materials/Fabrics/RUG4", + "wallMaterial": "AI2-THOR/Materials/Walls/YellowDrywall", + "performerStart": { + "position": { + "x": 0, + "z": 0 + }, + "rotation": { + "x": 0, + "y": 0 + } + }, + "objects": [] +} diff --git a/setup.py b/setup.py index 2f691eede..0fe3849da 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setuptools.setup( name='machine_common_sense', - version='0.3.3', + version='0.3.5', maintainer='Next Century, a wholly owned subsidiary of CACI', maintainer_email='mcs-ta2@machinecommonsense.com', url='https://github.com/NextCenturyCorporation/MCS/', @@ -28,7 +28,7 @@ 'matplotlib>=3.3', ('ai2thor @ ' 'git+https://github.com/NextCenturyCorporation/ai2thor' - '@0.3.3#egg=ai2thor') + '@0.3.5#egg=ai2thor') ], entry_points={ 'console_scripts': [ diff --git a/tests/test_history_writer.py b/tests/test_history_writer.py index 4a3278116..c24ef03e6 100644 --- a/tests/test_history_writer.py +++ b/tests/test_history_writer.py @@ -10,9 +10,28 @@ def test_init(self): config_data = {"name": "test_scene_file.json"} writer = mcs.HistoryWriter(config_data) + self.assertEqual(writer.info_obj.keys(), {'name', 'timestamp'}) self.assertEqual(writer.info_obj['name'], "test_scene_file") self.assertTrue(os.path.exists(writer.HISTORY_DIRECTORY)) + def test_init_with_hist_info(self): + config_data = {"name": "test_scene_file.json"} + writer = mcs.HistoryWriter(config_data, { + 'team': 'team1', + 'metadata': 'level1' + }) + + self.assertEqual(writer.info_obj.keys(), { + 'team', + 'metadata', + 'name', + 'timestamp' + }) + self.assertEqual(writer.info_obj['name'], "test_scene_file") + self.assertEqual(writer.info_obj['team'], "team1") + self.assertEqual(writer.info_obj['metadata'], "level1") + self.assertTrue(os.path.exists(writer.HISTORY_DIRECTORY)) + def test_add_step(self): config_data = {"name": "test_scene_file.json"} writer = mcs.HistoryWriter(config_data)