Merge pull request #85 from Shreyanand/extend-pipeline

Fix Image env. for pipeline to work
os-climate · Nov 2, 2021 · 1aa312c · 1aa312c
2 parents c26a3d8 + f58142a
commit 1aa312c
Showing 1 changed file with 5 additions and 4 deletions.
diff --git a/src/data/s3_communication.py b/src/data/s3_communication.py
@@ -139,7 +139,8 @@ def download_files_in_prefix_to_dir(self, s3_prefix, destination_dir):
             # download files at the root of this prefix
             for file in result.get("Contents", []):
                 dest_filename = osp.basename(file.get("Key"))
-                dest_pathname = osp.join(destination_dir, dest_filename)
-                if not osp.exists(osp.dirname(dest_pathname)):
-                    os.makedirs(osp.dirname(dest_pathname))
-                self.download_file_from_s3(dest_pathname, s3_prefix, dest_filename)
+                if dest_filename:
+                    dest_pathname = osp.join(destination_dir, dest_filename)
+                    if not osp.exists(osp.dirname(dest_pathname)):
+                        os.makedirs(osp.dirname(dest_pathname))
+                    self.download_file_from_s3(dest_pathname, s3_prefix, dest_filename)