From f1fcc47d27df2acaadf8a2dbb0846e6aeff3b05f Mon Sep 17 00:00:00 2001 From: Niklas Siemer <70580458+niklassiemer@users.noreply.github.com> Date: Wed, 14 Apr 2021 20:33:20 +0200 Subject: [PATCH 1/7] Add _load_ipynb method returning nbformat.read of the notebook --- pyiron_base/generic/filedata.py | 43 ++++++++++++++++++++++++++------- pyiron_base/project/generic.py | 2 +- 2 files changed, 35 insertions(+), 10 deletions(-) diff --git a/pyiron_base/generic/filedata.py b/pyiron_base/generic/filedata.py index 18d045a28..361c6af55 100644 --- a/pyiron_base/generic/filedata.py +++ b/pyiron_base/generic/filedata.py @@ -8,7 +8,7 @@ import pandas from functools import lru_cache -from pyiron_base import ImportAlarm, FileHDFio +from pyiron_base import ImportAlarm, FileHDFio, ProjectHDFio __author__ = "Niklas Siemer" __copyright__ = ( @@ -26,28 +26,40 @@ try: from PIL import Image _has_imported['PIL'] = True - import_alarm = ImportAlarm() except ImportError: _has_imported['PIL'] = False +try: + import nbformat + _has_imported['nbformat'] = True +except ImportError: + _has_imported['nbformat'] = False + +if all(_has_imported.values()): + import_alarm = ImportAlarm() +else: import_alarm = ImportAlarm( - "Reduced functionality, since " + 'PIL' + " could not be imported." + "Reduced functionality, since " + + str([package for package in _has_imported.keys() if not _has_imported[package]]) + + " could not be imported." ) @import_alarm -def load_file(filename, filetype=None): +def load_file(filename, filetype=None, project=None): """ Load the file and return an appropriate object containing the data. Args: filename (str): path to the file to be displayed. filetype (str/None): File extension, if given this overwrites the assumption based on the filename. + project (pyiron-Project/None): Project calling this function, provided to all objects referring to such. Supported file types are: '.h5', '.hdf' '.json' '.txt' '.csv' + '.ipynb' Image extensions supported by PIL Returns: @@ -61,6 +73,9 @@ def _load_txt(file): with open(file, encoding='utf8') as f: return f.readlines() + def _load_ipynb(file): + return nbformat.read(file, as_version=4) + def _load_json(file): with open(file) as f: return json.load(f) @@ -83,17 +98,27 @@ def _load_default(file): filetype = '.' + filetype if filetype.lower() in ['.h5', '.hdf']: - return FileHDFio(file_name=filename) + if project is None: + return FileHDFio(file_name=filename) + else: + return ProjectHDFio(file_name=filename, project=project) if filetype.lower() in ['.json']: return _load_json(filename) elif filetype.lower() in ['.txt']: return _load_txt(filename) elif filetype.lower() in ['.csv']: return _load_csv(filename) - elif _has_imported['PIL'] and filetype.lower() in Image.registered_extensions(): - return _load_img(filename) - else: - return _load_default(filename) + try: + if filetype.lower() in ['.ipynb']: + return _load_ipynb(filename) + except NameError: + pass + try: + if filetype.lower() in Image.registered_extensions(): + return _load_img(filename) + except NameError: + pass + return _load_default(filename) class FileData: diff --git a/pyiron_base/project/generic.py b/pyiron_base/project/generic.py index f21eeb283..110523396 100644 --- a/pyiron_base/project/generic.py +++ b/pyiron_base/project/generic.py @@ -1441,7 +1441,7 @@ def _get_item_helper(self, item, convert_to_object=True): return ProjectHDFio(project=self, file_name=file_name) if item in self.list_files(): file_name = posixpath.join(self.path, "{}".format(item)) - return load_file(file_name) + return load_file(file_name, project=self) if item in self.list_dirs(): with self.open(item) as new_item: return new_item.copy() From a856b20222e3af0c55d8c9f05538991c0a3c0e87 Mon Sep 17 00:00:00 2001 From: Niklas Siemer <70580458+niklassiemer@users.noreply.github.com> Date: Wed, 5 May 2021 22:36:59 +0200 Subject: [PATCH 2/7] Use OwnNotebookNode(nbformat.NotebookNode) to add html representation --- pyiron_base/generic/filedata.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/pyiron_base/generic/filedata.py b/pyiron_base/generic/filedata.py index 361c6af55..6d64c7c2d 100644 --- a/pyiron_base/generic/filedata.py +++ b/pyiron_base/generic/filedata.py @@ -5,9 +5,10 @@ import json import os -import pandas from functools import lru_cache +import pandas + from pyiron_base import ImportAlarm, FileHDFio, ProjectHDFio __author__ = "Niklas Siemer" @@ -29,7 +30,7 @@ except ImportError: _has_imported['PIL'] = False try: - import nbformat + import nbformat, nbconvert _has_imported['nbformat'] = True except ImportError: _has_imported['nbformat'] = False @@ -44,6 +45,15 @@ ) +class OwnNotebookNode(nbformat.NotebookNode): + """Wrapper for nbformat.NotebookNode with some additional representation based on nbconvert.""" + def _repr_html_(self): + html_exporter = nbconvert.HTMLExporter() + html_exporter.template_name = "classic" + (html_output, resources) = html_exporter.from_notebook_node(self) + return html_output + + @import_alarm def load_file(filename, filetype=None, project=None): """ @@ -74,7 +84,7 @@ def _load_txt(file): return f.readlines() def _load_ipynb(file): - return nbformat.read(file, as_version=4) + return OwnNotebookNode(nbformat.read(file, as_version=4)) def _load_json(file): with open(file) as f: From 961fc54cdb88216501c5d6218f6bd10640239839 Mon Sep 17 00:00:00 2001 From: Niklas Siemer <70580458+niklassiemer@users.noreply.github.com> Date: Wed, 5 May 2021 23:02:04 +0200 Subject: [PATCH 3/7] wrap class definition in try/except --- pyiron_base/generic/filedata.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/pyiron_base/generic/filedata.py b/pyiron_base/generic/filedata.py index 6d64c7c2d..0ff7bf5f9 100644 --- a/pyiron_base/generic/filedata.py +++ b/pyiron_base/generic/filedata.py @@ -45,13 +45,17 @@ ) -class OwnNotebookNode(nbformat.NotebookNode): - """Wrapper for nbformat.NotebookNode with some additional representation based on nbconvert.""" - def _repr_html_(self): - html_exporter = nbconvert.HTMLExporter() - html_exporter.template_name = "classic" - (html_output, resources) = html_exporter.from_notebook_node(self) - return html_output +try: + class OwnNotebookNode(nbformat.NotebookNode): + """Wrapper for nbformat.NotebookNode with some additional representation based on nbconvert.""" + def _repr_html_(self): + html_exporter = nbconvert.HTMLExporter() + html_exporter.template_name = "classic" + (html_output, resources) = html_exporter.from_notebook_node(self) + return html_output + +except NameError: + pass @import_alarm From 3a554717d10c8c52333bba112081145feff44a47 Mon Sep 17 00:00:00 2001 From: Niklas Siemer <70580458+niklassiemer@users.noreply.github.com> Date: Thu, 6 May 2021 14:32:10 +0200 Subject: [PATCH 4/7] return to if _has_imported[package] instead of try/except NameError --- pyiron_base/generic/filedata.py | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/pyiron_base/generic/filedata.py b/pyiron_base/generic/filedata.py index 0ff7bf5f9..76b8d2f1d 100644 --- a/pyiron_base/generic/filedata.py +++ b/pyiron_base/generic/filedata.py @@ -45,7 +45,7 @@ ) -try: +if _has_imported['nbformat']: class OwnNotebookNode(nbformat.NotebookNode): """Wrapper for nbformat.NotebookNode with some additional representation based on nbconvert.""" def _repr_html_(self): @@ -54,9 +54,6 @@ def _repr_html_(self): (html_output, resources) = html_exporter.from_notebook_node(self) return html_output -except NameError: - pass - @import_alarm def load_file(filename, filetype=None, project=None): @@ -122,16 +119,10 @@ def _load_default(file): return _load_txt(filename) elif filetype.lower() in ['.csv']: return _load_csv(filename) - try: - if filetype.lower() in ['.ipynb']: - return _load_ipynb(filename) - except NameError: - pass - try: - if filetype.lower() in Image.registered_extensions(): - return _load_img(filename) - except NameError: - pass + elif _has_imported['nbformat'] and filetype.lower() in ['.ipynb']: + return _load_ipynb(filename) + elif _has_imported['PIL'] and filetype.lower() in Image.registered_extensions(): + return _load_img(filename) return _load_default(filename) From b9fb6557823c1594d51f19b183807a0457a69488 Mon Sep 17 00:00:00 2001 From: Niklas Siemer <70580458+niklassiemer@users.noreply.github.com> Date: Thu, 6 May 2021 22:06:33 +0200 Subject: [PATCH 5/7] Clear up code structure --- pyiron_base/generic/filedata.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/pyiron_base/generic/filedata.py b/pyiron_base/generic/filedata.py index 76b8d2f1d..a9681054e 100644 --- a/pyiron_base/generic/filedata.py +++ b/pyiron_base/generic/filedata.py @@ -107,23 +107,25 @@ def _load_default(file): _, filetype = os.path.splitext(filename) elif filetype[0] != '.': filetype = '.' + filetype + filetype = filetype.lower() - if filetype.lower() in ['.h5', '.hdf']: + if filetype in ['.h5', '.hdf']: if project is None: return FileHDFio(file_name=filename) else: return ProjectHDFio(file_name=filename, project=project) - if filetype.lower() in ['.json']: + elif filetype in ['.json']: return _load_json(filename) - elif filetype.lower() in ['.txt']: + elif filetype in ['.txt']: return _load_txt(filename) - elif filetype.lower() in ['.csv']: + elif filetype in ['.csv']: return _load_csv(filename) - elif _has_imported['nbformat'] and filetype.lower() in ['.ipynb']: + elif _has_imported['nbformat'] and filetype in ['.ipynb']: return _load_ipynb(filename) - elif _has_imported['PIL'] and filetype.lower() in Image.registered_extensions(): + elif _has_imported['PIL'] and filetype in Image.registered_extensions(): return _load_img(filename) - return _load_default(filename) + else: + return _load_default(filename) class FileData: From 14258453a46c5bde136c48a287a78ff5f38c1cf0 Mon Sep 17 00:00:00 2001 From: Niklas Siemer <70580458+niklassiemer@users.noreply.github.com> Date: Thu, 6 May 2021 23:44:15 +0200 Subject: [PATCH 6/7] Codacy --- pyiron_base/generic/filedata.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyiron_base/generic/filedata.py b/pyiron_base/generic/filedata.py index d3097617a..5b4a4809a 100644 --- a/pyiron_base/generic/filedata.py +++ b/pyiron_base/generic/filedata.py @@ -49,11 +49,12 @@ if _has_imported['nbformat']: class OwnNotebookNode(nbformat.NotebookNode): + """Wrapper for nbformat.NotebookNode with some additional representation based on nbconvert.""" def _repr_html_(self): html_exporter = nbconvert.HTMLExporter() html_exporter.template_name = "classic" - (html_output, resources) = html_exporter.from_notebook_node(self) + (html_output, _) = html_exporter.from_notebook_node(self) return html_output From 41f5980a34057c2b403f51db465b3122d58cdf27 Mon Sep 17 00:00:00 2001 From: Niklas Siemer <70580458+niklassiemer@users.noreply.github.com> Date: Fri, 7 May 2021 09:20:56 +0200 Subject: [PATCH 7/7] Add test for load_file into ProjectHDFio --- tests/generic/test_filedata.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/tests/generic/test_filedata.py b/tests/generic/test_filedata.py index 50f10fcf8..c30ca3d3a 100644 --- a/tests/generic/test_filedata.py +++ b/tests/generic/test_filedata.py @@ -1,10 +1,12 @@ -import unittest -import os import json +import os +import unittest + import pandas as pd +from pyiron_base import FileHDFio, ProjectHDFio from pyiron_base.generic.filedata import FileData, load_file -from pyiron_base import FileHDFio +from pyiron_base.project.generic import Project class TestLoadFile(unittest.TestCase): @@ -50,6 +52,13 @@ def test_load_file_hdf(self): self.assertIsInstance(hdf, FileHDFio) self.assertEqual(hdf['content/key'], 'value') + def test_load_file_ProjectHDF(self): + pr = Project(self.current_dir + '/test_pr') + pr_hdf = load_file(self.current_dir + '/test_data.h5', project=pr) + self.assertIsInstance(pr_hdf, ProjectHDFio) + self.assertEqual(pr_hdf['content/key'], 'value') + pr.remove(enable=True) + def test_load_file_default(self): """Test default load for text file and h5 file without extension.""" filename = self.current_dir + '/test_data'