Widespread the use of pathlib.Path (#846)

- neurom.io.utils get_files_by_path and get_morph_files returns pathlib objects
- fix name == None when calling load_neuron with a pathlib object
- all unit tests use pathlib instead of os.path

apps/morph_check

@@ -32,11 +32,10 @@
 import argparse
 import json
 import logging
-import os
 import sys
+from pathlib import Path
 
 import pkg_resources
-
 from neurom.apps import get_config
 from neurom.check.runner import CheckRunner
 from neurom.exceptions import ConfigError
@@ -164,7 +163,7 @@ def main(args):
     _setup_logging(args.debug, args.log_file)
 
     try:
-        config = get_config(args.config, os.path.join(CONFIG_PATH, 'morph_check.yaml'))
+        config = get_config(args.config, Path(CONFIG_PATH, 'morph_check.yaml'))
         checker = CheckRunner(config)
     except ConfigError as e:
         L.error(str(e))

apps/raw_data_check

@@ -30,7 +30,7 @@
 
 """Examples of basic data checks."""
 import argparse
-import os
+from pathlib import Path
 import sys
 
 from neurom.check import structural_checks as st_chk
@@ -81,10 +81,10 @@ def parse_args():
 
 def run(args):
     """Check if all files have a soma points and sequential ids."""
-    data_path = args.datapath
-    if os.path.isfile(data_path):
+    data_path = Path(args.datapath)
+    if data_path.is_file():
         files = [data_path]
-    elif os.path.isdir(data_path):
+    elif data_path.is_dir():
         print('Checking files in directory', data_path)
         files = get_morph_files(data_path)
     else:

benchmarks/benchmarks.py

@@ -1,32 +1,31 @@
-import os
+from pathlib import Path
 
 import neurom as nm
 import neurom.io
 import neurom.fst._core
 from neurom.check import neuron_checks as nc
 from neurom.check import structural_checks as sc
 
-DATA_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)),
-                        '../test_data/')
+DATA_DIR = Path(__file__).parent.parent / 'test_data/'
 
 
 class TimeLoadMorphology(object):
     def time_swc(self):
-        path = os.path.join(DATA_DIR, 'swc/Neuron.swc')
+        path = Path(DATA_DIR, 'swc/Neuron.swc')
         nm.load_neuron(path)
 
     def time_neurolucida_asc(self):
-        path = os.path.join(DATA_DIR, 'neurolucida/bio_neuron-000.asc')
+        path = Path(DATA_DIR, 'neurolucida/bio_neuron-000.asc')
         nm.load_neuron(path)
 
     def time_h5(self):
-        path = os.path.join(DATA_DIR, 'h5/v1/bio_neuron-000.h5')
+        path = Path(DATA_DIR, 'h5/v1/bio_neuron-000.h5')
         nm.load_neuron(path)
 
 
 class TimeFeatures(object):
     def setup(self):
-        path = os.path.join(DATA_DIR, 'h5/v1/bio_neuron-000.h5')
+        path = Path(DATA_DIR, 'h5/v1/bio_neuron-000.h5')
         self.neuron = nm.load_neuron(path)
 
     def time_total_length(self):
@@ -113,7 +112,7 @@ def time_sholl_frequency(self):
 
 class TimeChecks:
     def setup(self):
-        path = os.path.join(DATA_DIR, 'h5/v1/bio_neuron-000.h5')
+        path = Path(DATA_DIR, 'h5/v1/bio_neuron-000.h5')
         self.data_wrapper = neurom.io.load_data(path)
         self.neuron = neurom.fst._core.FstNeuron(self.data_wrapper)
 

examples/features_graph_table.py

@@ -29,6 +29,7 @@
 
 """Example for comparison of the same feature of multiple cells."""
 import argparse
+from pathlib import Path
 
 import pylab as pl
 import neurom as nm
@@ -106,8 +107,6 @@ def plot_feature(feature, cell):
 
 
 if __name__ == '__main__':
-    import os
-
     args = parse_args()
 
     for morph_file in get_morph_files(args.datapath):
@@ -116,5 +115,5 @@ def plot_feature(feature, cell):
         for _feature in args.features:
             f = plot_feature(_feature, nrn)
             figname = "{0}_{1}.eps".format(_feature, nrn.name)
-            f.savefig(os.path.join(args.odir, figname))
+            f.savefig(Path(args.odir, figname))
             pl.close(f)

examples/plot_somas.py

@@ -29,15 +29,15 @@
 
 """Load and view multiple somas."""
 
-import os
+from pathlib import Path
+
 from neurom import load_neuron
 import neurom.view.common as common
 import matplotlib.pyplot as plt
 import numpy as np
 
-_path = os.path.dirname(os.path.abspath(__file__))
-DATA_PATH = os.path.join(_path, '../test_data')
-SWC_PATH = os.path.join(DATA_PATH, 'swc')
+DATA_PATH = Path(__file__).parent.parent / 'test_data'
+SWC_PATH = Path(DATA_PATH, 'swc')
 
 
 def random_color():
@@ -56,7 +56,7 @@ def plot_somas(somas):
 
 if __name__ == '__main__':
     #  define set of files containing relevant neurons
-    file_nms = [os.path.join(SWC_PATH, file_nm) for file_nm in ['Soma_origin.swc',
+    file_nms = [Path(SWC_PATH, file_nm) for file_nm in ['Soma_origin.swc',
                                                                 'Soma_translated_1.swc',
                                                                 'Soma_translated_2.swc']]
 

neurom/apps/morph_stats.py

@@ -30,21 +30,20 @@
 import logging
 from collections import defaultdict
 from itertools import product
-import os
-import pkg_resources
-
+from pathlib import Path
 
 import numpy as np
 import pandas as pd
+import pkg_resources
 
 import neurom as nm
-from neurom.fst._core import FstNeuron
-from neurom.features import NEURONFEATURES, NEURITEFEATURES
 from neurom.exceptions import ConfigError
+from neurom.features import NEURITEFEATURES, NEURONFEATURES
+from neurom.fst._core import FstNeuron
 
 L = logging.getLogger(__name__)
 
-EXAMPLE_CONFIG = os.path.join(pkg_resources.resource_filename(
+EXAMPLE_CONFIG = Path(pkg_resources.resource_filename(
     'neurom', 'config'), 'morph_stats.yaml')
 
 

neurom/apps/tests/test_morph_stats.py

@@ -26,7 +26,7 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-import os
+from pathlib import Path
 
 import numpy as np
 from nose.tools import (assert_almost_equal, assert_equal,
@@ -40,9 +40,8 @@
 from neurom.features import NEURITEFEATURES, NEURONFEATURES
 
 
-_path = os.path.dirname(os.path.abspath(__file__))
-DATA_PATH = os.path.join(_path, '../../../test_data')
-SWC_PATH = os.path.join(DATA_PATH, 'swc')
+DATA_PATH = Path(__file__).parent.parent.parent.parent / 'test_data'
+SWC_PATH = DATA_PATH / 'swc'
 
 
 REF_CONFIG = {
@@ -131,7 +130,7 @@ def test_eval_stats_applies_numpy_function():
 
 
 def test_extract_stats_single_neuron():
-    nrn = nm.load_neuron(os.path.join(SWC_PATH, 'Neuron.swc'))
+    nrn = nm.load_neuron(Path(SWC_PATH, 'Neuron.swc'))
     res = ms.extract_stats(nrn, REF_CONFIG)
     assert_equal(set(res.keys()), set(REF_OUT.keys()))
     # Note: soma radius is calculated from the sphere that gives the area
@@ -146,19 +145,19 @@ def test_extract_stats_single_neuron():
 
 def test_extract_dataframe():
     # Vanilla test
-    nrns = nm.load_neurons([os.path.join(SWC_PATH, name)
+    nrns = nm.load_neurons([Path(SWC_PATH, name)
                             for name in ['Neuron.swc', 'simple.swc']])
     actual = ms.extract_dataframe(nrns, REF_CONFIG)
-    expected = pd.read_csv(os.path.join(DATA_PATH, 'extracted-stats.csv'), index_col=0)
+    expected = pd.read_csv(Path(DATA_PATH, 'extracted-stats.csv'), index_col=0)
     assert_frame_equal(actual, expected)
 
     # Test with a single neuron in the population
-    nrns = nm.load_neurons(os.path.join(SWC_PATH, 'Neuron.swc'))
+    nrns = nm.load_neurons(Path(SWC_PATH, 'Neuron.swc'))
     actual = ms.extract_dataframe(nrns, REF_CONFIG)
     assert_frame_equal(actual, expected[expected.name == 'Neuron'], check_dtype=False)
 
     # Test with a config without the 'neuron' key
-    nrns = nm.load_neurons([os.path.join(SWC_PATH, name)
+    nrns = nm.load_neurons([Path(SWC_PATH, name)
                             for name in ['Neuron.swc', 'simple.swc']])
     config = {'neurite': {'section_lengths': ['total']},
               'neurite_type': ['AXON', 'APICAL_DENDRITE', 'BASAL_DENDRITE', 'ALL']}
@@ -167,12 +166,12 @@ def test_extract_dataframe():
     assert_frame_equal(actual, expected)
 
     # Test with a FstNeuron argument
-    nrn = nm.load_neuron(os.path.join(SWC_PATH, 'Neuron.swc'))
+    nrn = nm.load_neuron(Path(SWC_PATH, 'Neuron.swc'))
     actual = ms.extract_dataframe(nrn, config)
     assert_frame_equal(actual, expected[expected.name == 'Neuron'], check_dtype=False)
 
     # Test with a List[FstNeuron] argument
-    nrns = [nm.load_neuron(os.path.join(SWC_PATH, name))
+    nrns = [nm.load_neuron(Path(SWC_PATH, name))
             for name in ['Neuron.swc', 'simple.swc']]
     actual = ms.extract_dataframe(nrns, config)
     assert_frame_equal(actual, expected)

neurom/apps/tests/tests.py

@@ -26,19 +26,17 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-import os
-
-from nose import tools as nt
+from pathlib import Path
 
 from neurom.apps import get_config
 from neurom.exceptions import ConfigError
+from nose import tools as nt
 
 
 def test_get_config():
     # get the default
 
-    test_yaml = os.path.abspath(os.path.join(os.path.dirname(__file__),
-                                             '../../config/morph_stats.yaml'))
+    test_yaml = Path(__file__).parent.parent.parent / 'config/morph_stats.yaml'
 
     expected = {'neurite': {'section_lengths': ['max', 'total'], 'section_volumes': ['total'], 'section_branch_orders': ['max']}, 'neurite_type': ['AXON', 'APICAL_DENDRITE', 'BASAL_DENDRITE', 'ALL'], 'neuron': {'soma_radii': ['mean']}}
 

neurom/check/runner.py

@@ -134,7 +134,7 @@ def _check_file(self, f):
         for m, s in full_summary.items():
             self._log_msg(m, s)
 
-        return full_result, {f: full_summary}
+        return full_result, {str(f): full_summary}
 
     def _log_msg(self, msg, ok):
         """Helper to log message to the right level."""

neurom/check/tests/test_morphtree.py

@@ -26,19 +26,19 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+from copy import deepcopy
 from io import StringIO
+from pathlib import Path
+
+import numpy as np
 from neurom import load_neuron
 from neurom.check import morphtree as mt
 from neurom.core import Neurite, NeuriteType, Section
 from neurom.core.dataformat import COLS
 from nose import tools as nt
-import numpy as np
-import os
-from copy import deepcopy
 
-_path = os.path.dirname(os.path.abspath(__file__))
-DATA_PATH = os.path.join(_path, '../../../test_data')
-SWC_PATH = os.path.join(DATA_PATH, 'swc')
+DATA_PATH = Path(__file__).parent.parent.parent.parent / 'test_data'
+SWC_PATH = DATA_PATH / 'swc'
 
 
 def _make_flat(neuron):
@@ -158,7 +158,7 @@ def test_is_monotonic():
 
 def test_is_flat():
 
-    neu_tree = load_neuron(os.path.join(SWC_PATH, 'Neuron.swc')).neurites[0]
+    neu_tree = load_neuron(Path(SWC_PATH, 'Neuron.swc')).neurites[0]
 
     nt.assert_false(mt.is_flat(neu_tree, 1e-6, method='tolerance'))
     nt.assert_false(mt.is_flat(neu_tree, 0.1, method='ratio'))
@@ -191,7 +191,7 @@ def test_is_back_tracking():
 
 def test_get_flat_neurites():
 
-    n = load_neuron(os.path.join(SWC_PATH, 'Neuron.swc'))
+    n = load_neuron(Path(SWC_PATH, 'Neuron.swc'))
 
     nt.assert_equal(len(mt.get_flat_neurites(n, 1e-6, method='tolerance')), 0)
     nt.assert_equal(len(mt.get_flat_neurites(n, 0.1, method='ratio')), 0)
@@ -204,7 +204,7 @@ def test_get_flat_neurites():
 
 def test_get_nonmonotonic_neurites():
 
-    n = load_neuron(os.path.join(SWC_PATH, 'Neuron.swc'))
+    n = load_neuron(Path(SWC_PATH, 'Neuron.swc'))
 
     nt.assert_equal(len(mt.get_nonmonotonic_neurites(n)), 4)
 
@@ -215,5 +215,5 @@ def test_get_nonmonotonic_neurites():
 
 def test_get_back_tracking_neurites():
 
-    n = load_neuron(os.path.join(SWC_PATH, 'Neuron.swc'))
+    n = load_neuron(Path(SWC_PATH, 'Neuron.swc'))
     nt.assert_equal(len(mt.get_back_tracking_neurites(n)), 4)

neurom/check/tests/test_neuron_checks.py

@@ -26,7 +26,7 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-import os
+from pathlib import Path
 from copy import deepcopy
 from io import StringIO
 
@@ -39,20 +39,19 @@
 from neurom.core.dataformat import COLS
 from neurom.core.types import dendrite_filter
 
-_path = os.path.dirname(os.path.abspath(__file__))
-DATA_PATH = os.path.join(_path, '../../../test_data')
-SWC_PATH = os.path.join(DATA_PATH, 'swc')
-ASC_PATH = os.path.join(DATA_PATH, 'neurolucida')
-H5V1_PATH = os.path.join(DATA_PATH, 'h5/v1')
+DATA_PATH = Path(__file__).parent.parent.parent.parent / 'test_data'
+SWC_PATH = Path(DATA_PATH, 'swc')
+ASC_PATH = Path(DATA_PATH, 'neurolucida')
+H5V1_PATH = Path(DATA_PATH, 'h5/v1')
 
 
 def _load_neuron(name):
     if name.endswith('.swc'):
-        path = os.path.join(SWC_PATH, name)
+        path = Path(SWC_PATH, name)
     elif name.endswith('.h5'):
-        path = os.path.join(H5V1_PATH, name)
+        path = Path(H5V1_PATH, name)
     else:
-        path = os.path.join(ASC_PATH, name)
+        path = Path(ASC_PATH, name)
     return name, load_neuron(path)
 
 
@@ -281,13 +280,13 @@ def test_nonzero_section_lengths_threshold():
 
 def test_has_nonzero_soma_radius():
 
-    nrn = load_neuron(os.path.join(SWC_PATH, 'Neuron.swc'))
+    nrn = load_neuron(Path(SWC_PATH, 'Neuron.swc'))
     nt.assert_true(nrn_chk.has_nonzero_soma_radius(nrn))
 
 
 def test_has_nonzero_soma_radius_bad_data():
 
-    nrn = load_neuron(os.path.join(SWC_PATH, 'Single_basal.swc'))
+    nrn = load_neuron(Path(SWC_PATH, 'Single_basal.swc'))
     nt.assert_false(nrn_chk.has_nonzero_soma_radius(nrn).status)