Merge branch 'main' into issue/166/generalize-mixmod_pdf

pyproject.toml

@@ -44,6 +44,11 @@ dev = [
     "mpi4py",
     "coverage",
 ]
+full = [
+    "tables-io[full]",
+    "matplotlib",
+    "scikit-learn",
+]
 all = [
     "tables-io[full]",
     "matplotlib",

src/qp/__init__.py

@@ -12,7 +12,7 @@
 from .scipy_pdfs import *
 from .packed_interp_pdf import *
 from .ensemble import Ensemble
-from .factory import instance, add_class, create, read, read_metadata, convert, concatenate, iterator, data_length, from_tables
+from .factory import instance, add_class, create, read, read_metadata, convert, concatenate, iterator, data_length, from_tables, is_qp_file
 from .lazy_modules import *
 
 from . import utils

src/qp/factory.py

@@ -159,6 +159,29 @@ def read_metadata(self, filename):
         tables = io.read(filename, NUMPY_DICT, keys=['meta'])
         return tables["meta"]
 
+    def is_qp_file(self, filename):
+        """Test if a file is a qp file
+
+        Parameters
+        ----------
+        filename : `str`
+            File to test
+
+        Returns
+        -------
+        value : bool
+            True if the file is a qp file
+        """
+        try:
+            # If this isn't a table-like file with a 'meta' table this will throw an exception
+            tables = io.readNative(filename, keys=['meta'])
+            # If the 'meta' tables doesn't have 'pdf_name' or it is empty this will throw an exception or fail
+            return len(tables['meta']['pdf_name']) > 0
+        except Exception as msg:
+            # Any exception means it isn't a qp file
+            print(f"This is not a qp file because {msg}")
+        return False
+
     def read(self, filename):
         """Read this ensemble from a file
 
@@ -344,3 +367,4 @@ def instance():
 concatenate = _FACTORY.concatenate
 data_length = _FACTORY.data_length
 from_tables = _FACTORY.from_tables
+is_qp_file = _FACTORY.is_qp_file

src/qp/metrics/pit.py

@@ -46,6 +46,14 @@ def __init__(self, qp_ens, true_vals, eval_grid=DEFAULT_QUANTS):
         # For each distribution in the Ensemble, calculate the CDF where x = known_true_value
         self._pit_samps = np.array([qp_ens[i].cdf(self._true_vals[i])[0][0] for i in range(len(self._true_vals))])
 
+        # These two lines set all `NaN` values to 0. This may or may not make sense
+        # Alternatively if it's better to simply remove the `NaN`, this can be done
+        # efficiently on line 61 with `data_quants = np.nanquantile(...)`.`
+        samp_mask = np.isfinite(self._pit_samps)
+        self._pit_samps[~samp_mask] = 0
+        if not np.all(samp_mask):
+            logging.warning('Some PIT samples were `NaN`. They have been replacd with 0.')
+
         n_pit = np.min([len(self._pit_samps), len(eval_grid)])
         if n_pit < len(eval_grid):
             logging.warning('Number of pit samples is smaller than the evaluation grid size. '

src/qp/mixmod_pdf.py

@@ -51,7 +51,7 @@ def __init__(self, gen_func, weights, data, ancil=None, *args, **kwargs):
         stds:  array_like
             The standard deviations of the basis functions
         weights : array_like
-            The weights to attach to the basis functions. Weights should sum up to one. If not, the weights are interpreted as relative weights.
+            The weights to attach to the Gaussians. Weights should sum up to one. If not, the weights are interpreted as relative weights.
         """
         self._gen_func = gen_func
         self._frozen = self._gen_func(**data)

tests/qp/test_ensemble.py

@@ -213,5 +213,13 @@ def test_iterator(self):
             test_vals = ens_i.pdf(test_grid)
             assert np.allclose(check_vals, test_vals)
 
+    def test_mixmod_with_negative_weights(self):
+        """Verify that an exception is raised when setting up a mixture model with negative weights"""
+        means = np.array([0.5,1.1, 2.9])
+        sigmas = np.array([0.15,0.13,0.14])
+        weights = np.array([1,0.5,-0.25])
+        with self.assertRaises(ValueError):
+            _ = qp.mixmod(weights=weights, means=means, stds=sigmas)
+
 if __name__ == '__main__':
     unittest.main()

tests/qp/test_infrastructure.py

@@ -4,7 +4,7 @@
 import os
 import unittest
 import qp
-
+from qp.test_funcs import build_ensemble
 from qp import test_data
 
 
@@ -55,6 +55,14 @@ def test_get_val_or_default():
         test_dict.pop(None)
         assert qp.dict_utils.get_val_or_default(test_dict, 'nokey') is None
 
+    def test_is_qp_file(self):
+        fname = 'norm_ensemble.hdf5'
+        norm_test_data = qp.stats.norm_gen.test_data['norm']
+        ens_norm = build_ensemble(norm_test_data)
+        ens_norm.write_to(fname)
+        self.files.append(fname)
+        assert qp.instance().is_qp_file(fname)
+        assert not qp.instance().is_qp_file('test_pit.py')
 
 if __name__ == '__main__':
     unittest.main()