Merge pull request #254 from winedarksea/dev

0.6.19

TODO.md

@@ -13,14 +13,8 @@
 * Forecasts are desired for the future immediately following the most recent data.
 * trimmed_mean to AverageValueNaive
 
-# 0.6.18 🇺🇦 🇺🇦 🇺🇦
-* minor tweaks and bug fixes
-* updated python version required to 3.9 to allow more typing options
-* validation_indexes can now be a list of tuples with mixed forecast lengths
-* "mixed_length" validation option, for limited testing of longer forecast lengths
-* kalman method for ensemble aggregation (ala sensor fusion)
-* UpscaleDownscaleTransformer added
-* PreprocessingExperts model added (experimental)
+# 0.6.19 🇺🇦 🇺🇦 🇺🇦
+* bug fix for transformer_list="all"
 
 ### Unstable Upstream Pacakges (those that are frequently broken by maintainers)
 * Pytorch-Forecasting

autots/__init__.py

@@ -27,7 +27,7 @@
 from autots.models.cassandra import Cassandra
 
 
-__version__ = '0.6.18'
+__version__ = '0.6.19'
 
 TransformTS = GeneralTransformer
 

autots/evaluator/auto_model.py

@@ -2633,15 +2633,16 @@ def trans_dict_recomb(dict_array):
     """Recombine two transformation param dictionaries from array of dicts."""
     empty_trans = (None, {})
     a, b = random.sample(dict_array, 2)
-    na_choice = random.sample([a, b], 1)[0]['fillna']
+
+    na_choice = random.sample([a, b], 1)[0].get('fillna', None)
 
     a_result = [
         (a['transformations'][key], a['transformation_params'][key])
-        for key in sorted(a['transformations'].keys())
+        for key in sorted(a.get('transformations', {}).keys())
     ]
     b_result = [
         (b['transformations'][key], b['transformation_params'][key])
-        for key in sorted(b['transformations'].keys())
+        for key in sorted(b.get('transformations', {}).keys())
     ]
     combi = zip_longest(a_result, b_result, fillvalue=empty_trans)
     selected = [random.choice(x) for x in combi]

autots/evaluator/auto_ts.py

@@ -439,9 +439,9 @@ def __init__(
 
                 full_params['transformations'] = transformations
                 full_params['transformation_params'] = transformation_params
-                self.initial_template.loc[
-                    index, 'TransformationParameters'
-                ] = json.dumps(full_params)
+                self.initial_template.loc[index, 'TransformationParameters'] = (
+                    json.dumps(full_params)
+                )
 
         self.regressor_used = False
         self.subset_flag = False
@@ -1975,6 +1975,10 @@ def _run_template(
         additional_msg="",
     ):
         """Get results for one batch of models."""
+        # this fillna is the result of a as-of-yet untraced bug producing "null" transformation params
+        template["TransformationParameters"] = (
+            template["TransformationParameters"].replace("null", "{}").fillna('{}')
+        )
         model_count = self.model_count if model_count is None else model_count
         template_result = TemplateWizard(
             template,
@@ -2014,10 +2018,10 @@ def _run_template(
             self.model_count = template_result.model_count
         # capture results from lower-level template run
         if "TotalRuntime" in template_result.model_results.columns:
-            template_result.model_results[
-                'TotalRuntime'
-            ] = template_result.model_results['TotalRuntime'].fillna(
-                pd.Timedelta(seconds=60)
+            template_result.model_results['TotalRuntime'] = (
+                template_result.model_results['TotalRuntime'].fillna(
+                    pd.Timedelta(seconds=60)
+                )
             )
         else:
             # trying to catch a rare and sneaky bug (perhaps some variety of beetle?)
@@ -2143,9 +2147,9 @@ def _run_validations(
                         frac=0.8, random_state=self.random_seed
                     ).reindex(idx)
                 nan_frac = val_df_train.shape[1] / num_validations
-                val_df_train.iloc[
-                    -2:, int(nan_frac * y) : int(nan_frac * (y + 1))
-                ] = np.nan
+                val_df_train.iloc[-2:, int(nan_frac * y) : int(nan_frac * (y + 1))] = (
+                    np.nan
+                )
 
             # run validation template on current slice
             result = self._run_template(
@@ -4812,9 +4816,9 @@ def diagnose_params(self, target='runtime', waterfall_plots=True):
                     )
                     y = pd.json_normalize(json.loads(row["ModelParameters"]))
                     y.index = [row['ID']]
-                    y[
-                        'Model'
-                    ] = x  # might need to remove this and do analysis independently for each
+                    y['Model'] = (
+                        x  # might need to remove this and do analysis independently for each
+                    )
                     res.append(
                         pd.DataFrame(
                             {

autots/models/base.py

@@ -490,18 +490,18 @@ def long_form_results(
             value_name=value_name,
             id_vars="datetime",
         ).set_index("datetime")
-        upload_upper[
-            interval_name
-        ] = f"{round(100 - ((1- self.prediction_interval)/2) * 100, 0)}%"
+        upload_upper[interval_name] = (
+            f"{round(100 - ((1- self.prediction_interval)/2) * 100, 0)}%"
+        )
         upload_lower = pd.melt(
             self.lower_forecast.rename_axis(index='datetime').reset_index(),
             var_name=id_name,
             value_name=value_name,
             id_vars="datetime",
         ).set_index("datetime")
-        upload_lower[
-            interval_name
-        ] = f"{round(((1- self.prediction_interval)/2) * 100, 0)}%"
+        upload_lower[interval_name] = (
+            f"{round(((1- self.prediction_interval)/2) * 100, 0)}%"
+        )
 
         upload = pd.concat([upload, upload_upper, upload_lower], axis=0)
         if datetime_column is not None:

autots/models/ensemble.py

@@ -2340,15 +2340,15 @@ def _buildup_mosaics(
             f"Mosaic Ensemble failed on model {row[3]} series {row[2]} and period {row[1]} due to missing model: {e} "
             + mi
         ) from e
-    melted[
-        'forecast'
-    ] = fore  # [forecasts[row[3]][row[2]].iloc[row[1]] for row in melted.itertuples()]
-    melted[
-        'upper_forecast'
-    ] = u_fore  # [upper_forecasts[row[3]][row[2]].iloc[row[1]] for row in melted.itertuples()]
-    melted[
-        'lower_forecast'
-    ] = l_fore  # [lower_forecasts[row[3]][row[2]].iloc[row[1]] for row in melted.itertuples()]
+    melted['forecast'] = (
+        fore  # [forecasts[row[3]][row[2]].iloc[row[1]] for row in melted.itertuples()]
+    )
+    melted['upper_forecast'] = (
+        u_fore  # [upper_forecasts[row[3]][row[2]].iloc[row[1]] for row in melted.itertuples()]
+    )
+    melted['lower_forecast'] = (
+        l_fore  # [lower_forecasts[row[3]][row[2]].iloc[row[1]] for row in melted.itertuples()]
+    )
 
     forecast_df = melted.pivot(
         values="forecast", columns="series_id", index="forecast_period"

autots/models/sklearn.py

@@ -335,8 +335,7 @@ def rolling_x_regressor_regressor(
         X = X.set_index("series_id", append=True)
     if series_id is not None:
         hashed = (
-            int(hashlib.sha256(str(series_id).encode('utf-8')).hexdigest(), 16)
-            % 10**16
+            int(hashlib.sha256(str(series_id).encode('utf-8')).hexdigest(), 16) % 10**16
         )
         X['series_id'] = hashed
     return X
@@ -3985,9 +3984,7 @@ def _rbf_kernel(self, x1, x2, gamma):
         if gamma is None:
             gamma = 1.0 / x1.shape[1]
         distance = (
-            np.sum(x1**2, 1).reshape(-1, 1)
-            + np.sum(x2**2, 1)
-            - 2 * np.dot(x1, x2.T)
+            np.sum(x1**2, 1).reshape(-1, 1) + np.sum(x2**2, 1) - 2 * np.dot(x1, x2.T)
         )
         return np.exp(-gamma * distance)
 

autots/templates/general.py

@@ -446,7 +446,7 @@
     "71": {
         'Model': 'Cassandra',
         'ModelParameters': '''{"preprocessing_transformation": {"fillna": "mean", "transformations": {"0": "ClipOutliers", "1": "bkfilter"}, "transformation_params": {"0": {"method": "clip", "std_threshold": 3.5, "fillna": null}, "1": {}}}, "scaling": "BaseScaler", "past_impacts_intervention": null, "seasonalities": [7, 365.25], "ar_lags": [7], "ar_interaction_seasonality": null, "anomaly_detector_params": null, "anomaly_intervention": null, "holiday_detector_params": {"threshold": 0.9, "splash_threshold": null, "use_dayofmonth_holidays": true, "use_wkdom_holidays": true, "use_wkdeom_holidays": false, "use_lunar_holidays": false, "use_lunar_weekday": false, "use_islamic_holidays": false, "use_hebrew_holidays": true, "anomaly_detector_params": {"method": "mad", "method_params": {"distribution": "norm", "alpha": 0.05}, "fillna": "rolling_mean_24", "transform_dict": {"fillna": null, "transformations": {"0": "EWMAFilter"}, "transformation_params": {"0": {"span": 7}}}}, "remove_excess_anomalies": false, "impact": null, "regression_params": null, "output": "multivariate"}, "holiday_countries_used": true, "multivariate_feature": null, "multivariate_transformation": null, "regressor_transformation": null, "regressors_used": false, "linear_model": {"model": "linalg_solve", "lambda": 1, "recency_weighting": 0.1}, "randomwalk_n": null, "trend_window": 365, "trend_standin": "rolling_trend", "trend_anomaly_detector_params": {"method": "mad", "method_params": {"distribution": "uniform", "alpha": 0.05}, "fillna": "rolling_mean_24", "transform_dict": {"fillna": null, "transformations": {"0": "EWMAFilter"}, "transformation_params": {"0": {"span": 7}}}}, "trend_transformation": {"fillna": "nearest", "transformations": {"0": "StandardScaler", "1": "AnomalyRemoval"}, "transformation_params": {"0": {}, "1": {"method": "IQR", "transform_dict": {"fillna": null, "transformations": {"0": "ClipOutliers"}, "transformation_params": {"0": {"method": "clip", "std_threshold": 6}}}, "method_params": {"iqr_threshold": 2.5, "iqr_quantiles": [0.25, 0.75]}, "fillna": "ffill"}}}, "trend_model": {"Model": "SectionalMotif", "ModelParameters": {"window": 50, "point_method": "mean", "distance_metric": "dice", "include_differenced": true, "k": 5, "stride_size": 1, "regression_type": null}}, "trend_phi": null}''',
-        'TransformationParameters': '{"fillna": "pad", "transformations": {"0": "EWMAFilter", "1": "LevelShiftTransformer", "2": "StandardScaler", "3": "DatepartRegression"}, "transformation_params": {"0": {"span": 7}, "1": {"window_size": 7, "alpha": 2.0, "grouping_forward_limit": 2, "max_level_shifts": 5, "alignment": "average"}, "2": {}, "3": {"regression_model": {"model": "ElasticNet", "model_params": {}}, "datepart_method": "recurring", "polynomial_degree": null, "transform_dict": {"fillna": null, "transformations": {"0": "ScipyFilter"}, "transformation_params": {"0": {"method": "savgol_filter", "method_args": {"window_length": 31, "polyorder": 3, "deriv": 0, "mode": "interp"}}}}, "holiday_countries_used": false}}}',
+        'TransformationParameters': '{"fillna": "ffill", "transformations": {"0": "EWMAFilter", "1": "LevelShiftTransformer", "2": "StandardScaler", "3": "DatepartRegression"}, "transformation_params": {"0": {"span": 7}, "1": {"window_size": 7, "alpha": 2.0, "grouping_forward_limit": 2, "max_level_shifts": 5, "alignment": "average"}, "2": {}, "3": {"regression_model": {"model": "ElasticNet", "model_params": {}}, "datepart_method": "recurring", "polynomial_degree": null, "transform_dict": {"fillna": null, "transformations": {"0": "ScipyFilter"}, "transformation_params": {"0": {"method": "savgol_filter", "method_args": {"window_length": 31, "polyorder": 3, "deriv": 0, "mode": "interp"}}}}, "holiday_countries_used": false}}}',
         'Ensemble': 0,
     },
     "72": {  # optimized on M5, 58.5 SMAPE

autots/tools/profile.py

@@ -129,9 +129,9 @@ def profile_time_series(
         & (metrics_df['cv_squared'] >= cvar_threshold),
         'PROFILE',
     ] = 'lumpy'
-    metrics_df.loc[
-        metrics_df['zero_diff_proportion'] >= flat_threshold, 'PROFILE'
-    ] = 'flat'
+    metrics_df.loc[metrics_df['zero_diff_proportion'] >= flat_threshold, 'PROFILE'] = (
+        'flat'
+    )
     metrics_df.loc[
         metrics_df['null_percentage'] >= new_product_threshold, 'PROFILE'
     ] = 'new_product'

autots/tools/transform.py

@@ -7041,7 +7041,7 @@ def RandomTransform(
     num_trans = random.randint(transformer_min_depth, transformer_max_depth)
     # sometimes return no transformation
     if num_trans == 1 and allow_none:
-        test = random.choices(["None", "Some"], [0.1, 0.9])[0]
+        test = random.choices(["None", "Some"], [0.05, 0.95])[0]
         if test == "None":
             return {
                 "fillna": na_choice,
@@ -7095,13 +7095,15 @@ def RandomTransform(
             keys = list(range(len(trans)))
         else:
             keys = list(range(num_trans))
-        # now get the parameters for the specified transformers
-        params = [get_transformer_params(x, method=params_method) for x in trans]
-        return {
-            "fillna": na_choice,
-            "transformations": dict(zip(keys, trans)),
-            "transformation_params": dict(zip(keys, params)),
-        }
+    else:
+        keys = list(range(num_trans))
+    # now get the parameters for the specified transformers
+    params = [get_transformer_params(x, method=params_method) for x in trans]
+    return {
+        "fillna": na_choice,
+        "transformations": dict(zip(keys, trans)),
+        "transformation_params": dict(zip(keys, params)),
+    }
 
 
 def random_cleaners():

docs/build/html/.buildinfo

@@ -1,4 +1,4 @@
 # Sphinx build info version 1
 # This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done.
-config: 9d5cb101099b8cc7bbd8db30575a9d26
+config: df61a3ec39f9063c17424b4b25add683
 tags: 645f666f9bcd5a90fca523b33c5a78b7

docs/build/html/_static/documentation_options.js

@@ -1,5 +1,5 @@
 const DOCUMENTATION_OPTIONS = {
-    VERSION: '0.6.18',
+    VERSION: '0.6.19',
     LANGUAGE: 'en',
     COLLAPSE_INDEX: false,
     BUILDER: 'html',

docs/build/html/genindex.html

@@ -13,10 +13,10 @@
         gtag('config', 'G-P2KLF8302E');
     </script>
 
-    <title>Index &#8212; AutoTS 0.6.18 documentation</title>
+    <title>Index &#8212; AutoTS 0.6.19 documentation</title>
     <link rel="stylesheet" type="text/css" href="_static/pygments.css?v=b76e3c8a" />
     <link rel="stylesheet" type="text/css" href="_static/alabaster.css?v=a4fa0645" />
-    <script src="_static/documentation_options.js?v=e53b5979"></script>
+    <script src="_static/documentation_options.js?v=4143dd60"></script>
     <script src="_static/doctools.js?v=888ff710"></script>
     <script src="_static/sphinx_highlight.js?v=dc90522c"></script>
     <link rel="index" title="Index" href="#" />

docs/build/html/index.html

@@ -14,10 +14,10 @@
         gtag('config', 'G-P2KLF8302E');
     </script>
 
-    <title>AutoTS &#8212; AutoTS 0.6.18 documentation</title>
+    <title>AutoTS &#8212; AutoTS 0.6.19 documentation</title>
     <link rel="stylesheet" type="text/css" href="_static/pygments.css?v=b76e3c8a" />
     <link rel="stylesheet" type="text/css" href="_static/alabaster.css?v=a4fa0645" />
-    <script src="_static/documentation_options.js?v=e53b5979"></script>
+    <script src="_static/documentation_options.js?v=4143dd60"></script>
     <script src="_static/doctools.js?v=888ff710"></script>
     <script src="_static/sphinx_highlight.js?v=dc90522c"></script>
     <link rel="index" title="Index" href="genindex.html" />

docs/build/html/py-modindex.html

@@ -13,10 +13,10 @@
         gtag('config', 'G-P2KLF8302E');
     </script>
 
-    <title>Python Module Index &#8212; AutoTS 0.6.18 documentation</title>
+    <title>Python Module Index &#8212; AutoTS 0.6.19 documentation</title>
     <link rel="stylesheet" type="text/css" href="_static/pygments.css?v=b76e3c8a" />
     <link rel="stylesheet" type="text/css" href="_static/alabaster.css?v=a4fa0645" />
-    <script src="_static/documentation_options.js?v=e53b5979"></script>
+    <script src="_static/documentation_options.js?v=4143dd60"></script>
     <script src="_static/doctools.js?v=888ff710"></script>
     <script src="_static/sphinx_highlight.js?v=dc90522c"></script>
     <link rel="index" title="Index" href="genindex.html" />

docs/build/html/search.html

@@ -13,11 +13,11 @@
         gtag('config', 'G-P2KLF8302E');
     </script>
 
-    <title>Search &#8212; AutoTS 0.6.18 documentation</title>
+    <title>Search &#8212; AutoTS 0.6.19 documentation</title>
     <link rel="stylesheet" type="text/css" href="_static/pygments.css?v=b76e3c8a" />
     <link rel="stylesheet" type="text/css" href="_static/alabaster.css?v=a4fa0645" />
 
-    <script src="_static/documentation_options.js?v=e53b5979"></script>
+    <script src="_static/documentation_options.js?v=4143dd60"></script>
     <script src="_static/doctools.js?v=888ff710"></script>
     <script src="_static/sphinx_highlight.js?v=dc90522c"></script>
     <script src="_static/searchtools.js"></script>

docs/build/html/source/autots.datasets.html

@@ -14,10 +14,10 @@
         gtag('config', 'G-P2KLF8302E');
     </script>
 
-    <title>autots.datasets package &#8212; AutoTS 0.6.18 documentation</title>
+    <title>autots.datasets package &#8212; AutoTS 0.6.19 documentation</title>
     <link rel="stylesheet" type="text/css" href="../_static/pygments.css?v=b76e3c8a" />
     <link rel="stylesheet" type="text/css" href="../_static/alabaster.css?v=a4fa0645" />
-    <script src="../_static/documentation_options.js?v=e53b5979"></script>
+    <script src="../_static/documentation_options.js?v=4143dd60"></script>
     <script src="../_static/doctools.js?v=888ff710"></script>
     <script src="../_static/sphinx_highlight.js?v=dc90522c"></script>
     <link rel="index" title="Index" href="../genindex.html" />

docs/build/html/source/autots.evaluator.html

@@ -14,10 +14,10 @@
         gtag('config', 'G-P2KLF8302E');
     </script>
 
-    <title>autots.evaluator package &#8212; AutoTS 0.6.18 documentation</title>
+    <title>autots.evaluator package &#8212; AutoTS 0.6.19 documentation</title>
     <link rel="stylesheet" type="text/css" href="../_static/pygments.css?v=b76e3c8a" />
     <link rel="stylesheet" type="text/css" href="../_static/alabaster.css?v=a4fa0645" />
-    <script src="../_static/documentation_options.js?v=e53b5979"></script>
+    <script src="../_static/documentation_options.js?v=4143dd60"></script>
     <script src="../_static/doctools.js?v=888ff710"></script>
     <script src="../_static/sphinx_highlight.js?v=dc90522c"></script>
     <link rel="index" title="Index" href="../genindex.html" />

docs/build/html/source/autots.html

@@ -14,10 +14,10 @@
         gtag('config', 'G-P2KLF8302E');
     </script>
 
-    <title>autots package &#8212; AutoTS 0.6.18 documentation</title>
+    <title>autots package &#8212; AutoTS 0.6.19 documentation</title>
     <link rel="stylesheet" type="text/css" href="../_static/pygments.css?v=b76e3c8a" />
     <link rel="stylesheet" type="text/css" href="../_static/alabaster.css?v=a4fa0645" />
-    <script src="../_static/documentation_options.js?v=e53b5979"></script>
+    <script src="../_static/documentation_options.js?v=4143dd60"></script>
     <script src="../_static/doctools.js?v=888ff710"></script>
     <script src="../_static/sphinx_highlight.js?v=dc90522c"></script>
     <link rel="index" title="Index" href="../genindex.html" />

docs/build/html/source/autots.models.html

@@ -14,10 +14,10 @@
         gtag('config', 'G-P2KLF8302E');
     </script>
 
-    <title>autots.models package &#8212; AutoTS 0.6.18 documentation</title>
+    <title>autots.models package &#8212; AutoTS 0.6.19 documentation</title>
     <link rel="stylesheet" type="text/css" href="../_static/pygments.css?v=b76e3c8a" />
     <link rel="stylesheet" type="text/css" href="../_static/alabaster.css?v=a4fa0645" />
-    <script src="../_static/documentation_options.js?v=e53b5979"></script>
+    <script src="../_static/documentation_options.js?v=4143dd60"></script>
     <script src="../_static/doctools.js?v=888ff710"></script>
     <script src="../_static/sphinx_highlight.js?v=dc90522c"></script>
     <link rel="index" title="Index" href="../genindex.html" />