diff --git a/CHANGELOG.md b/CHANGELOG.md index 4af2433c..7e21d4a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,17 @@ This is the current release cycle, so stay tuned for future releases! +### v1.4.12 – v1.4.13 + +- **Fixed an issue when syncing empty DataFrames [(#95)](https://github.com/bmeares/Meerschaum/issues/95).** + When syncing an empty list of documents, `Pipe.filter_existing()` would trigger pulling the entire table into memory. This patch adds a check if the dataframe is empty. + +- **Allow the `datetime` column to be omitted in the `bootstrap` wizard.** + Now that the `datetime` index is optional, the bootstrapping wizard allows users to skip this index. + +- **Fixed a small issue when syncing to MySQL.** + Due to the addition of MySQL 5.7 support in v1.4.11, a slight edge case arose which broke SQL definitions. This patch fixes MySQL behavior when a `WHERE` clause is present in the definition. + ### v1.4.11 - **Add support for older versions of MySQL.** diff --git a/docs/mkdocs/news/changelog.md b/docs/mkdocs/news/changelog.md index 4af2433c..7e21d4a6 100644 --- a/docs/mkdocs/news/changelog.md +++ b/docs/mkdocs/news/changelog.md @@ -4,6 +4,17 @@ This is the current release cycle, so stay tuned for future releases! +### v1.4.12 – v1.4.13 + +- **Fixed an issue when syncing empty DataFrames [(#95)](https://github.com/bmeares/Meerschaum/issues/95).** + When syncing an empty list of documents, `Pipe.filter_existing()` would trigger pulling the entire table into memory. This patch adds a check if the dataframe is empty. + +- **Allow the `datetime` column to be omitted in the `bootstrap` wizard.** + Now that the `datetime` index is optional, the bootstrapping wizard allows users to skip this index. + +- **Fixed a small issue when syncing to MySQL.** + Due to the addition of MySQL 5.7 support in v1.4.11, a slight edge case arose which broke SQL definitions. This patch fixes MySQL behavior when a `WHERE` clause is present in the definition. + ### v1.4.11 - **Add support for older versions of MySQL.** diff --git a/meerschaum/actions/start.py b/meerschaum/actions/start.py index 8fe277f7..7b58e7dd 100644 --- a/meerschaum/actions/start.py +++ b/meerschaum/actions/start.py @@ -520,7 +520,7 @@ def connect(conn): msg = ( f"Successfully started connector" + ('s' if len(successes) != 1 else '') + ' ' + items_str(successes) + '.' - ) + ) if success else f"Failed to start {len(fails)} connectors." if len(fails) > 0: msg += ( f"\n Failed to start connector" + ('s' if len(fails) != 1 else '') diff --git a/meerschaum/config/_version.py b/meerschaum/config/_version.py index b4827f3e..ab90cc5c 100644 --- a/meerschaum/config/_version.py +++ b/meerschaum/config/_version.py @@ -2,4 +2,4 @@ Specify the Meerschaum release version. """ -__version__ = "1.4.11" +__version__ = "1.4.13" diff --git a/meerschaum/connectors/sql/_fetch.py b/meerschaum/connectors/sql/_fetch.py index 9f8bd197..7f0fc6e6 100644 --- a/meerschaum/connectors/sql/_fetch.py +++ b/meerschaum/connectors/sql/_fetch.py @@ -157,7 +157,7 @@ def get_pipe_metadef( ) else _join_fetch_query(pipe, debug=debug, **kw) ) - has_where = 'where' in meta_def.lower()[meta_def.lower().rfind('select'):] + has_where = 'where' in meta_def.lower()[meta_def.lower().rfind('definition'):] if dt_name and (begin_da or end_da): definition_dt_name = dateadd_str(self.flavor, 'minute', 0, f"definition.{dt_name}") meta_def += "\n" + ("AND" if has_where else "WHERE") + " " diff --git a/meerschaum/core/Pipe/_bootstrap.py b/meerschaum/core/Pipe/_bootstrap.py index 93829291..53c3f50a 100644 --- a/meerschaum/core/Pipe/_bootstrap.py +++ b/meerschaum/core/Pipe/_bootstrap.py @@ -220,12 +220,11 @@ def _ask_for_columns(pipe, debug: bool=False) -> Dict[str, str]: info(f"Please enter column names for {pipe}:") while True: try: - datetime_name = prompt(f"Datetime column:", icon=False) + datetime_name = prompt(f"Datetime column (empty to omit):", icon=False) except KeyboardInterrupt: return False, f"Cancelled bootstrapping {pipe}." if datetime_name == '': - warn(f"Please enter a datetime column.", stack=False) - continue + datetime_name = None try: id_name = prompt(f"ID column (empty to omit):", icon=False) @@ -248,5 +247,3 @@ def _ask_for_columns(pipe, debug: bool=False) -> Dict[str, str]: 'id': id_name, 'value': value_name, } - - diff --git a/meerschaum/core/Pipe/_sync.py b/meerschaum/core/Pipe/_sync.py index 272842dd..2ad47cd0 100644 --- a/meerschaum/core/Pipe/_sync.py +++ b/meerschaum/core/Pipe/_sync.py @@ -435,6 +435,10 @@ def filter_existing( pd = import_pandas() if not isinstance(df, pd.DataFrame): df = self.enforce_dtypes(df, debug=debug) + + if df.empty: + return df, df, df + ### begin is the oldest data in the new dataframe try: min_dt = pd.to_datetime( diff --git a/meerschaum/utils/venv/__init__.py b/meerschaum/utils/venv/__init__.py index 11737310..347b4263 100644 --- a/meerschaum/utils/venv/__init__.py +++ b/meerschaum/utils/venv/__init__.py @@ -56,12 +56,9 @@ def activate_venv( A bool indicating whether the virtual environment was successfully activated. """ - # debug = True thread_id = get_ident() if active_venvs_order and active_venvs_order[0] == venv: return True - # if venv in threads_active_venvs.get(thread_id, {}): - # return True import sys, platform, os from meerschaum.config._paths import VIRTENV_RESOURCES_PATH if debug: @@ -80,9 +77,12 @@ def activate_venv( threads_active_venvs[thread_id][venv] += 1 target = str(venv_target_path(venv, debug=debug)) - if target in active_venvs_order: + if venv in active_venvs_order: sys.path.remove(target) - active_venvs_order.remove(target) + try: + active_venvs_order.remove(venv) + except Exception as e: + pass if venv is not None: sys.path.insert(0, target) else: @@ -90,7 +90,10 @@ def activate_venv( sys.path.insert(1, target) else: sys.path.insert(0, target) - active_venvs_order.insert(0, target) + try: + active_venvs_order.insert(0, venv) + except Exception as e: + pass return True @@ -188,7 +191,10 @@ def deactivate_venv( with LOCKS['sys.path']: if target in sys.path: sys.path.remove(target) - active_venvs_order.remove(target) + try: + active_venvs_order.remove(venv) + except Exception as e: + pass return True