🐛 v1.4.13 Fixed performance issue (#95) and syncing from MySQL.

CHANGELOG.md

@@ -4,6 +4,17 @@
 
 This is the current release cycle, so stay tuned for future releases!
 
+### v1.4.12 – v1.4.13
+
+- **Fixed an issue when syncing empty DataFrames [(#95)](https://github.com/bmeares/Meerschaum/issues/95).**  
+  When syncing an empty list of documents, `Pipe.filter_existing()` would trigger pulling the entire table into memory. This patch adds a check if the dataframe is empty.
+
+- **Allow the `datetime` column to be omitted in the `bootstrap` wizard.**  
+  Now that the `datetime` index is optional, the bootstrapping wizard allows users to skip this index.
+
+- **Fixed a small issue when syncing to MySQL.**  
+  Due to the addition of MySQL 5.7 support in v1.4.11, a slight edge case arose which broke SQL definitions. This patch fixes MySQL behavior when a `WHERE` clause is present in the definition.
+
 ### v1.4.11
 
 - **Add support for older versions of MySQL.**  

docs/mkdocs/news/changelog.md

@@ -4,6 +4,17 @@
 
 This is the current release cycle, so stay tuned for future releases!
 
+### v1.4.12 – v1.4.13
+
+- **Fixed an issue when syncing empty DataFrames [(#95)](https://github.com/bmeares/Meerschaum/issues/95).**  
+  When syncing an empty list of documents, `Pipe.filter_existing()` would trigger pulling the entire table into memory. This patch adds a check if the dataframe is empty.
+
+- **Allow the `datetime` column to be omitted in the `bootstrap` wizard.**  
+  Now that the `datetime` index is optional, the bootstrapping wizard allows users to skip this index.
+
+- **Fixed a small issue when syncing to MySQL.**  
+  Due to the addition of MySQL 5.7 support in v1.4.11, a slight edge case arose which broke SQL definitions. This patch fixes MySQL behavior when a `WHERE` clause is present in the definition.
+
 ### v1.4.11
 
 - **Add support for older versions of MySQL.**  

meerschaum/actions/start.py

@@ -520,7 +520,7 @@ def connect(conn):
     msg = (
         f"Successfully started connector" + ('s' if len(successes) != 1 else '')
         + ' ' + items_str(successes) + '.'
-    )
+    ) if success else f"Failed to start {len(fails)} connectors."
     if len(fails) > 0:
         msg += (
             f"\n    Failed to start connector" + ('s' if len(fails) != 1 else '')

meerschaum/config/_version.py

@@ -2,4 +2,4 @@
 Specify the Meerschaum release version.
 """
 
-__version__ = "1.4.11"
+__version__ = "1.4.13"

meerschaum/connectors/sql/_fetch.py

@@ -157,7 +157,7 @@ def get_pipe_metadef(
         ) else _join_fetch_query(pipe, debug=debug, **kw)
     )
 
-    has_where = 'where' in meta_def.lower()[meta_def.lower().rfind('select'):]
+    has_where = 'where' in meta_def.lower()[meta_def.lower().rfind('definition'):]
     if dt_name and (begin_da or end_da):
         definition_dt_name = dateadd_str(self.flavor, 'minute', 0, f"definition.{dt_name}")
         meta_def += "\n" + ("AND" if has_where else "WHERE") + " "

meerschaum/core/Pipe/_bootstrap.py

@@ -220,12 +220,11 @@ def _ask_for_columns(pipe, debug: bool=False) -> Dict[str, str]:
     info(f"Please enter column names for {pipe}:")
     while True:
         try:
-            datetime_name = prompt(f"Datetime column:", icon=False)
+            datetime_name = prompt(f"Datetime column (empty to omit):", icon=False)
         except KeyboardInterrupt:
             return False, f"Cancelled bootstrapping {pipe}."
         if datetime_name == '':
-            warn(f"Please enter a datetime column.", stack=False)
-            continue
+            datetime_name = None
 
         try:
             id_name = prompt(f"ID column (empty to omit):", icon=False)
@@ -248,5 +247,3 @@ def _ask_for_columns(pipe, debug: bool=False) -> Dict[str, str]:
         'id': id_name,
         'value': value_name,
     }
-
-

meerschaum/core/Pipe/_sync.py

@@ -435,6 +435,10 @@ def filter_existing(
     pd = import_pandas()
     if not isinstance(df, pd.DataFrame):
         df = self.enforce_dtypes(df, debug=debug)
+
+    if df.empty:
+        return df, df, df
+
     ### begin is the oldest data in the new dataframe
     try:
         min_dt = pd.to_datetime(

meerschaum/utils/venv/__init__.py

@@ -56,12 +56,9 @@ def activate_venv(
     A bool indicating whether the virtual environment was successfully activated.
 
     """
-    #  debug = True
     thread_id = get_ident()
     if active_venvs_order and active_venvs_order[0] == venv:
         return True
-    #  if venv in threads_active_venvs.get(thread_id, {}):
-        #  return True
     import sys, platform, os
     from meerschaum.config._paths import VIRTENV_RESOURCES_PATH
     if debug:
@@ -80,17 +77,23 @@ def activate_venv(
             threads_active_venvs[thread_id][venv] += 1
 
         target = str(venv_target_path(venv, debug=debug))
-        if target in active_venvs_order:
+        if venv in active_venvs_order:
             sys.path.remove(target)
-            active_venvs_order.remove(target)
+            try:
+                active_venvs_order.remove(venv)
+            except Exception as e:
+                pass
         if venv is not None:
             sys.path.insert(0, target)
         else:
             if sys.path and sys.path[0] in (os.getcwd(), ''):
                 sys.path.insert(1, target)
             else:
                 sys.path.insert(0, target)
-        active_venvs_order.insert(0, target)
+        try:
+            active_venvs_order.insert(0, venv)
+        except Exception as e:
+            pass
 
     return True
 
@@ -188,7 +191,10 @@ def deactivate_venv(
     with LOCKS['sys.path']:
         if target in sys.path:
             sys.path.remove(target)
-            active_venvs_order.remove(target)
+            try:
+                active_venvs_order.remove(venv)
+            except Exception as e:
+                pass
 
     return True