From 39e1a6ae65035eb5713dd882f123b48afc724747 Mon Sep 17 00:00:00 2001 From: bmeares Date: Sat, 12 Nov 2022 23:35:59 -0500 Subject: [PATCH] =?UTF-8?q?=E2=9A=A1=EF=B8=8F=20v1.4.9=20Performance=20boo?= =?UTF-8?q?st,=20bugfixes,=20and=20more.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 26 +++++++++++ docs/mkdocs/news/changelog.md | 26 +++++++++++ meerschaum/actions/api.py | 4 +- meerschaum/config/__init__.py | 6 +-- meerschaum/config/_version.py | 2 +- meerschaum/connectors/__init__.py | 28 +++++++++++ meerschaum/connectors/sql/_pipes.py | 20 ++++---- meerschaum/connectors/sql/_users.py | 2 +- meerschaum/connectors/sql/tables/__init__.py | 15 +++--- meerschaum/core/Pipe/_attributes.py | 18 +++++-- meerschaum/core/Pipe/_bootstrap.py | 7 ++- meerschaum/core/Pipe/_clear.py | 15 ++++-- meerschaum/core/Pipe/_data.py | 49 ++++++++++++-------- meerschaum/core/Pipe/_delete.py | 8 +++- meerschaum/core/Pipe/_drop.py | 7 ++- meerschaum/core/Pipe/_edit.py | 9 +++- meerschaum/core/Pipe/_register.py | 5 +- meerschaum/core/Pipe/_sync.py | 42 +++++++++++------ meerschaum/plugins/__init__.py | 5 ++ meerschaum/utils/packages/_packages.py | 1 - meerschaum/utils/sql.py | 6 +-- meerschaum/utils/venv/__init__.py | 31 +++++++++++-- requirements/api.txt | 1 - requirements/full.txt | 1 - scripts/portable/build.sh | 12 ++--- 25 files changed, 258 insertions(+), 88 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 01250652..39733a60 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,32 @@ This is the current release cycle, so stay tuned for future releases! +### v1.4.9 + +- **Fixed in-place syncs for aggregate queries.** + In-place SQL syncs which use aggregation functions are now handled correctly. This version addresses differences in column types between backtrack and new data. For example, the following query will now be correctly synced: + + ```sql + WITH days_src AS ( + SELECT *, DATE_TRUNC('day', "datetime") AS days + FROM plugin_stress_test + ) + SELECT days, AVG(val) AS avg_value + FROM days_src + GROUP BY days + ``` + +- **Activate virtual environments for custom instance connectors.** + All pipe methods now activate virtual environments for custom instance connectors. + +- **Improved database connection performance.** + Cold connections to a SQL database have been sped up by replacing `sqlalchemy_utils` with handwritten logic (JSON for PostgreSQL-like and SQLite). + +- **Fixed an issue with virtual environment verification in a portable environment.** + The portable build has been updated to Python 3.9.15, and this patch includes a check to determine the known `site-package` path for a virtual environment of `None` instead of relying on the default user `site-packages` directory. + +- **Fixed some environment warnings when starting the API** + ### v1.4.5 – v1.4.8 - **Bugfixes and stability improvements.** diff --git a/docs/mkdocs/news/changelog.md b/docs/mkdocs/news/changelog.md index 01250652..39733a60 100644 --- a/docs/mkdocs/news/changelog.md +++ b/docs/mkdocs/news/changelog.md @@ -4,6 +4,32 @@ This is the current release cycle, so stay tuned for future releases! +### v1.4.9 + +- **Fixed in-place syncs for aggregate queries.** + In-place SQL syncs which use aggregation functions are now handled correctly. This version addresses differences in column types between backtrack and new data. For example, the following query will now be correctly synced: + + ```sql + WITH days_src AS ( + SELECT *, DATE_TRUNC('day', "datetime") AS days + FROM plugin_stress_test + ) + SELECT days, AVG(val) AS avg_value + FROM days_src + GROUP BY days + ``` + +- **Activate virtual environments for custom instance connectors.** + All pipe methods now activate virtual environments for custom instance connectors. + +- **Improved database connection performance.** + Cold connections to a SQL database have been sped up by replacing `sqlalchemy_utils` with handwritten logic (JSON for PostgreSQL-like and SQLite). + +- **Fixed an issue with virtual environment verification in a portable environment.** + The portable build has been updated to Python 3.9.15, and this patch includes a check to determine the known `site-package` path for a virtual environment of `None` instead of relying on the default user `site-packages` directory. + +- **Fixed some environment warnings when starting the API** + ### v1.4.5 – v1.4.8 - **Bugfixes and stability improvements.** diff --git a/meerschaum/actions/api.py b/meerschaum/actions/api.py index 7e6862df..bea13797 100644 --- a/meerschaum/actions/api.py +++ b/meerschaum/actions/api.py @@ -260,8 +260,8 @@ def _api_start( env_text = '' for key, val in env_dict.items(): - value = json.dumps(json.dumps(val)) if isinstance(val, dict) else val - env_text += f"{key}={value}\n" + value = json.dumps(val) if isinstance(val, dict) else val + env_text += f"{key}='{value}'\n" with open(uvicorn_env_path, 'w+', encoding='utf-8') as f: if debug: dprint(f"Writing ENV file to '{uvicorn_env_path}'.") diff --git a/meerschaum/config/__init__.py b/meerschaum/config/__init__.py index cadfe894..30f59688 100644 --- a/meerschaum/config/__init__.py +++ b/meerschaum/config/__init__.py @@ -16,7 +16,7 @@ from meerschaum.config._version import __version__ from meerschaum.config._edit import edit_config, write_config -from meerschaum.config.static import _static_config +from meerschaum.config.static import STATIC_CONFIG from meerschaum.config._paths import ( PERMANENT_PATCH_DIR_PATH, @@ -125,7 +125,7 @@ def get_config( """ import json - symlinks_key = _static_config()['config']['symlinks_key'] + symlinks_key = STATIC_CONFIG['config']['symlinks_key'] if debug: from meerschaum.utils.debug import dprint dprint(f"Indexing keys: {keys}", color=False) @@ -318,7 +318,7 @@ def write_plugin_config( ### Make sure readline is available for the portable version. -environment_runtime = _static_config()['environment']['runtime'] +environment_runtime = STATIC_CONFIG['environment']['runtime'] if environment_runtime in os.environ: if os.environ[environment_runtime] == 'portable': from meerschaum.utils.packages import ensure_readline diff --git a/meerschaum/config/_version.py b/meerschaum/config/_version.py index bd71f3ba..a52988ff 100644 --- a/meerschaum/config/_version.py +++ b/meerschaum/config/_version.py @@ -2,4 +2,4 @@ Specify the Meerschaum release version. """ -__version__ = "1.4.8" +__version__ = "1.4.9" diff --git a/meerschaum/connectors/__init__.py b/meerschaum/connectors/__init__.py index 4d905c7e..f91944a9 100644 --- a/meerschaum/connectors/__init__.py +++ b/meerschaum/connectors/__init__.py @@ -317,3 +317,31 @@ def load_plugin_connectors(): if not to_import: return import_plugins(*to_import) + + +def get_connector_plugin( + connector: Connector, + ) -> Union[str, None, 'meerschaum.Plugin']: + """ + Determine the plugin for a connector. + This is useful for handling virtual environments for custom instance connectors. + + Parameters + ---------- + connector: Connector + The connector which may require a virtual environment. + + Returns + ------- + A Plugin, 'mrsm', or None. + """ + if not hasattr(connector, 'type'): + return None + from meerschaum import Plugin + return ( + Plugin(connector.__module__.replace('plugins.', '').split('.')[0]) + if connector.type in custom_types else ( + Plugin(connector.label) if connector.type == 'plugin' + else 'mrsm' + ) + ) diff --git a/meerschaum/connectors/sql/_pipes.py b/meerschaum/connectors/sql/_pipes.py index ea13647a..a19ab792 100644 --- a/meerschaum/connectors/sql/_pipes.py +++ b/meerschaum/connectors/sql/_pipes.py @@ -56,8 +56,7 @@ def register_pipe( 'metric_key' : pipe.metric_key, 'location_key' : pipe.location_key, 'parameters' : ( - json.dumps(parameters) if self.flavor in ('duckdb',) else parameters - # json.dumps(parameters) if self.flavor not in json_flavors else parameters + json.dumps(parameters) if self.flavor not in json_flavors else parameters ), } query = sqlalchemy.insert(pipes).values(**values) @@ -1384,6 +1383,7 @@ def get_temp_table_name(label: str) -> str: debug = debug, ) backtrack_cols = {str(col.name): str(col.type) for col in backtrack_table_obj.columns} + common_cols = [col for col in new_cols if col in backtrack_cols] on_cols = { col: new_cols.get(col, 'object') for col_key, col in pipe.columns.items() @@ -1419,17 +1419,17 @@ def get_temp_table_name(label: str) -> str: + '\nAND\n'.join([ ( 'COALESCE(new.' + sql_item_name(c, self.flavor) + ", " - + get_null_replacement(typ, self.flavor) + ") " + + get_null_replacement(new_cols[c], self.flavor) + ") " + ' = ' + 'COALESCE(old.' + sql_item_name(c, self.flavor) + ", " - + get_null_replacement(typ, self.flavor) + ") " - ) for c, typ in new_cols.items() + + get_null_replacement(backtrack_cols[c], self.flavor) + ") " + ) for c in common_cols ]) + "\nWHERE\n" + '\nAND\n'.join([ ( 'old.' + sql_item_name(c, self.flavor) + ' IS NULL' - ) for c in new_cols + ) for c in common_cols ]) # + "\nAND\n" # + '\nAND\n'.join([ @@ -1455,17 +1455,17 @@ def get_temp_table_name(label: str) -> str: + '\nAND\n'.join([ ( 'COALESCE(new.' + sql_item_name(c, self.flavor) + ", " - + get_null_replacement(typ, self.flavor) + ") " + + get_null_replacement(new_cols[c], self.flavor) + ") " + ' = ' + 'COALESCE(old.' + sql_item_name(c, self.flavor) + ", " - + get_null_replacement(typ, self.flavor) + ") " - ) for c, typ in new_cols.items() + + get_null_replacement(backtrack_cols[c], self.flavor) + ") " + ) for c in common_cols ]) + "\nWHERE\n" + '\nAND\n'.join([ ( 'old.' + sql_item_name(c, self.flavor) + ' IS NULL' - ) for c in new_cols + ) for c in common_cols ]) # + "\nAND\n" # + '\nAND\n'.join([ diff --git a/meerschaum/connectors/sql/_users.py b/meerschaum/connectors/sql/_users.py index 361115b9..dd9a8947 100644 --- a/meerschaum/connectors/sql/_users.py +++ b/meerschaum/connectors/sql/_users.py @@ -41,7 +41,7 @@ def register_user( 'password_hash' : user.password_hash, 'user_type' : user.type, 'attributes' : ( - json.dumps(user.attributes) if self.flavor in ('duckdb',) else user.attributes + json.dumps(user.attributes) if self.flavor not in json_flavors else user.attributes ), } if old_id is not None: diff --git a/meerschaum/connectors/sql/tables/__init__.py b/meerschaum/connectors/sql/tables/__init__.py index 55f6b006..e6a36ea6 100644 --- a/meerschaum/connectors/sql/tables/__init__.py +++ b/meerschaum/connectors/sql/tables/__init__.py @@ -45,12 +45,12 @@ def get_tables( from meerschaum.utils.warnings import error from meerschaum.connectors.parse import parse_instance_keys from meerschaum.utils.packages import attempt_import + from meerschaum.utils.sql import json_flavors from meerschaum import get_connector - sqlalchemy, sqlalchemy_dialects_postgresql, sqlalchemy_utils_types_json = attempt_import( + sqlalchemy, sqlalchemy_dialects_postgresql = attempt_import( 'sqlalchemy', 'sqlalchemy.dialects.postgresql', - 'sqlalchemy_utils.types.json', lazy = False ) if not sqlalchemy: @@ -79,15 +79,14 @@ def get_tables( dprint(f"Creating tables for connector '{conn}'.") id_type = sqlalchemy.Integer - params_type = ( - sqlalchemy_utils_types_json.JSONType if conn.flavor not in ('duckdb',) - else sqlalchemy.String - ) - + if conn.flavor in json_flavors: + params_type = sqlalchemy.types.JSON + else: + params_type = sqlalchemy.types.Text id_names = ('user_id', 'plugin_id', 'pipe_id') sequences = { k: sqlalchemy.Sequence(k + '_seq') - for k in id_names + for k in id_names } id_col_args = { k: [k, id_type] for k in id_names } id_col_kw = { k: {'primary_key': True} for k in id_names } diff --git a/meerschaum/core/Pipe/_attributes.py b/meerschaum/core/Pipe/_attributes.py index 72a302ce..0c1ab957 100644 --- a/meerschaum/core/Pipe/_attributes.py +++ b/meerschaum/core/Pipe/_attributes.py @@ -19,6 +19,9 @@ def attributes(self) -> Dict[str, Any]: import time from meerschaum.config import get_config from meerschaum.config._patch import apply_patch_to_config + from meerschaum.utils.venv import Venv + from meerschaum.connectors import get_connector_plugin + timeout_seconds = get_config('pipes', 'attributes', 'local_cache_timeout_seconds') if '_attributes' not in self.__dict__: @@ -34,7 +37,8 @@ def attributes(self) -> Dict[str, Any]: if timed_out: self._attributes_sync_time = now local_attributes = self.__dict__.get('_attributes', {}) - instance_attributes = self.instance_connector.get_pipe_attributes(self) + with Venv(get_connector_plugin(self.instance_connector)): + instance_attributes = self.instance_connector.get_pipe_attributes(self) self._attributes = apply_patch_to_config(instance_attributes, local_attributes) return self._attributes @@ -199,7 +203,11 @@ def get_columns_types(self, debug: bool = False) -> Union[Dict[str, str], None]: } >>> """ - return self.instance_connector.get_pipe_columns_types(self, debug=debug) + from meerschaum.utils.venv import Venv + from meerschaum.connectors import get_connector_plugin + + with Venv(get_connector_plugin(self.instance_connector)): + return self.instance_connector.get_pipe_columns_types(self, debug=debug) def get_id(self, **kw: Any) -> Union[int, None]: @@ -207,7 +215,11 @@ def get_id(self, **kw: Any) -> Union[int, None]: Fetch a pipe's ID from its instance connector. If the pipe does not exist, return `None`. """ - return self.instance_connector.get_pipe_id(self, **kw) + from meerschaum.utils.venv import Venv + from meerschaum.connectors import get_connector_plugin + + with Venv(get_connector_plugin(self.instance_connector)): + return self.instance_connector.get_pipe_id(self, **kw) @property diff --git a/meerschaum/core/Pipe/_bootstrap.py b/meerschaum/core/Pipe/_bootstrap.py index aa050771..93829291 100644 --- a/meerschaum/core/Pipe/_bootstrap.py +++ b/meerschaum/core/Pipe/_bootstrap.py @@ -53,6 +53,8 @@ def bootstrap( from meerschaum.utils.formatting._shell import clear_screen from meerschaum.utils.formatting import print_tuple from meerschaum.actions import actions + from meerschaum.utils.venv import Venv + from meerschaum.connectors import get_connector_plugin _clear = get_config('shell', 'clear_screen', patch=True) @@ -74,7 +76,10 @@ def bootstrap( ) except KeyboardInterrupt as e: return False, f"Aborting bootstrapping {self}." - register_tuple = self.instance_connector.register_pipe(self, debug=debug) + + with Venv(get_connector_plugin(self.instance_connector)): + register_tuple = self.instance_connector.register_pipe(self, debug=debug) + if not register_tuple[0]: return register_tuple diff --git a/meerschaum/core/Pipe/_clear.py b/meerschaum/core/Pipe/_clear.py index d8b0a138..ba281377 100644 --- a/meerschaum/core/Pipe/_clear.py +++ b/meerschaum/core/Pipe/_clear.py @@ -52,12 +52,17 @@ def clear( """ from meerschaum.utils.warnings import warn + from meerschaum.utils.venv import Venv + from meerschaum.connectors import get_connector_plugin + if self.cache_pipe is not None: success, msg = self.cache_pipe.clear(begin=begin, end=end, debug=debug, **kw) if not success: warn(msg) - return self.instance_connector.clear_pipe( - self, - begin=begin, end=end, params=params, debug=debug, - **kw - ) + + with Venv(get_connector_plugin(self.instance_connector)): + return self.instance_connector.clear_pipe( + self, + begin=begin, end=end, params=params, debug=debug, + **kw + ) diff --git a/meerschaum/core/Pipe/_data.py b/meerschaum/core/Pipe/_data.py index 4f73c42b..812ae520 100644 --- a/meerschaum/core/Pipe/_data.py +++ b/meerschaum/core/Pipe/_data.py @@ -51,6 +51,8 @@ def get_data( """ from meerschaum.utils.warnings import warn + from meerschaum.utils.venv import Venv + from meerschaum.connectors import get_connector_plugin kw.update({'begin': begin, 'end': end, 'params': params,}) if not self.exists(debug=debug): @@ -69,14 +71,15 @@ def get_data( ) ### If `fresh` or the syncing failed, directly pull from the instance connector. - return self.enforce_dtypes( - self.instance_connector.get_pipe_data( - pipe = self, + with Venv(get_connector_plugin(self.instance_connector)): + return self.enforce_dtypes( + self.instance_connector.get_pipe_data( + pipe = self, + debug = debug, + **kw + ), debug = debug, - **kw - ), - debug = debug, - ) + ) def get_backtrack_data( @@ -84,8 +87,8 @@ def get_backtrack_data( backtrack_minutes: int = 0, begin: Optional['datetime.datetime'] = None, fresh: bool = False, - debug : bool = False, - **kw : Any + debug: bool = False, + **kw: Any ) -> Optional['pd.DataFrame']: """ Get the most recent data from the instance connector as a Pandas DataFrame. @@ -125,6 +128,9 @@ def get_backtrack_data( """ from meerschaum.utils.warnings import warn + from meerschaum.utils.venv import Venv + from meerschaum.connectors import get_connector_plugin + kw.update({'backtrack_minutes': backtrack_minutes, 'begin': begin,}) if not self.exists(debug=debug): @@ -143,14 +149,15 @@ def get_backtrack_data( ) ### If `fresh` or the syncing failed, directly pull from the instance connector. - return self.enforce_dtypes( - self.instance_connector.get_backtrack_data( - pipe = self, + with Venv(get_connector_plugin(self.instance_connector)): + return self.enforce_dtypes( + self.instance_connector.get_backtrack_data( + pipe = self, + debug = debug, + **kw + ), debug = debug, - **kw - ), - debug = debug, - ) + ) def get_rowcount( @@ -186,11 +193,15 @@ def get_rowcount( """ from meerschaum.utils.warnings import warn + from meerschaum.utils.venv import Venv + from meerschaum.connectors import get_connector_plugin + connector = self.instance_connector if not remote else self.connector try: - return connector.get_pipe_rowcount( - self, begin=begin, end=end, remote=remote, params=params, debug=debug - ) + with Venv(get_connector_plugin(connector)): + return connector.get_pipe_rowcount( + self, begin=begin, end=end, remote=remote, params=params, debug=debug + ) except AttributeError as e: warn(e) if remote: diff --git a/meerschaum/core/Pipe/_delete.py b/meerschaum/core/Pipe/_delete.py index 00a17c39..5b9904de 100644 --- a/meerschaum/core/Pipe/_delete.py +++ b/meerschaum/core/Pipe/_delete.py @@ -28,6 +28,9 @@ def delete( """ import os, pathlib from meerschaum.utils.warnings import warn + from meerschaum.utils.venv import Venv + from meerschaum.connectors import get_connector_plugin + if self.cache_pipe is not None: _delete_cache_tuple = self.cache_pipe.delete(debug=debug, **kw) if not _delete_cache_tuple[0]: @@ -37,7 +40,10 @@ def delete( os.remove(_cache_db_path) except Exception as e: warn(f"Could not delete cache file '{_cache_db_path}' for {self}:\n{e}") - result = self.instance_connector.delete_pipe(self, debug=debug, **kw) + + with Venv(get_connector_plugin(self.instance_connector)): + result = self.instance_connector.delete_pipe(self, debug=debug, **kw) + if not isinstance(result, tuple): return False, f"Received unexpected result from '{self.instance_connector}': {result}" if result[0]: diff --git a/meerschaum/core/Pipe/_drop.py b/meerschaum/core/Pipe/_drop.py index 8f705752..99772ecd 100644 --- a/meerschaum/core/Pipe/_drop.py +++ b/meerschaum/core/Pipe/_drop.py @@ -28,8 +28,13 @@ def drop( """ from meerschaum.utils.warnings import warn + from meerschaum.utils.venv import Venv + from meerschaum.connectors import get_connector_plugin + if self.cache_pipe is not None: _drop_cache_tuple = self.cache_pipe.drop(debug=debug, **kw) if not _drop_cache_tuple[0]: warn(_drop_cache_tuple[1]) - return self.instance_connector.drop_pipe(self, debug=debug, **kw) + + with Venv(get_connector_plugin(self.instance_connector)): + return self.instance_connector.drop_pipe(self, debug=debug, **kw) diff --git a/meerschaum/core/Pipe/_edit.py b/meerschaum/core/Pipe/_edit.py index 34d66351..ea006cb4 100644 --- a/meerschaum/core/Pipe/_edit.py +++ b/meerschaum/core/Pipe/_edit.py @@ -41,8 +41,12 @@ def edit( A `SuccessTuple` of success, message. """ + from meerschaum.utils.venv import Venv + from meerschaum.connectors import get_connector_plugin + if not interactive: - return self.instance_connector.edit_pipe(self, patch=patch, debug=debug, **kw) + with Venv(get_connector_plugin(self.instance_connector)): + return self.instance_connector.edit_pipe(self, patch=patch, debug=debug, **kw) from meerschaum.config._paths import PIPES_CACHE_RESOURCES_PATH from meerschaum.utils.misc import edit_file parameters_filename = str(self) + '.yaml' @@ -84,7 +88,8 @@ def edit( from meerschaum.utils.formatting import pprint pprint(self.parameters) - return self.instance_connector.edit_pipe(self, patch=patch, debug=debug, **kw) + with Venv(get_connector_plugin(self.instance_connector)): + return self.instance_connector.edit_pipe(self, patch=patch, debug=debug, **kw) def edit_definition( diff --git a/meerschaum/core/Pipe/_register.py b/meerschaum/core/Pipe/_register.py index 110e3b88..6e396015 100644 --- a/meerschaum/core/Pipe/_register.py +++ b/meerschaum/core/Pipe/_register.py @@ -28,6 +28,8 @@ def register( from meerschaum.connectors import custom_types from meerschaum.utils.formatting import get_console from meerschaum.utils.venv import Venv + from meerschaum.connectors import get_connector_plugin + import warnings with warnings.catch_warnings(): warnings.simplefilter('ignore') @@ -69,4 +71,5 @@ def register( 'columns': cols, } - return self.instance_connector.register_pipe(self, debug=debug) + with Venv(get_connector_plugin(self.instance_connector)): + return self.instance_connector.register_pipe(self, debug=debug) diff --git a/meerschaum/core/Pipe/_sync.py b/meerschaum/core/Pipe/_sync.py index c2b9b323..c833421d 100644 --- a/meerschaum/core/Pipe/_sync.py +++ b/meerschaum/core/Pipe/_sync.py @@ -115,6 +115,8 @@ def sync( from meerschaum.plugins import Plugin from meerschaum.utils.formatting import get_console from meerschaum.utils.venv import Venv + from meerschaum.connectors import get_connector_plugin + from meerschaum.config import get_config import datetime import time @@ -178,7 +180,8 @@ def _sync( and get_config('system', 'experimental', 'inplace_sync') ): - return self.instance_connector.sync_pipe_inplace(p, debug=debug, **kw) + with Venv(get_connector_plugin(self.instance_connector)): + return self.instance_connector.sync_pipe_inplace(p, debug=debug, **kw) ### Activate and invoke `sync(pipe)` for plugin connectors with `sync` methods. @@ -249,12 +252,13 @@ def _sync( run = True _retries = 1 while run: - return_tuple = p.instance_connector.sync_pipe( - pipe = p, - df = df, - debug = debug, - **kw - ) + with Venv(get_connector_plugin(self.instance_connector)): + return_tuple = p.instance_connector.sync_pipe( + pipe = p, + df = df, + debug = debug, + **kw + ) _retries += 1 run = (not return_tuple[0]) and force and _retries <= retries if run and debug: @@ -348,13 +352,17 @@ def get_sync_time( A `datetime.datetime` object if the pipe exists, otherwise `None`. """ - return self.instance_connector.get_sync_time( - self, - params = params, - newest = newest, - round_down = round_down, - debug = debug, - ) + from meerschaum.utils.venv import Venv + from meerschaum.connectors import get_connector_plugin + + with Venv(get_connector_plugin(self.instance_connector)): + return self.instance_connector.get_sync_time( + self, + params = params, + newest = newest, + round_down = round_down, + debug = debug, + ) def exists( @@ -374,7 +382,11 @@ def exists( A `bool` corresponding to whether a pipe's underlying table exists. """ - return self.instance_connector.pipe_exists(pipe=self, debug=debug) + from meerschaum.utils.venv import Venv + from meerschaum.connectors import get_connector_plugin + + with Venv(get_connector_plugin(self.instance_connector)): + return self.instance_connector.pipe_exists(pipe=self, debug=debug) def filter_existing( diff --git a/meerschaum/plugins/__init__.py b/meerschaum/plugins/__init__.py index 5f4c8d21..cd7cd049 100644 --- a/meerschaum/plugins/__init__.py +++ b/meerschaum/plugins/__init__.py @@ -187,6 +187,11 @@ def import_plugins( _warn(f"Unable to remove symlink {PLUGINS_INTERNAL_DIR_PATH}:\n {e}") if not PLUGINS_INTERNAL_DIR_PATH.exists(): + try: + ### It could be a broken symlink. + PLUGINS_INTERNAL_DIR_PATH.unlink() + except Exception as e: + pass try: # success, msg = make_symlink(PLUGINS_INTERNAL_DIR_PATH, PLUGINS_RESOURCES_PATH) success, msg = make_symlink(PLUGINS_RESOURCES_PATH, PLUGINS_INTERNAL_DIR_PATH) diff --git a/meerschaum/utils/packages/_packages.py b/meerschaum/utils/packages/_packages.py index 4d79e765..dca7d3fe 100644 --- a/meerschaum/utils/packages/_packages.py +++ b/meerschaum/utils/packages/_packages.py @@ -130,7 +130,6 @@ 'pytz' : 'pytz>=2022.1.0', 'joblib' : 'joblib>=0.17.0', 'sqlalchemy' : 'SQLAlchemy>=1.4.42', - 'sqlalchemy_utils' : 'sqlalchemy-utils>=0.38.3', 'databases' : 'databases>=0.4.0', 'aiosqlite' : 'aiosqlite>=0.16.0', 'asyncpg' : 'asyncpg>=0.21.0', diff --git a/meerschaum/utils/sql.py b/meerschaum/utils/sql.py index ceb7d1b5..b23e72dc 100644 --- a/meerschaum/utils/sql.py +++ b/meerschaum/utils/sql.py @@ -122,7 +122,7 @@ 'mysql' : 64, 'mariadb' : 64, } -json_flavors = {'postgresql', 'timescaledb', 'citus'} +json_flavors = {'postgresql', 'timescaledb', 'citus', 'sqlite', 'cockroachdb'} OMIT_NULLSFIRST_FLAVORS = {'mariadb', 'mysql', 'mssql'} DB_TO_PD_DTYPES = { 'FLOAT': 'float64', @@ -943,6 +943,6 @@ def get_null_replacement(typ: str, flavor: str) -> str: return '0' if 'time' in typ.lower() or 'date' in typ.lower(): return dateadd_str(flavor=flavor, begin='1900-01-01') - if 'float' in typ.lower(): + if 'float' in typ.lower() or 'double' in typ.lower(): return '-987654321.0' - return ('n' if flavor == 'oracle' else '') + "''" + return ('n' if flavor == 'oracle' else '') + "'-987654321'" diff --git a/meerschaum/utils/venv/__init__.py b/meerschaum/utils/venv/__init__.py index 71e5c1b9..45f087f0 100644 --- a/meerschaum/utils/venv/__init__.py +++ b/meerschaum/utils/venv/__init__.py @@ -368,7 +368,7 @@ def init_venv( verified_venvs.add(venv) return True - import sys, platform, os, pathlib + import sys, platform, os, pathlib, shutil from meerschaum.config._paths import VIRTENV_RESOURCES_PATH from meerschaum.utils.packages import run_python_package, attempt_import global tried_virtualenv @@ -395,7 +395,7 @@ def init_venv( venv=None, debug=debug ) == 0 if not _venv_success: - print("Please install python3-venv! Falling back to virtualenv...") + print(f"Please install python3-venv.\n{f.getvalue()}\nFalling back to virtualenv...") if not venv_exists(venv, debug=debug): _venv = None if not _venv_success: @@ -422,10 +422,15 @@ def init_venv( local_bin_path = VIRTENV_RESOURCES_PATH / venv / 'local' / 'bin' bin_path = VIRTENV_RESOURCES_PATH / venv / 'bin' vtp = venv_target_path(venv=venv, allow_nonexistent=True, debug=debug) + if bin_path.exists(): + try: + shutil.rmtree(bin_path) + except Exception as e: + import traceback + traceback.print_exc() virtualenv.cli_run([str(venv_path)]) if dist_packages_path.exists(): vtp.mkdir(exist_ok=True, parents=True) - import shutil for file_path in dist_packages_path.glob('*'): shutil.move(file_path, vtp) shutil.rmtree(dist_packages_path) @@ -559,9 +564,29 @@ def venv_target_path( """ import os, sys, platform, pathlib, site from meerschaum.config._paths import VIRTENV_RESOURCES_PATH + from meerschaum.config.static import STATIC_CONFIG ### Check sys.path for a user-writable site-packages directory. if venv is None: + + ### Return the known value for the portable environment. + environment_runtime = STATIC_CONFIG['environment']['runtime'] + if os.environ.get(environment_runtime, None) == 'portable': + python_version_folder = ( + 'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor) + ) + executable_path = pathlib.Path(sys.executable) + site_packages_path = ( + ( + executable_path.parent.parent / 'lib' / python_version_folder / 'site-packages' + ) if platform.system() != 'Windows' else ( + executable_path.parent / 'Lib' / 'site-packages' + ) + ) + if not site_packages_path.exists(): + raise EnvironmentError(f"Could not find '{site_packages_path}'. Does it exist?") + return site_packages_path + if not inside_venv(): site_path = pathlib.Path(site.getusersitepackages()) ### Allow for dist-level paths (running as root). diff --git a/requirements/api.txt b/requirements/api.txt index dd084561..6a5eac82 100644 --- a/requirements/api.txt +++ b/requirements/api.txt @@ -13,7 +13,6 @@ pandas>=1.3.0 pytz>=2022.1.0 joblib>=0.17.0 SQLAlchemy>=1.4.42 -sqlalchemy-utils>=0.38.3 databases>=0.4.0 aiosqlite>=0.16.0 asyncpg>=0.21.0 diff --git a/requirements/full.txt b/requirements/full.txt index 3eced601..b9c79f87 100644 --- a/requirements/full.txt +++ b/requirements/full.txt @@ -48,7 +48,6 @@ pandas>=1.3.0 pytz>=2022.1.0 joblib>=0.17.0 SQLAlchemy>=1.4.42 -sqlalchemy-utils>=0.38.3 databases>=0.4.0 aiosqlite>=0.16.0 asyncpg>=0.21.0 diff --git a/scripts/portable/build.sh b/scripts/portable/build.sh index 5c4ae600..7884c72d 100755 --- a/scripts/portable/build.sh +++ b/scripts/portable/build.sh @@ -23,12 +23,12 @@ else fi declare -A urls -# urls["WINDOWS"]="https://github.com/indygreg/python-build-standalone/releases/download/20210506/cpython-3.9.5-x86_64-pc-windows-msvc-shared-pgo-20210506T0943.tar.zst" -urls["WINDOWS"]="https://github.com/indygreg/python-build-standalone/releases/download/20211017/cpython-3.9.7-x86_64-pc-windows-msvc-shared-pgo-20211017T1616.tar.zst" -# urls["LINUX"]="https://github.com/indygreg/python-build-standalone/releases/download/20210506/cpython-3.9.5-x86_64-unknown-linux-gnu-pgo+lto-20210506T0943.tar.zst" -urls["LINUX"]="https://github.com/indygreg/python-build-standalone/releases/download/20211017/cpython-3.9.7-x86_64-unknown-linux-gnu-pgo-20211017T1616.tar.zst" -# urls["MACOS"]="https://github.com/indygreg/python-build-standalone/releases/download/20210103/cpython-3.9.1-x86_64-apple-darwin-pgo-20210103T1125.tar.zst" -urls["MACOS"]="https://github.com/indygreg/python-build-standalone/releases/download/20211017/cpython-3.9.7-x86_64-apple-darwin-pgo+lto-20211017T1616.tar.zst" +# urls["WINDOWS"]="https://github.com/indygreg/python-build-standalone/releases/download/20211017/cpython-3.9.7-x86_64-pc-windows-msvc-shared-pgo-20211017T1616.tar.zst" +urls["WINDOWS"]="https://github.com/indygreg/python-build-standalone/releases/download/20221106/cpython-3.9.15+20221106-x86_64-pc-windows-msvc-shared-pgo-full.tar.zst" +# urls["LINUX"]="https://github.com/indygreg/python-build-standalone/releases/download/20211017/cpython-3.9.7-x86_64-unknown-linux-gnu-pgo-20211017T1616.tar.zst" +urls["LINUX"]="https://github.com/indygreg/python-build-standalone/releases/download/20221106/cpython-3.9.15+20221106-x86_64-unknown-linux-gnu-pgo+lto-full.tar.zst" +# urls["MACOS"]="https://github.com/indygreg/python-build-standalone/releases/download/20211017/cpython-3.9.7-x86_64-apple-darwin-pgo+lto-20211017T1616.tar.zst" +urls["MACOS"]="https://github.com/indygreg/python-build-standalone/releases/download/20221106/cpython-3.9.15+20221106-x86_64-apple-darwin-pgo+lto-full.tar.zst" urls["get-pip.py"]="https://bootstrap.pypa.io/get-pip.py" declare -A tars