v1.3.4 ✨ Custom environment connectors, bugfixes, + more (#81)

* Change mutable default argument to `None`.

* Set to empty list if `keys` is `None`.

* Return if the given config is not a dictionary.

* Ensure `parsed_config` is a dictionary.

* Bump to version 1.3.2

* Adding parameter editing to dashboard

* v1.3.3 Added params editor, fixed broken packages, and more.

* Bump sqlalchemy version

* ✅ v1.3.4 Environment connectors improvements, bugfixes, + more.

* 🐛 Replaced as many instances of `.copy()` with `copy.deepcopy()` as possible.

meerschaum/__main__.py

@@ -19,12 +19,12 @@
 limitations under the License.
 """
 
-import sys, os
+import sys, os, copy
 
 def main(sysargs: list = None) -> None:
     """Main CLI entry point."""
     if sysargs is None:
-        sysargs = sys.argv[1:].copy()
+        sysargs = copy.deepcopy(sys.argv[1:])
     old_cwd = os.getcwd()
 
     ### Catch help flags.
@@ -63,13 +63,15 @@ def main(sysargs: list = None) -> None:
 
     return _exit(rc, old_cwd=old_cwd)
 
+
 def _exit(return_code: int = 0, old_cwd: str = None) -> None:
     _close_pools()
     ### Restore the previous working directory.
     if old_cwd is not None and old_cwd != os.getcwd():
         os.chdir(old_cwd)
     sys.exit(return_code)
 
+
 def _close_pools():
     """Close multiprocessing pools before exiting."""
     ### Final step: close global pools.

meerschaum/_internal/gui/app/_windows.py

@@ -21,6 +21,7 @@ def get_windows(**kw) -> Dict[str, toga.window.Window]:
 def get_main_window(instance: Optional[str], debug: bool = False, **kw) -> toga.window.Window:
     from meerschaum.config.static import _static_config
     from meerschaum.utils.misc import get_connector_labels
+    from meerschaum.connectors import instance_types
     from meerschaum._internal.gui.app.pipes import build_pipes_tree
 
     main_window = toga.MainWindow(title=_static_config()['setup']['formal_name'], size=(1280, 720))
@@ -29,7 +30,7 @@ def get_main_window(instance: Optional[str], debug: bool = False, **kw) -> toga.
 
     left_box = toga.Box(children=[
         toga.Box(children=[
-            toga.Selection(items=get_connector_labels('sql', 'api'), style=toga.style.Pack(flex=1)),
+            toga.Selection(items=get_connector_labels(*instance_types), style=toga.style.Pack(flex=1)),
             tree,
             toga.Button('Hello, world!', style=toga.style.Pack(flex=1, padding=10), on_press=show_test_window),
         ], style=toga.style.Pack(flex=1, padding=10, direction='column', width=200))

meerschaum/_internal/shell/Shell.py

@@ -7,6 +7,7 @@
 
 from __future__ import annotations
 import os
+from copy import deepcopy
 from meerschaum.utils.typing import Union, SuccessTuple, Any, Callable, Optional, List, Dict
 from meerschaum.utils.packages import attempt_import
 from meerschaum.config import __doc__, __version__ as version, get_config
@@ -412,8 +413,7 @@ def precmd(self, line: str):
         old_cwd = os.getcwd()
 
         ### make a backup of line for later
-        import copy
-        original_line = copy.deepcopy(line)
+        original_line = deepcopy(line)
 
         ### cmd2 support: check if command exists
         try:
@@ -832,9 +832,6 @@ def bottom_toolbar():
             colored(shell.repo_keys, 'on ' + get_config('shell', 'ansi', 'repo', 'rich', 'style'))
             if ANSI else colored(shell.repo_keys, 'on white')
         )
-        #  connected = (
-            #  is_connected(shell.instance_keys) if shell._update_bottom_toolbar else last_connected
-        #  )
         try:
             typ, label = shell.instance_keys.split(':')
             connected = typ in connectors and label in connectors[typ]

meerschaum/actions/api.py

@@ -142,6 +142,7 @@ def _api_start(
     from meerschaum.utils.pool import get_pool
     import shutil
     import os
+    from copy import deepcopy
 
     if action is None:
         action = []
@@ -156,7 +157,7 @@ def _api_start(
     uvicorn_config_path = API_UVICORN_RESOURCES_PATH / SERVER_ID / 'config.json'
     uvicorn_env_path = API_UVICORN_RESOURCES_PATH / SERVER_ID / 'uvicorn.env'
 
-    api_config = get_config('system', 'api').copy()
+    api_config = deepcopy(get_config('system', 'api'))
     cf = _config()
     uvicorn_config = api_config['uvicorn']
     if port is None:

meerschaum/actions/arguments/_parser.py

@@ -53,13 +53,14 @@ def parse_help(sysargs : Union[List[str], Dict[str, Any]]) -> None:
     from meerschaum.actions.arguments._parse_arguments import parse_arguments, parse_line
     from meerschaum.actions import actions, get_subactions
     import importlib, inspect, textwrap
+    from copy import deepcopy
     if isinstance(sysargs, list):
         args = parse_arguments(sysargs)
     elif isinstance(sysargs, dict):
         args = sysargs
     elif isinstance(sysargs, str):
         args = parse_line(sysargs)
-    _args = args.copy()
+    _args = deepcopy(args)
     del _args['action']
     if len(args['action']) == 0:
         return actions['show'](['help'], **_args)

meerschaum/actions/install.py

@@ -196,19 +196,23 @@ def _install_required(
         `install required noaa covid`
     """
     from meerschaum.core import Plugin
-    from meerschaum.utils.warnings import warn
+    from meerschaum.utils.warnings import warn, info
     from meerschaum.connectors.parse import parse_repo_keys
     from meerschaum.utils.formatting import print_tuple
+    from meerschaum.plugins import get_plugins_names
     repo_connector = parse_repo_keys(repository)
 
+    plugins_names = action or get_plugins_names()
+
     success_count = 0
     fail_count = 0
 
-    for plugin_name in action:
+    for plugin_name in plugins_names:
         plugin = Plugin(plugin_name, repo_connector=repo_connector)
         if not plugin.is_installed():
             warn(f"Plugin '{plugin}' is not installed. Skipping...", stack=False)
             continue
+        info(f"Installing required packages for plugin '{plugin}'...")
         success = plugin.install_dependencies(force=force, debug=debug)
         if not success:
             warn(f"Failed to install required packages for plugin '{plugin}'.", stack=False)

meerschaum/actions/sync.py

@@ -62,7 +62,7 @@ def _pipes_lap(
     import queue
     import multiprocessing
     import contextlib
-    import time, os
+    import time, os, copy
     from meerschaum.utils.packages import venv_exec
     from meerschaum.utils.process import poll_process
     import json
@@ -72,7 +72,7 @@ def _pipes_lap(
     rich_table, rich_text, rich_box = attempt_import(
         'rich.table', 'rich.text', 'rich.box',
     )
-    all_kw = kw.copy()
+    all_kw = copy.deepcopy(kw)
     all_kw.update({'workers': workers, 'debug': debug, 'unblock': unblock, 'force': force,
         'min_seconds': min_seconds, 'timeout_seconds': timeout_seconds,
         'mrsm_instance': mrsm_instance,})

meerschaum/api/dash/actions.py

@@ -7,7 +7,7 @@
 """
 
 from __future__ import annotations
-import platform, sys, io, os, shlex, time
+import platform, sys, io, os, shlex, time, copy
 from dash.exceptions import PreventUpdate
 from meerschaum.utils.threading import Thread
 from meerschaum.utils.typing import SuccessTuple, Tuple, Dict, Any, WebState
@@ -200,7 +200,7 @@ def do_process():
             ### Don't forget to pass the existing environment
             ### in case MRSM_CONFIG and MRSM_PATCH are set.
             try:
-                _env = os.environ.copy()
+                _env = copy.deepcopy(os.environ)
             except Exception as e:
                 _env = {}
             _env.update({'LINES': '120', 'COLUMNS': '100'})

meerschaum/api/dash/components.py

@@ -15,6 +15,7 @@
 from meerschaum.utils.misc import remove_ansi
 from meerschaum.actions import get_shell
 from meerschaum.api import endpoints, CHECK_UPDATE
+from meerschaum.connectors import instance_types
 from meerschaum.utils.misc import get_connector_labels
 from meerschaum.config import __doc__ as doc
 dbc = attempt_import('dash_bootstrap_components', lazy=False, check_update=CHECK_UPDATE)
@@ -92,7 +93,7 @@
     size = 'sm',
     options = [
         {'label': i, 'value': i}
-        for i in get_connector_labels('sql', 'api')
+        for i in get_connector_labels(*instance_types)
     ],
     class_name = 'dbc_dark custom-select custom-select-sm',
 )
@@ -102,13 +103,34 @@
     [
         dbc.Row(
             [
-                dbc.Col(html.Img(src=endpoints['dash'] + "/assets/logo_48x48.png", style = {'padding': '0.5em', 'padding-left': '2em'}), width='auto', align='start'),
-                dbc.Col(dbc.NavbarBrand("Meerschaum Web Console", class_name='ms-2', style={'margin-top': '10px', 'display': 'inline-block'}), align='start', width=2),
+                dbc.Col(
+                    html.Img(
+                        src = endpoints['dash'] + "/assets/logo_48x48.png",
+                        style = {'padding': '0.5em', 'padding-left': '2em'}
+                    ),
+                    width = 'auto', 
+                    align = 'start'
+                ),
+                dbc.Col(
+                    dbc.NavbarBrand(
+                        "Meerschaum Web Console",
+                        class_name = 'ms-2',
+                        style = {'margin-top': '10px', 'display': 'inline-block'}
+                    ), align='start', width=2
+                ),
                 dbc.Col(md=True, lg=True, sm=False),
-                dbc.Col(html.Center(instance_select), sm=2, md=2, lg=1, align='end', class_name='d-flex justify-content-center text-center'),
+                dbc.Col(
+                    html.Center(
+                        instance_select
+                    ),
+                    sm = 2,
+                    md = 2,
+                    lg = 1,
+                    align = 'end',
+                    class_name = 'd-flex justify-content-center text-center'
+                ),
                 dbc.Col(html.Pre(html.A(doc, href='/docs')), width='auto', align='end'),
             ],
-            #  align = 'center',
             style = {'width': '100%'},
             justify = 'around',
         ),
@@ -117,7 +139,6 @@
 )
 
 
-
 def alert_from_success_tuple(success: SuccessTuple) -> dbc.Alert:
     """
     Return a `dbc.Alert` from a `SuccessTuple`.
@@ -133,6 +154,7 @@ def alert_from_success_tuple(success: SuccessTuple) -> dbc.Alert:
         )
     )
 
+
 def build_cards_grid(cards: List[dbc.Card], num_columns: int = 3) -> html.Div:
     """
     Because `CardColumns` were removed in Bootstrap 5, this function recreates a similar effect.
@@ -152,4 +174,3 @@ def build_cards_grid(cards: List[dbc.Card], num_columns: int = 3) -> html.Div:
         rows.append(r)
         rows.append(html.Br())
     return html.Div(rows)
-

meerschaum/api/dash/pages/dashboard.py

@@ -64,21 +64,15 @@
                             ),
                             html.Br(),
                             bottom_buttons_content,
-                            #  action_row,
                             test_button,
-                            #  go_button,
-                            #  show_pipes_button,
-                            #  get_items_menu,
                             html.Div(id='ws-div'),
-                            #  search_parameters_editor,
                         ],
                         id = 'content-col-left',
                         md=12, lg=6,
                     ),
                     dbc.Col(
                         children = [
                             dbc.Col([
-                                ### Place alert divs here.
                                     html.Div(id='success-alert-div'),
                                     html.Div(id='instance-alert-div')
                                 ],

meerschaum/config/__init__.py

@@ -9,7 +9,7 @@
 
 from __future__ import annotations
 
-import os, shutil, sys, pathlib
+import os, shutil, sys, pathlib, copy
 from meerschaum.utils.typing import Any, Dict, Optional, Union
 from meerschaum.utils.threading import RLock
 from meerschaum.utils.warnings import warn
@@ -196,7 +196,7 @@ def get_config(
             in_default = True
             patched_default_config = (
                 search_and_substitute_config(default_config)
-                if substitute else default_config.copy()
+                if substitute else copy.deepcopy(default_config)
             )
             _c = patched_default_config
             for k in keys:

meerschaum/config/_environment.py

@@ -8,6 +8,7 @@
 
 import os
 import re
+import json
 from meerschaum.utils.typing import List, Union, Dict, Any
 from meerschaum.config.static import STATIC_CONFIG
 
@@ -82,6 +83,17 @@ def apply_environment_uris() -> None:
         apply_connector_uri(env_var)
 
 
+def get_connector_env_regex() -> str:
+    """
+    Return the regex pattern for valid environment variable names for instance connectors.
+    """
+    from meerschaum.connectors import connectors, load_plugin_connectors
+    load_plugin_connectors()
+    return STATIC_CONFIG['environment']['uri_regex'].replace(
+        '{TYPES}', '|'.join([typ.upper() for typ in connectors if typ != 'plugin'])
+    )
+
+
 def get_connector_env_vars() -> List[str]:
     """
     Get the names of the environment variables which match the Meerschaum connector regex.
@@ -91,7 +103,7 @@ def get_connector_env_vars() -> List[str]:
     >>> get_connector_environment_vars()
     ['MRSM_SQL_FOO']
     """
-    uri_regex = STATIC_CONFIG['environment']['uri_regex']
+    uri_regex = get_connector_env_regex()
     env_vars = []
     for env_var in os.environ:
         matched = re.match(uri_regex, env_var)
@@ -105,23 +117,31 @@ def apply_connector_uri(env_var: str) -> None:
     """
     Parse and validate a URI obtained from an environment variable.
     """
-    from meerschaum.connectors import get_connector
     from meerschaum.config import get_config, set_config, _config
     from meerschaum.config._patch import apply_patch_to_config
     from meerschaum.config._read_config import search_and_substitute_config
-    uri_regex = STATIC_CONFIG['environment']['uri_regex']
+    from meerschaum.utils.warnings import warn
+    uri_regex = get_connector_env_regex()
     matched = re.match(uri_regex, env_var)
     groups = matched.groups()
     typ, label = groups[0].lower(), groups[1].lower()
     uri = os.environ[env_var]
+    if uri.startswith('{') and uri.endswith('}'):
+        try:
+            conn_attrs = json.loads(uri)
+        except Exception as e:
+            warn(f"Unable to parse JSON for environment connector '{typ}:{label}'.")
+            conn_attrs = {'uri': uri}
+    else:
+        conn_attrs = {'uri': uri}
     cf = _config()
     if 'meerschaum' not in cf:
         cf['meerschaum'] = {}
     if 'connectors' not in cf['meerschaum']:
         cf['meerschaum']['connectors'] = {}
     if typ not in cf['meerschaum']['connectors']:
         cf['meerschaum']['connectors'][typ] = {}
-    cf['meerschaum']['connectors'][typ][label] = {'uri': uri}
+    cf['meerschaum']['connectors'][typ][label] = conn_attrs
     #  set_config(
         #  apply_patch_to_config(
             #  {'meerschaum': get_config('meerschaum')},

meerschaum/config/_patch.py

@@ -7,14 +7,17 @@
 """
 
 import sys
+import copy
 from meerschaum.utils.typing import Dict, Any
 
 def apply_patch_to_config(
         config: Dict[str, Any],
         patch: Dict[str, Any],
     ):
     """Patch the config dict with a new dict (cascade patching)."""
-    _base = config.copy()
+    _base = copy.deepcopy(config) if isinstance(config, dict) else {}
+    if not isinstance(patch, dict):
+        return config
 
     def update_dict(base, patch):
         for key, value in patch.items():

meerschaum/config/_version.py

@@ -2,4 +2,4 @@
 Specify the Meerschaum release version.
 """
 
-__version__ = "1.3.3"
+__version__ = "1.3.4"

meerschaum/config/static/__init__.py

@@ -43,7 +43,7 @@
         'plugins': 'MRSM_PLUGINS_DIR',
         'runtime': 'MRSM_RUNTIME',
         'id': 'MRSM_SERVER_ID',
-        'uri_regex': r'MRSM_(SQL|API)_(\d*[a-zA-Z][a-zA-Z0-9-_+]*$)',
+        'uri_regex': 'MRSM_({TYPES})_(\d*[a-zA-Z][a-zA-Z0-9-_+]*$)',
         'prefix': 'MRSM_',
     },
     'config': {