diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index 8999d3fe..f0ae16da 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -18,7 +18,7 @@ jobs: build_and_test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Setup dev tools env: MRSM_SKIP_DOCKER_EXPERIMENTAL: 1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 86ad15ac..ae350a39 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,116 @@ This is the current release cycle, so stay tuned for future releases! +### v2.2.2 + +- **Speed up package installation in virtual environments.** + Dynamic dependencies will now be installed via `uv`, which dramatically speeds up installation times. + +- **Add sub-cards for children pipes.** + Pipes with `children` defined now include cards for these pipes under the Parameters menu item. This is especially useful when working managing pipeline hierarchies. + +- **Add "Open in Python" to pipe cards.** + Clicking "Open in Python" on a pipe's card will now launch `ptpython` with the pipe object already created. + + ```python + # Clicking "Open in Python" executes the following: + # $ mrsm python "pipe = mrsm.Pipe('plugin:noaa', 'weather', 'gvl', instance='sql:main')" + >>> import meerschaum as mrsm + >>> pipe = mrsm.Pipe('plugin:noaa', 'weather', 'gvl', instance='sql:main') + ``` + +- **Add the decorators `@web_page` and `@dash_plugin`.** + You may now quickly add your own pages to the web console by decorating your layout functions with `@web_page`: + + ```python + # example.py + from meerschaum.plugins import dash_plugin, web_page + + @dash_plugin + def init_dash(dash_app): + + import dash.html as html + import dash_bootstrap_components as dbc + from dash import Input, Output, no_update + + @web_page('/my-page', login_required=False) + def my_page(): + return dbc.Container([ + html.H1("Hello, World!"), + dbc.Button("Click me", id='my-button'), + html.Div(id="my-output-div"), + ]) + + @dash_app.callback( + Output('my-output-div', 'children'), + Input('my-button', 'n_clicks'), + ) + def my_button_click(n_clicks): + if not n_clicks: + return no_update + return html.P(f'You clicked {n_clicks} times!') + ``` + +- **Use `ptpython` for the `python` action.** + Rather than opening a classic REPL, the `python` action will now open a `ptpython` shell. + +- **Add `--venv` to the `python` action.** + Launching a Python REPL with `mrsm python` will now default to `--venv mrsm`. Run `mrsm install package` to make packages importable. + + ```python + # $ mrsm python + >>> import requests + Traceback (most recent call last): + File "", line 1, in + ModuleNotFoundError: No module named 'requests' + + # $ mrsm install package requests + >>> import requests + >>> requests.__file__ + '/meerschaum/venvs/mrsm/lib/python3.12/site-packages/requests/__init__.py' + + # $ mrsm install plugin noaa + # $ mrsm python --venv noaa + >>> import requests + >>> requests.__file__ + '/meerschaum/venvs/noaa/lib/python3.12/site-packages/requests/__init__.py' + ``` + +- **Allow passing flags to venv `ptpython` binaries.** + You may now pass flags directly to the `ptpython` binary of a virtual environment (by escaping with `[]`): + + ```bash + mrsm python [--help] + ``` + +- **Allow for custom connectors to implement a `sync()` method.** + Like module-level `sync()` functions for `plugin` connectors, any custom connector may implement `sync()` instead of `fetch()`. + + ```python + # example.py + from typing import Any + import meerschaum as mrsm + from meerschaum.connectors import Connector, make_connector + + @make_connector + class ExampleConnector(Connector): + + def register(self, pipe: mrsm.Pipe) -> dict[str, Any]: + return { + 'columns': { + 'datetime': 'ts', + 'id': 'example_id', + }, + } + + def sync(self, pipe: mrsm.Pipe, **kwargs) -> mrsm.SuccessTuple: + ### Implement a custom sync. + return True, f"Successfully synced {pipe}!" + ``` + +- **Install `uvicorn` and `gunicorn` in virtual environments.** + The packages `uvicorn` and `gunicorn` are now installed into the default virtual environment. + ### v2.2.1 - **Fix `--schedule` in the interactive shell.** diff --git a/Dockerfile b/Dockerfile index 695f383b..090d5b3d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,7 +30,8 @@ WORKDIR $MRSM_WORK_DIR ### Layer 2: Install Python packages. ### Only rebuilds cache if dependencies have changed. COPY --chown=$MRSM_USER:$MRSM_USER requirements $MRSM_HOME/requirements -RUN python -m pip install --user --no-cache-dir -r $MRSM_HOME/requirements/$MRSM_DEP_GROUP.txt && \ +RUN python -m pip install --user --no-cache-dir \ + -r $MRSM_HOME/requirements/$MRSM_DEP_GROUP.txt && \ rm -rf $MRSM_HOME/requirements ### Layer 3: Install Meerschaum. diff --git a/docker-compose.yaml b/docker-compose.yaml index ce091f99..28c19170 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,6 +1,8 @@ services: mrsm-dev: - image: "bmeares/meerschaum" + image: "bmeares/meerschaum:minimal" + # image: ubuntu + # image: python entrypoint: ["/scripts/dev.sh"] network_mode: "host" init: true diff --git a/docs/mkdocs/news/changelog.md b/docs/mkdocs/news/changelog.md index 86ad15ac..ae350a39 100644 --- a/docs/mkdocs/news/changelog.md +++ b/docs/mkdocs/news/changelog.md @@ -4,6 +4,116 @@ This is the current release cycle, so stay tuned for future releases! +### v2.2.2 + +- **Speed up package installation in virtual environments.** + Dynamic dependencies will now be installed via `uv`, which dramatically speeds up installation times. + +- **Add sub-cards for children pipes.** + Pipes with `children` defined now include cards for these pipes under the Parameters menu item. This is especially useful when working managing pipeline hierarchies. + +- **Add "Open in Python" to pipe cards.** + Clicking "Open in Python" on a pipe's card will now launch `ptpython` with the pipe object already created. + + ```python + # Clicking "Open in Python" executes the following: + # $ mrsm python "pipe = mrsm.Pipe('plugin:noaa', 'weather', 'gvl', instance='sql:main')" + >>> import meerschaum as mrsm + >>> pipe = mrsm.Pipe('plugin:noaa', 'weather', 'gvl', instance='sql:main') + ``` + +- **Add the decorators `@web_page` and `@dash_plugin`.** + You may now quickly add your own pages to the web console by decorating your layout functions with `@web_page`: + + ```python + # example.py + from meerschaum.plugins import dash_plugin, web_page + + @dash_plugin + def init_dash(dash_app): + + import dash.html as html + import dash_bootstrap_components as dbc + from dash import Input, Output, no_update + + @web_page('/my-page', login_required=False) + def my_page(): + return dbc.Container([ + html.H1("Hello, World!"), + dbc.Button("Click me", id='my-button'), + html.Div(id="my-output-div"), + ]) + + @dash_app.callback( + Output('my-output-div', 'children'), + Input('my-button', 'n_clicks'), + ) + def my_button_click(n_clicks): + if not n_clicks: + return no_update + return html.P(f'You clicked {n_clicks} times!') + ``` + +- **Use `ptpython` for the `python` action.** + Rather than opening a classic REPL, the `python` action will now open a `ptpython` shell. + +- **Add `--venv` to the `python` action.** + Launching a Python REPL with `mrsm python` will now default to `--venv mrsm`. Run `mrsm install package` to make packages importable. + + ```python + # $ mrsm python + >>> import requests + Traceback (most recent call last): + File "", line 1, in + ModuleNotFoundError: No module named 'requests' + + # $ mrsm install package requests + >>> import requests + >>> requests.__file__ + '/meerschaum/venvs/mrsm/lib/python3.12/site-packages/requests/__init__.py' + + # $ mrsm install plugin noaa + # $ mrsm python --venv noaa + >>> import requests + >>> requests.__file__ + '/meerschaum/venvs/noaa/lib/python3.12/site-packages/requests/__init__.py' + ``` + +- **Allow passing flags to venv `ptpython` binaries.** + You may now pass flags directly to the `ptpython` binary of a virtual environment (by escaping with `[]`): + + ```bash + mrsm python [--help] + ``` + +- **Allow for custom connectors to implement a `sync()` method.** + Like module-level `sync()` functions for `plugin` connectors, any custom connector may implement `sync()` instead of `fetch()`. + + ```python + # example.py + from typing import Any + import meerschaum as mrsm + from meerschaum.connectors import Connector, make_connector + + @make_connector + class ExampleConnector(Connector): + + def register(self, pipe: mrsm.Pipe) -> dict[str, Any]: + return { + 'columns': { + 'datetime': 'ts', + 'id': 'example_id', + }, + } + + def sync(self, pipe: mrsm.Pipe, **kwargs) -> mrsm.SuccessTuple: + ### Implement a custom sync. + return True, f"Successfully synced {pipe}!" + ``` + +- **Install `uvicorn` and `gunicorn` in virtual environments.** + The packages `uvicorn` and `gunicorn` are now installed into the default virtual environment. + ### v2.2.1 - **Fix `--schedule` in the interactive shell.** diff --git a/docs/mkdocs/reference/plugins/writing-plugins.md b/docs/mkdocs/reference/plugins/writing-plugins.md index edf7dfe2..53b8d9c5 100644 --- a/docs/mkdocs/reference/plugins/writing-plugins.md +++ b/docs/mkdocs/reference/plugins/writing-plugins.md @@ -94,6 +94,8 @@ Plugins are just modules with functions. This section explains the roles of the Create new commands. - **`#!python @api_plugin`** Create new FastAPI endpoints. +- **`#!python @dash_plugin` and `#!python @web_page`** + Create new Dash pages on the Web Console. - **`#!python @pre_sync_hook` and `#!python @post_sync_hook`** Inject callbacks when pipes are synced by the `sync pipes` action. - **`#!python setup(**kwargs)`** @@ -445,6 +447,89 @@ For your endpoints, arguments will be used as HTTP parameters, and to require th ![Custom Meerschaum API endpoint which requires a login.](/assets/screenshots/api-plugin-endpoint-login-required.png) +### **The `#!python @dash_plugin` and `#!python @web_page` Decorators** + +Add new [Plotly Dash](https://dash.plotly.com/) pages to the Web Console with `#!python @dash_plugin` and `#!python @web_page`. + +Like `#!python @api_plugin`, `#!python @dash_app` designates an initialization function which accepts the Dash app. Within this function, you can create callbacks with `#!python @dash_app.callback()`. + +!!! note inline end "" + + `#!python @web_page` may be used with or without arguments. When invoked without arguments, it will use the name of the function as the path string. + +Functions decorated with `#!python @web_page` return a [Dash layout](https://dash.plotly.com/layout). These don't need to reside within the `#!python @dash_app`-decorated function, but this is recommended to improve lazy-loading performance. + +??? example "Dash Plugin Example" + + ```python + ### ~/.config/meerschaum/plugins/example.py + + from meerschaum.plugins import dash_plugin, web_page + + @dash_plugin + def init_dash(dash_app): + + import dash.html as html + import dash_bootstrap_components as dbc + from dash import Input, Output, no_update + + ### Routes to '/dash/my-page' + @web_page('/my-page', login_required=False) + def my_page(): + return dbc.Container([ + html.H1("Hello, World!"), + dbc.Button("Click me", id='my-button'), + html.Div(id="my-output-div"), + ]) + + @dash_app.callback( + Output('my-output-div', 'children'), + Input('my-button', 'n_clicks'), + ) + def my_button_click(n_clicks): + if not n_clicks: + return no_update + return html.P(f'You clicked {n_clicks} times!') + ``` + +??? tip "Dynamic Routing" + + Sub-paths will be routed to your base callback, e.g. `/items/apple` will be accepted by `#!python @web_page('/items')`. You can then parse the path string with [`dcc.Location`](https://dash.plotly.com/dash-core-components/location). + + ```python + from meerschaum.plugins import web_page, dash_plugin + + @dash_plugin + def init_dash(dash_app): + + import dash.html as html + import dash.dcc as dcc + from dash import Input, Output, State, no_update + import dash_bootstrap_components as dbc + + ### Omitting arguments will default to the path '/dash/items'. + ### `login_required` is `True` by default. + @web_page + def items(): + return dbc.Container([ + dcc.Location(id='my-location'), + html.Div(id='my-output-div'), + ]) + + @dash_app.callback( + Output('my-output-div', 'children'), + Input('my-location', 'pathname'), + ) + def my_button_click(pathname): + if not str(pathname).startswith('/dash/items'): + return no_update + + item_id = pathname.replace('/dash/items', '').lstrip('/').rstrip('/').split('/')[0] + if not item_id: + return html.P("You're looking at the base /items page.") + + return html.P(f"You're looking at item '{item_id}'.") + ``` ### **The `#!python @pre_sync_hook` and `#!python @post_sync_hook` Decorators** diff --git a/meerschaum/_internal/shell/Shell.py b/meerschaum/_internal/shell/Shell.py index 7ebdacd9..c25f0ada 100644 --- a/meerschaum/_internal/shell/Shell.py +++ b/meerschaum/_internal/shell/Shell.py @@ -203,6 +203,29 @@ def _check_complete_keys(line: str) -> Optional[List[str]]: return None +def get_shell_intro(with_color: bool = True) -> str: + """ + Return the introduction message string. + """ + from meerschaum.utils.formatting import CHARSET, ANSI, colored + intro = get_config('shell', CHARSET, 'intro', patch=patch) + intro += '\n' + ''.join( + [' ' + for i in range( + string_width(intro) - len('v' + version) + ) + ] + ) + 'v' + version + + if not with_color or not ANSI: + return intro + + return colored( + intro, + **get_config('shell', 'ansi', 'intro', 'rich') + ) + + class Shell(cmd.Cmd): def __init__( self, @@ -277,25 +300,20 @@ def __init__( except Exception as e: pass - def load_config(self, instance: Optional[str] = None): """ Set attributes from the shell configuration. """ from meerschaum.utils.misc import remove_ansi - from meerschaum.utils.formatting import CHARSET, ANSI, UNICODE, colored + from meerschaum.utils.formatting import CHARSET, ANSI, colored if shell_attrs.get('intro', None) != '': - self.intro = get_config('shell', CHARSET, 'intro', patch=patch) - self.intro += '\n' + ''.join( - [' ' - for i in range( - string_width(self.intro) - len('v' + version) - ) - ] - ) + 'v' + version - else: - self.intro = "" + self.intro = ( + get_shell_intro(with_color=False) + if shell_attrs.get('intro', None) != '' + else "" + ) + shell_attrs['intro'] = self.intro shell_attrs['_prompt'] = get_config('shell', CHARSET, 'prompt', patch=patch) self.prompt = shell_attrs['_prompt'] @@ -822,7 +840,7 @@ def input_with_sigint(_input, session, shell: Optional[Shell] = None): """ Replace built-in `input()` with prompt_toolkit.prompt. """ - from meerschaum.utils.formatting import CHARSET, ANSI, UNICODE, colored + from meerschaum.utils.formatting import CHARSET, ANSI, colored from meerschaum.connectors import is_connected, connectors from meerschaum.utils.misc import remove_ansi from meerschaum.config import get_config @@ -849,11 +867,17 @@ def bottom_toolbar(): shell_attrs['instance_keys'], 'on ' + get_config( 'shell', 'ansi', 'instance', 'rich', 'style' ) - ) if ANSI else colored(shell_attrs['instance_keys'], 'on white') + ) + if ANSI + else colored(shell_attrs['instance_keys'], 'on white') ) repo_colored = ( - colored(shell_attrs['repo_keys'], 'on ' + get_config('shell', 'ansi', 'repo', 'rich', 'style')) - if ANSI else colored(shell_attrs['repo_keys'], 'on white') + colored( + shell_attrs['repo_keys'], + 'on ' + get_config('shell', 'ansi', 'repo', 'rich', 'style') + ) + if ANSI + else colored(shell_attrs['repo_keys'], 'on white') ) try: typ, label = shell_attrs['instance_keys'].split(':') diff --git a/meerschaum/_internal/term/__init__.py b/meerschaum/_internal/term/__init__.py index 82b444ed..9659705f 100644 --- a/meerschaum/_internal/term/__init__.py +++ b/meerschaum/_internal/term/__init__.py @@ -14,9 +14,10 @@ from meerschaum.utils.packages import attempt_import from meerschaum._internal.term.TermPageHandler import TermPageHandler from meerschaum.config._paths import API_TEMPLATES_PATH, API_STATIC_PATH +from meerschaum.utils.venv import venv_executable tornado, tornado_ioloop, terminado = attempt_import( - 'tornado', 'tornado.ioloop', 'terminado', lazy=False, venv=None, + 'tornado', 'tornado.ioloop', 'terminado', lazy=False, ) def get_webterm_app_and_manager() -> Tuple[ @@ -31,7 +32,7 @@ def get_webterm_app_and_manager() -> Tuple[ A tuple of the Tornado web application and term manager. """ commands = [ - sys.executable, + venv_executable('mrsm'), '-c', "import os; _ = os.environ.pop('COLUMNS', None); _ = os.environ.pop('LINES', None); " "from meerschaum._internal.entry import get_shell; " diff --git a/meerschaum/_internal/term/tools.py b/meerschaum/_internal/term/tools.py index 431bf014..2e00ba70 100644 --- a/meerschaum/_internal/term/tools.py +++ b/meerschaum/_internal/term/tools.py @@ -17,7 +17,7 @@ def is_webterm_running(host: str, port: int, protocol: str = 'http') -> int: requests = attempt_import('requests') url = f'{protocol}://{host}:{port}' try: - r = requests.get(url) + r = requests.get(url, timeout=3) except Exception as e: return False if not r: diff --git a/meerschaum/actions/api.py b/meerschaum/actions/api.py index 108f594d..397a607f 100644 --- a/meerschaum/actions/api.py +++ b/meerschaum/actions/api.py @@ -156,6 +156,7 @@ def _api_start( from meerschaum.config.static import STATIC_CONFIG, SERVER_ID from meerschaum.connectors.parse import parse_instance_keys from meerschaum.utils.pool import get_pool + from meerschaum.utils.venv import get_module_venv import shutil from copy import deepcopy @@ -169,7 +170,10 @@ def _api_start( ### `check_update` must be False, because otherwise Uvicorn's hidden imports will break things. dotenv = attempt_import('dotenv', lazy=False) uvicorn, gunicorn = attempt_import( - 'uvicorn', 'gunicorn', venv=None, lazy=False, check_update=False, + 'uvicorn', 'gunicorn', + lazy = False, + check_update = False, + venv = 'mrsm', ) uvicorn_config_path = API_UVICORN_RESOURCES_PATH / SERVER_ID / 'config.json' @@ -305,19 +309,51 @@ def _api_start( ### remove custom keys before calling uvicorn def _run_uvicorn(): - try: - uvicorn.run( - **filter_keywords( - uvicorn.run, - **{ - k: v - for k, v in uvicorn_config.items() - if k not in custom_keys - } - ) - ) - except KeyboardInterrupt: - pass + uvicorn_flags = [ + '--host', host, + '--port', str(port), + ( + '--proxy-headers' + if uvicorn_config.get('proxy_headers') + else '--no-proxy-headers' + ), + ( + '--use-colors' + if uvicorn_config.get('use_colors') + else '--no-use-colors' + ), + '--env-file', uvicorn_config['env_file'], + ] + if uvicorn_reload := uvicorn_config.get('reload'): + uvicorn_flags.append('--reload') + if ( + uvicorn_reload + and (reload_dirs := uvicorn_config.get('reload_dirs')) + ): + if not isinstance(reload_dirs, list): + reload_dirs = [reload_dirs] + for reload_dir in reload_dirs: + uvicorn_flags += ['--reload-dir', reload_dir] + if ( + uvicorn_reload + and (reload_excludes := uvicorn_config.get('reload_excludes')) + ): + if not isinstance(reload_excludes, list): + reload_excludes = [reload_excludes] + for reload_exclude in reload_excludes: + uvicorn_flags += ['--reload-exclude', reload_exclude] + if (uvicorn_workers := uvicorn_config.get('workers')) is not None: + uvicorn_flags += ['--workers', str(uvicorn_workers)] + + uvicorn_args = uvicorn_flags + ['meerschaum.api:app'] + run_python_package( + 'uvicorn', + uvicorn_args, + venv = get_module_venv(uvicorn), + as_proc = False, + foreground = True, + debug = debug, + ) def _run_gunicorn(): gunicorn_args = [ @@ -338,23 +374,21 @@ def _run_gunicorn(): ] if debug: gunicorn_args += ['--log-level=debug', '--enable-stdio-inheritance', '--reload'] - try: - run_python_package( - 'gunicorn', - gunicorn_args, - env = { - k: ( - json.dumps(v) - if isinstance(v, (dict, list)) - else v - ) - for k, v in env_dict.items() - }, - venv = None, - debug = debug, - ) - except KeyboardInterrupt: - pass + + run_python_package( + 'gunicorn', + gunicorn_args, + env = { + k: ( + json.dumps(v) + if isinstance(v, (dict, list)) + else v + ) + for k, v in env_dict.items() + }, + venv = get_module_venv(gunicorn), + debug = debug, + ) _run_uvicorn() if not production else _run_gunicorn() diff --git a/meerschaum/actions/python.py b/meerschaum/actions/python.py index 96a5f256..da64f898 100644 --- a/meerschaum/actions/python.py +++ b/meerschaum/actions/python.py @@ -10,37 +10,39 @@ def python( action: Optional[List[str]] = None, + venv: Optional[str] = 'mrsm', + sub_args: Optional[List[str]] = None, debug: bool = False, **kw: Any ) -> SuccessTuple: """ - Launch a Python interpreter with Meerschaum imported. Commands are optional. - Note that quotes must be escaped and commands must be separated by semicolons + Launch a virtual environment's Python interpreter with Meerschaum imported. + You may pass flags to the Python binary by surrounding each flag with `[]`. Usage: `python {commands}` - Example: - `python print(\\'Hello, World!\\'); pipes = mrsm.get_pipes()` - - ``` - Hello, World! - - >>> import meerschaum as mrsm - >>> print('Hello, World!') - >>> pipes = mrsm.get_pipes() - ``` + Examples: + mrsm python + mrsm python --venv noaa + mrsm python [-i] [-c 'print("hi")'] """ - import sys, subprocess + import sys, subprocess, os from meerschaum.utils.debug import dprint from meerschaum.utils.warnings import warn, error - from meerschaum.utils.process import run_process + from meerschaum.utils.venv import venv_executable + from meerschaum.utils.misc import generate_password from meerschaum.config import __version__ as _version + from meerschaum.config.paths import VIRTENV_RESOURCES_PATH, PYTHON_RESOURCES_PATH + from meerschaum.utils.packages import run_python_package, attempt_import if action is None: action = [] - joined_actions = ['import meerschaum as mrsm'] + if venv == 'None': + venv = None + + joined_actions = ["import meerschaum as mrsm"] line = "" for i, a in enumerate(action): if a == '': @@ -51,34 +53,64 @@ def python( line = "" ### ensure meerschaum is imported - # joined_actions = ['import meerschaum as mrsm;'] + joined_actions if debug: - dprint(joined_actions) + dprint(str(joined_actions)) - print_command = 'import sys; print("""' - ps1 = ">>> " + ### TODO: format the pre-executed code using the pygments lexer. + print_command = ( + 'from meerschaum.utils.packages import attempt_import; ' + + 'ptft = attempt_import("prompt_toolkit.formatted_text", lazy=False); ' + + 'pts = attempt_import("prompt_toolkit.shortcuts"); ' + + 'ansi = ptft.ANSI("""' + ) + ps1 = "\\033[1m>>> \\033[0m" for i, a in enumerate(joined_actions): line = ps1 + f"{a}".replace(';', '\n') if '\n' not in line and i != len(joined_actions) - 1: line += "\n" print_command += line - print_command += '""")' + print_command += ( + '"""); ' + + 'pts.print_formatted_text(ansi); ' + ) - command = "" + command = print_command for a in joined_actions: command += a if not a.endswith(';'): command += ';' command += ' ' - command += print_command - if debug: dprint(f"command:\n{command}") + + init_script_path = PYTHON_RESOURCES_PATH / (generate_password(8) + '.py') + with open(init_script_path, 'w', encoding='utf-8') as f: + f.write(command) + + env_dict = os.environ.copy() + venv_path = (VIRTENV_RESOURCES_PATH / venv) if venv is not None else None + if venv_path is not None: + env_dict.update({'VIRTUAL_ENV': venv_path.as_posix()}) + try: - return_code = run_process([sys.executable, '-i', '-c', command], foreground=True) + ptpython = attempt_import('ptpython', venv=venv, allow_outside_venv=False) + return_code = run_python_package( + 'ptpython', + sub_args or ['--dark-bg', '-i', init_script_path.as_posix()], + venv = venv, + foreground = True, + env = env_dict, + ) except KeyboardInterrupt: return_code = 1 + + try: + if init_script_path.exists(): + init_script_path.unlink() + except Exception as e: + warn(f"Failed to clean up tempory file '{init_script_path}'.") + return return_code == 0, ( "Success" if return_code == 0 else f"Python interpreter returned {return_code}." diff --git a/meerschaum/actions/start.py b/meerschaum/actions/start.py index ecbcde9a..a5ceee4a 100644 --- a/meerschaum/actions/start.py +++ b/meerschaum/actions/start.py @@ -333,8 +333,7 @@ def _start_gui( from meerschaum.connectors.parse import parse_instance_keys from meerschaum._internal.term.tools import is_webterm_running import platform - webview = attempt_import('webview') - requests = attempt_import('requests') + webview, requests = attempt_import('webview', 'requests') import json import time @@ -365,7 +364,7 @@ def _start_gui( base_url = 'http://' + api_kw['host'] + ':' + str(api_kw['port']) process = venv_exec( - start_tornado_code, as_proc=True, venv=None, debug=debug, capture_output=(not debug) + start_tornado_code, as_proc=True, debug=debug, capture_output=(not debug) ) timeout = 10 start = time.perf_counter() @@ -446,7 +445,6 @@ def _start_webterm( + " Include `-f` to start another server on a new port\n" + " or specify a different port with `-p`." ) - if not nopretty: info(f"Starting the webterm at http://{host}:{port} ...\n Press CTRL+C to quit.") tornado_app.listen(port, host) diff --git a/meerschaum/actions/uninstall.py b/meerschaum/actions/uninstall.py index 0c21edf4..d2eafec2 100644 --- a/meerschaum/actions/uninstall.py +++ b/meerschaum/actions/uninstall.py @@ -7,7 +7,7 @@ """ from __future__ import annotations -from meerschaum.utils.typing import List, Any, SuccessTuple, Optional +from meerschaum.utils.typing import List, Any, SuccessTuple, Optional, Union def uninstall( action: Optional[List[str]] = None, @@ -145,13 +145,10 @@ def _complete_uninstall_plugins(action: Optional[List[str]] = None, **kw) -> Lis possibilities.append(name) return possibilities -class NoVenv: - pass - def _uninstall_packages( action: Optional[List[str]] = None, sub_args: Optional[List[str]] = None, - venv: Union[str, None, NoVenv] = NoVenv, + venv: Optional[str] = 'mrsm', yes: bool = False, force: bool = False, noask: bool = False, @@ -169,9 +166,7 @@ def _uninstall_packages( from meerschaum.utils.warnings import info from meerschaum.utils.packages import pip_uninstall - - if venv is NoVenv: - venv = 'mrsm' + from meerschaum.utils.misc import items_str if not (yes or force) and noask: return False, "Skipping uninstallation. Add `-y` or `-f` to agree to the uninstall prompt." @@ -183,7 +178,8 @@ def _uninstall_packages( debug = debug, ): return True, ( - f"Successfully removed packages from virtual environment 'mrsm':\n" + ', '.join(action) + f"Successfully removed packages from virtual environment '{venv}':\n" + + items_str(action) ) return False, f"Failed to uninstall packages:\n" + ', '.join(action) diff --git a/meerschaum/actions/upgrade.py b/meerschaum/actions/upgrade.py index 173f3465..adc79e11 100644 --- a/meerschaum/actions/upgrade.py +++ b/meerschaum/actions/upgrade.py @@ -130,7 +130,7 @@ def _upgrade_packages( upgrade packages docs ``` """ - from meerschaum.utils.packages import packages, pip_install + from meerschaum.utils.packages import packages, pip_install, get_prerelease_dependencies from meerschaum.utils.warnings import info, warn from meerschaum.utils.prompt import yes_no from meerschaum.utils.formatting import make_header, pprint @@ -140,7 +140,7 @@ def _upgrade_packages( if venv is NoVenv: venv = 'mrsm' if len(action) == 0: - group = 'full' + group = 'api' else: group = action[0] @@ -148,7 +148,7 @@ def _upgrade_packages( invalid_msg = f"Invalid dependency group '{group}'." avail_msg = make_header("Available groups:") for k in packages: - avail_msg += "\n - {k}" + avail_msg += f"\n - {k}" warn(invalid_msg + "\n\n" + avail_msg, stack=False) return False, invalid_msg @@ -160,10 +160,18 @@ def _upgrade_packages( f"(dependency group '{group}')?" ) to_install = [install_name for import_name, install_name in packages[group].items()] + prereleases_to_install = get_prerelease_dependencies(to_install) + to_install = [ + install_name + for install_name in to_install + if install_name not in prereleases_to_install + ] success, msg = False, f"Nothing installed." if force or yes_no(question, noask=noask, yes=yes): success = pip_install(*to_install, debug=debug) + if success and prereleases_to_install: + success = pip_install(*prereleases_to_install, debug=debug) msg = ( f"Successfully installed {len(packages[group])} packages." if success else f"Failed to install packages in dependency group '{group}'." diff --git a/meerschaum/api/__init__.py b/meerschaum/api/__init__.py index 8ac7d176..18f6de8c 100644 --- a/meerschaum/api/__init__.py +++ b/meerschaum/api/__init__.py @@ -31,6 +31,7 @@ endpoints = STATIC_CONFIG['api']['endpoints'] +uv = attempt_import('uv', lazy=False, check_update=CHECK_UPDATE) ( fastapi, aiofiles, diff --git a/meerschaum/api/dash/callbacks/__init__.py b/meerschaum/api/dash/callbacks/__init__.py index 920f7f0f..cb1661da 100644 --- a/meerschaum/api/dash/callbacks/__init__.py +++ b/meerschaum/api/dash/callbacks/__init__.py @@ -11,3 +11,7 @@ import meerschaum.api.dash.callbacks.plugins import meerschaum.api.dash.callbacks.jobs import meerschaum.api.dash.callbacks.register +from meerschaum.api.dash.callbacks.custom import init_dash_plugins, add_plugin_pages + +init_dash_plugins() +add_plugin_pages() diff --git a/meerschaum/api/dash/callbacks/custom.py b/meerschaum/api/dash/callbacks/custom.py new file mode 100644 index 00000000..98e97b41 --- /dev/null +++ b/meerschaum/api/dash/callbacks/custom.py @@ -0,0 +1,39 @@ +#! /usr/bin/env python3 +# -*- coding: utf-8 -*- +# vim:fenc=utf-8 + +""" +Import custom callbacks created by plugins. +""" + +import traceback +from meerschaum.api.dash import dash_app +from meerschaum.plugins import _dash_plugins, _plugin_endpoints_to_pages +from meerschaum.utils.warnings import warn +from meerschaum.api.dash.callbacks.dashboard import _paths, _required_login + + +def init_dash_plugins(): + """ + Fire the initial callbacks for Dash plugins. + """ + for _module_name, _functions in _dash_plugins.items(): + for _function in _functions: + try: + _function(dash_app) + except Exception as e: + warn( + f"Failed to load function '{_function.__name__}' " + + f"from plugin '{_module_name}':\n" + + traceback.format_exc() + ) + + +def add_plugin_pages(): + """ + Allow users to add pages via the `@web_page` decorator. + """ + for _endpoint, _page_dict in _plugin_endpoints_to_pages.items(): + _paths[_endpoint] = _page_dict['function']() + if _page_dict['login_required']: + _required_login.add(_endpoint) diff --git a/meerschaum/api/dash/callbacks/dashboard.py b/meerschaum/api/dash/callbacks/dashboard.py index e7aecc7d..0ed640d5 100644 --- a/meerschaum/api/dash/callbacks/dashboard.py +++ b/meerschaum/api/dash/callbacks/dashboard.py @@ -94,6 +94,8 @@ 'repo', 'instance', } + +### Map endpoints to page layouts. _paths = { 'login' : pages.login.layout, '' : pages.dashboard.layout, @@ -101,7 +103,8 @@ 'register': pages.register.layout, } _required_login = {''} - + + @dash_app.callback( Output('page-layout-div', 'children'), Output('session-store', 'data'), @@ -147,16 +150,31 @@ def update_page_layout_div( else: session_store_to_return = dash.no_update - _path = ( + base_path = ( pathname.rstrip('/') + '/' ).replace( (dash_endpoint + '/'), '' ).rstrip('/').split('/')[0] + + complete_path = ( + pathname.rstrip('/') + '/' + ).replace( + dash_endpoint + '/', + '' + ).rstrip('/') + + if complete_path in _paths: + path_str = complete_path + elif base_path in _paths: + path_str = base_path + else: + path_str = '' + path = ( - _path - if no_auth or _path not in _required_login else ( - _path + path_str + if no_auth or path_str not in _required_login else ( + path_str if session_id in active_sessions else 'login' ) @@ -868,10 +886,25 @@ def sync_documents_click(n_clicks, sync_editor_text): location = "None"; } + var subaction = "pipes"; + if (action == "python"){ + subaction = ( + '"' + "pipe = mrsm.Pipe('" + + pipe_meta.connector + + "', '" + + pipe_meta.metric + + "'" + ); + if (location != "None"){ + subaction += ", '" + location + "'"; + } + subaction += ", instance='" + pipe_meta.instance + "')" + '"'; + } + iframe.contentWindow.postMessage( { action: action, - subaction: "pipes", + subaction: subaction, connector_keys: [pipe_meta.connector], metric_keys: [pipe_meta.metric], location_keys: [location], diff --git a/meerschaum/api/dash/callbacks/login.py b/meerschaum/api/dash/callbacks/login.py index 33fb8810..d39e665c 100644 --- a/meerschaum/api/dash/callbacks/login.py +++ b/meerschaum/api/dash/callbacks/login.py @@ -45,6 +45,7 @@ def show_registration_disabled_collapse(n_clicks, is_open): State('username-input', 'value'), State('password-input', 'value'), State('location', 'href'), + State('location', 'pathname'), ) def login_button_click( username_submit, @@ -53,6 +54,7 @@ def login_button_click( username, password, location_href, + location_pathname, ): """ When the user submits the login form, check the login. @@ -69,4 +71,4 @@ def login_button_click( except HTTPException: form_class += ' is-invalid' session_data = None - return session_data, form_class, (dash.no_update if not session_data else endpoints['dash']) + return session_data, form_class, dash.no_update diff --git a/meerschaum/api/dash/components.py b/meerschaum/api/dash/components.py index 2adcdcb1..2d1f0c57 100644 --- a/meerschaum/api/dash/components.py +++ b/meerschaum/api/dash/components.py @@ -12,7 +12,7 @@ from meerschaum.utils.typing import SuccessTuple, List from meerschaum.config.static import STATIC_CONFIG from meerschaum.utils.misc import remove_ansi -from meerschaum.actions import get_shell +from meerschaum._internal.shell.Shell import get_shell_intro from meerschaum.api import endpoints, CHECK_UPDATE from meerschaum.connectors import instance_types from meerschaum.utils.misc import get_connector_labels @@ -69,7 +69,10 @@ ]) ) ) -console_div = html.Div(id='console-div', children=[html.Pre(get_shell().intro, id='console-pre')]) +console_div = html.Div( + id = 'console-div', + children = [html.Pre(get_shell_intro(), id='console-pre')], +) location = dcc.Location(id='location', refresh=False) diff --git a/meerschaum/api/dash/pipes.py b/meerschaum/api/dash/pipes.py index 531871ab..858ac951 100644 --- a/meerschaum/api/dash/pipes.py +++ b/meerschaum/api/dash/pipes.py @@ -22,7 +22,7 @@ dash_app, debug, _get_pipes ) from meerschaum.api.dash.connectors import get_web_connector -from meerschaum.api.dash.components import alert_from_success_tuple +from meerschaum.api.dash.components import alert_from_success_tuple, build_cards_grid from meerschaum.api.dash.users import is_session_authenticated from meerschaum.config import get_config import meerschaum as mrsm @@ -101,6 +101,132 @@ def pipes_from_state( return False, str(e) return _pipes + +def build_pipe_card( + pipe: mrsm.Pipe, + authenticated: bool = False, + _build_children_num: int = 10, + ) -> 'dbc.Card': + """ + Return a card for the given pipe. + + Parameters + ---------- + pipe: mrsm.Pipe + The pipe from which to build the card. + + authenticated: bool, default False + If `True`, allow editing functionality to the card. + + Returns + ------- + A dash bootstrap components Card representation of the pipe. + """ + meta_str = json.dumps(pipe.meta) + footer_children = dbc.Row( + [ + dbc.Col( + ( + dbc.DropdownMenu( + label = "Manage", + children = [ + dbc.DropdownMenuItem( + 'Open in Python', + id = { + 'type': 'manage-pipe-button', + 'index': meta_str, + 'action': 'python', + }, + ), + dbc.DropdownMenuItem( + 'Delete', + id = { + 'type': 'manage-pipe-button', + 'index': meta_str, + 'action': 'delete', + }, + ), + dbc.DropdownMenuItem( + 'Drop', + id = { + 'type': 'manage-pipe-button', + 'index': meta_str, + 'action': 'drop', + }, + ), + dbc.DropdownMenuItem( + 'Clear', + id = { + 'type': 'manage-pipe-button', + 'index': meta_str, + 'action': 'clear', + }, + ), + dbc.DropdownMenuItem( + 'Verify', + id = { + 'type': 'manage-pipe-button', + 'index': meta_str, + 'action': 'verify', + }, + ), + dbc.DropdownMenuItem( + 'Sync', + id = { + 'type': 'manage-pipe-button', + 'index': meta_str, + 'action': 'sync', + }, + ), + ], + direction = "up", + menu_variant = "dark", + size = 'sm', + color = 'secondary', + ) + ) if authenticated else [], + width = 2, + ), + dbc.Col(width=6), + dbc.Col( + dbc.Button( + 'Download CSV', + size = 'sm', + color = 'link', + style = {'float': 'right'}, + id = {'type': 'pipe-download-csv-button', 'index': meta_str}, + ), + width = 4, + ), + ], + justify = 'start', + ) + card_body_children = [ + html.H5( + html.B(str(pipe)), + className = 'card-title', + style = {'font-family': ['monospace']} + ), + html.Div( + dbc.Accordion( + accordion_items_from_pipe( + pipe, + authenticated = authenticated, + _build_children_num = _build_children_num, + ), + flush = True, + start_collapsed = True, + id = {'type': 'pipe-accordion', 'index': meta_str}, + ) + ) + + ] + return dbc.Card([ + dbc.CardBody(children=card_body_children), + dbc.CardFooter(children=footer_children), + ]) + + def get_pipes_cards(*keys, session_data: Optional[Dict[str, Any]] = None): """ Returns a tuple: @@ -120,99 +246,7 @@ def get_pipes_cards(*keys, session_data: Optional[Dict[str, Any]] = None): overflow_pipes = pipes[max_num_pipes_cards:] for pipe in pipes[:max_num_pipes_cards]: - meta_str = json.dumps(pipe.meta) - footer_children = dbc.Row( - [ - dbc.Col( - ( - dbc.DropdownMenu( - label = "Manage", - children = [ - dbc.DropdownMenuItem( - 'Delete', - id = { - 'type': 'manage-pipe-button', - 'index': meta_str, - 'action': 'delete', - }, - ), - dbc.DropdownMenuItem( - 'Drop', - id = { - 'type': 'manage-pipe-button', - 'index': meta_str, - 'action': 'drop', - }, - ), - dbc.DropdownMenuItem( - 'Clear', - id = { - 'type': 'manage-pipe-button', - 'index': meta_str, - 'action': 'clear', - }, - ), - dbc.DropdownMenuItem( - 'Verify', - id = { - 'type': 'manage-pipe-button', - 'index': meta_str, - 'action': 'verify', - }, - ), - dbc.DropdownMenuItem( - 'Sync', - id = { - 'type': 'manage-pipe-button', - 'index': meta_str, - 'action': 'sync', - }, - ), - ], - direction = "up", - menu_variant = "dark", - size = 'sm', - color = 'secondary', - ) - ) if authenticated else [], - width = 2, - ), - dbc.Col(width=6), - dbc.Col( - dbc.Button( - 'Download CSV', - size = 'sm', - color = 'link', - style = {'float': 'right'}, - id = {'type': 'pipe-download-csv-button', 'index': meta_str}, - ), - width = 4, - ), - ], - justify = 'start', - ) - card_body_children = [ - html.H5( - html.B(str(pipe)), - className = 'card-title', - style = {'font-family': ['monospace']} - ), - html.Div( - dbc.Accordion( - accordion_items_from_pipe(pipe, authenticated=authenticated), - flush = True, - start_collapsed = True, - id = {'type': 'pipe-accordion', 'index': meta_str}, - ) - ) - - ] - cards.append( - dbc.Card([ - dbc.CardBody(children=card_body_children), - dbc.CardFooter(children=footer_children), - ]) - ) + cards.append(build_pipe_card(pipe, authenticated=authenticated)) if overflow_pipes: cards.append( @@ -239,7 +273,8 @@ def accordion_items_from_pipe( pipe: mrsm.Pipe, active_items: Optional[List[str]] = None, authenticated: bool = False, - ) -> List[dbc.AccordionItem]: + _build_children_num: int = 10, + ) -> 'List[dbc.AccordionItem]': """ Build the accordion items for a given pipe. """ @@ -401,7 +436,7 @@ def accordion_items_from_pipe( size = 'sm', style = {'text-decoration': 'none', 'margin-left': '10px'}, ) - items_bodies['parameters'] = html.Div([ + parameters_div_children = [ parameters_editor, html.Br(), dbc.Row([ @@ -422,8 +457,21 @@ def accordion_items_from_pipe( width=True, ) ]), + ] + if _build_children_num > 0 and pipe.children: + children_cards = [ + build_pipe_card( + child_pipe, + authenticated = authenticated, + _build_children_num = (_build_children_num - 1), + ) + for child_pipe in pipe.children + ] + children_grid = build_cards_grid(children_cards, num_columns=1) + chidren_div_items = [html.Br(), html.H3('Children Pipes'), html.Br(), children_grid] + parameters_div_children.extend([html.Br()] + chidren_div_items) - ]) + items_bodies['parameters'] = html.Div(parameters_div_children) if 'sql' in active_items: query = dedent((get_pipe_query(pipe, warn=False) or "")).lstrip().rstrip() diff --git a/meerschaum/config/_default.py b/meerschaum/config/_default.py index ca88cf9c..2fd0bb1b 100644 --- a/meerschaum/config/_default.py +++ b/meerschaum/config/_default.py @@ -110,6 +110,7 @@ 'space': False, 'join_fetch': False, 'inplace_sync': True, + 'uv_pip': True, }, } default_pipes_config = { diff --git a/meerschaum/config/_paths.py b/meerschaum/config/_paths.py index edf777f3..b2b9de01 100644 --- a/meerschaum/config/_paths.py +++ b/meerschaum/config/_paths.py @@ -82,24 +82,24 @@ ENVIRONMENT_VENVS_DIR = STATIC_CONFIG['environment']['venvs'] if ENVIRONMENT_VENVS_DIR in os.environ: - VENVS_DIR_PATH = Path(os.environ[ENVIRONMENT_VENVS_DIR]).resolve() - if not VENVS_DIR_PATH.exists(): + _VENVS_DIR_PATH = Path(os.environ[ENVIRONMENT_VENVS_DIR]).resolve() + if not _VENVS_DIR_PATH.exists(): try: - VENVS_DIR_PATH.mkdir(parents=True, exist_ok=True) + _VENVS_DIR_PATH.mkdir(parents=True, exist_ok=True) except Exception as e: print( f"Invalid path set for environment variable '{ENVIRONMENT_VENVS_DIR}':\n" - + f"{VENVS_DIR_PATH}" + + f"{_VENVS_DIR_PATH}" ) - VENVS_DIR_PATH = (_ROOT_DIR_PATH / 'venvs').resolve() - print(f"Will use the following path for venvs instead:\n{VENVS_DIR_PATH}") + _VENVS_DIR_PATH = (_ROOT_DIR_PATH / 'venvs').resolve() + print(f"Will use the following path for venvs instead:\n{_VENVS_DIR_PATH}") else: - VENVS_DIR_PATH = _ROOT_DIR_PATH / 'venvs' + _VENVS_DIR_PATH = _ROOT_DIR_PATH / 'venvs' paths = { - 'PACKAGE_ROOT_PATH' : str(Path(__file__).parent.parent.resolve()), - 'ROOT_DIR_PATH' : str(_ROOT_DIR_PATH), - 'VIRTENV_RESOURCES_PATH' : str(VENVS_DIR_PATH), + 'PACKAGE_ROOT_PATH' : Path(__file__).parent.parent.resolve().as_posix(), + 'ROOT_DIR_PATH' : _ROOT_DIR_PATH.as_posix(), + 'VIRTENV_RESOURCES_PATH' : _VENVS_DIR_PATH.as_posix(), 'CONFIG_DIR_PATH' : ('{ROOT_DIR_PATH}', 'config'), 'DEFAULT_CONFIG_DIR_PATH' : ('{ROOT_DIR_PATH}', 'default_config'), 'PATCH_DIR_PATH' : ('{ROOT_DIR_PATH}', 'patch_config'), @@ -114,6 +114,7 @@ 'SHELL_RESOURCES_PATH' : ('{ROOT_DIR_PATH}', ), 'SHELL_HISTORY_PATH' : ('{SHELL_RESOURCES_PATH}', '.mrsm_history'), + 'PYTHON_RESOURCES_PATH' : ('{INTERNAL_RESOURCES_PATH}', 'python'), 'API_RESOURCES_PATH' : ('{PACKAGE_ROOT_PATH}', 'api', 'resources'), 'API_STATIC_PATH' : ('{API_RESOURCES_PATH}', 'static'), @@ -186,7 +187,6 @@ def __getattr__(name: str) -> Path: if name.endswith('RESOURCES_PATH') or name == 'CONFIG_DIR_PATH': path.mkdir(parents=True, exist_ok=True) elif 'FILENAME' in name: - path = str(path) + path = path.as_posix() return path - diff --git a/meerschaum/config/_version.py b/meerschaum/config/_version.py index 17ec906f..cc4c8dfb 100644 --- a/meerschaum/config/_version.py +++ b/meerschaum/config/_version.py @@ -2,4 +2,4 @@ Specify the Meerschaum release version. """ -__version__ = "2.2.1" +__version__ = "2.2.2" diff --git a/meerschaum/config/paths.py b/meerschaum/config/paths.py new file mode 100644 index 00000000..9be6b173 --- /dev/null +++ b/meerschaum/config/paths.py @@ -0,0 +1,10 @@ +#! /usr/bin/env python3 +# -*- coding: utf-8 -*- +# vim:fenc=utf-8 + +""" +External API for importing Meerschaum paths. +""" + +from meerschaum.config._paths import __getattr__, paths +__all__ = tuple(paths.keys()) diff --git a/meerschaum/config/static/__init__.py b/meerschaum/config/static/__init__.py index 5728146c..6cf27760 100644 --- a/meerschaum/config/static/__init__.py +++ b/meerschaum/config/static/__init__.py @@ -110,7 +110,7 @@ 'pbkdf2_sha256', ], 'default': 'pbkdf2_sha256', - 'pbkdf2_sha256__default_rounds': 3_000_000, + 'pbkdf2_sha256__default_rounds': 1_000_000, }, 'min_username_length': 1, 'max_username_length': 26, diff --git a/meerschaum/connectors/__init__.py b/meerschaum/connectors/__init__.py index 3be8b0ee..c7310e31 100644 --- a/meerschaum/connectors/__init__.py +++ b/meerschaum/connectors/__init__.py @@ -328,7 +328,7 @@ def load_plugin_connectors(): continue with open(plugin.__file__, encoding='utf-8') as f: text = f.read() - if 'make_connector' in text: + if 'make_connector' in text or 'Connector' in text: to_import.append(plugin.name) if not to_import: return diff --git a/meerschaum/core/Pipe/__init__.py b/meerschaum/core/Pipe/__init__.py index fe875a3e..5ba84f07 100644 --- a/meerschaum/core/Pipe/__init__.py +++ b/meerschaum/core/Pipe/__init__.py @@ -436,6 +436,11 @@ def __hash__(self): def __repr__(self, **kw) -> str: return pipe_repr(self, **kw) + def __pt_repr__(self): + from meerschaum.utils.packages import attempt_import + prompt_toolkit_formatted_text = attempt_import('prompt_toolkit.formatted_text', lazy=False) + return prompt_toolkit_formatted_text.ANSI(self.__repr__()) + def __getstate__(self) -> Dict[str, Any]: """ Define the state dictionary (pickling). diff --git a/meerschaum/core/Pipe/_sync.py b/meerschaum/core/Pipe/_sync.py index 695a7b15..8eceadcf 100644 --- a/meerschaum/core/Pipe/_sync.py +++ b/meerschaum/core/Pipe/_sync.py @@ -215,9 +215,8 @@ def _sync( ### Activate and invoke `sync(pipe)` for plugin connectors with `sync` methods. try: - if p.connector.type == 'plugin' and p.connector.sync is not None: - connector_plugin = Plugin(p.connector.label) - with Venv(connector_plugin, debug=debug): + if getattr(p.connector, 'sync', None) is not None: + with Venv(get_connector_plugin(p.connector), debug=debug): return_tuple = p.connector.sync(p, debug=debug, **kw) p._exists = None if not isinstance(return_tuple, tuple): diff --git a/meerschaum/plugins/__init__.py b/meerschaum/plugins/__init__.py index 23550548..b2e64c1a 100644 --- a/meerschaum/plugins/__init__.py +++ b/meerschaum/plugins/__init__.py @@ -7,6 +7,7 @@ """ from __future__ import annotations +import functools from meerschaum.utils.typing import Callable, Any, Union, Optional, Dict, List, Tuple from meerschaum.utils.threading import Lock, RLock from meerschaum.plugins._Plugin import Plugin @@ -16,6 +17,7 @@ _post_sync_hooks: Dict[str, List[Callable[[Any], Any]]] = {} _locks = { '_api_plugins': RLock(), + '_dash_plugins': RLock(), '_pre_sync_hooks': RLock(), '_post_sync_hooks': RLock(), '__path__': RLock(), @@ -156,6 +158,71 @@ def post_sync_hook( return function +_plugin_endpoints_to_pages = {} +def web_page( + page: Union[str, None, Callable[[Any], Any]] = None, + login_required: bool = True, + **kwargs + ) -> Any: + """ + Quickly add pages to the dash application. + + Examples + -------- + >>> import meerschaum as mrsm + >>> from meerschaum.plugins import web_page + >>> html = mrsm.attempt_import('dash.html') + >>> + >>> @web_page('foo/bar', login_required=False) + >>> def foo_bar(): + ... return html.Div([html.H1("Hello, World!")]) + >>> + """ + page_str = None + + def _decorator(_func: Callable[[Any], Any]) -> Callable[[Any], Any]: + nonlocal page_str + + @functools.wraps(_func) + def wrapper(*_args, **_kwargs): + return _func(*_args, **_kwargs) + + if page_str is None: + page_str = _func.__name__ + + page_str = page_str.lstrip('/').rstrip('/').strip() + _plugin_endpoints_to_pages[page_str] = { + 'function': _func, + 'login_required': login_required, + } + return wrapper + + if callable(page): + decorator_to_return = _decorator(page) + page_str = page.__name__ + else: + decorator_to_return = _decorator + page_str = page + + return decorator_to_return + + +_dash_plugins = {} +def dash_plugin(function: Callable[[Any], Any]) -> Callable[[Any], Any]: + """ + Execute the function when starting the Dash application. + """ + with _locks['_dash_plugins']: + try: + if function.__module__ not in _dash_plugins: + _dash_plugins[function.__module__] = [] + _dash_plugins[function.__module__].append(function) + except Exception as e: + from meerschaum.utils.warnings import warn + warn(e) + return function + + def api_plugin(function: Callable[[Any], Any]) -> Callable[[Any], Any]: """ Execute the function when initializing the Meerschaum API module. @@ -164,15 +231,6 @@ def api_plugin(function: Callable[[Any], Any]) -> Callable[[Any], Any]: The FastAPI app will be passed as the only parameter. - Parameters - ---------- - function: Callable[[Any, Any]] - The function to be called before starting the Meerschaum API. - - Returns - ------- - Another function (decorator function). - Examples -------- >>> from meerschaum.plugins import api_plugin diff --git a/meerschaum/utils/daemon/Daemon.py b/meerschaum/utils/daemon/Daemon.py index 1c6359d7..13bfc36c 100644 --- a/meerschaum/utils/daemon/Daemon.py +++ b/meerschaum/utils/daemon/Daemon.py @@ -465,8 +465,9 @@ def _handle_interrupt(self, signal_number: int, stack_frame: 'frame') -> None: Handle `SIGINT` within the Daemon context. This method is injected into the `DaemonContext`. """ - # from meerschaum.utils.daemon.FileDescriptorInterceptor import STOP_READING_FD_EVENT - # STOP_READING_FD_EVENT.set() + from meerschaum.utils.process import signal_handler + signal_handler(signal_number, stack_frame) + self.rotating_log.stop_log_fd_interception(unused_only=False) timer = self.__dict__.get('_log_refresh_timer', None) if timer is not None: @@ -477,6 +478,7 @@ def _handle_interrupt(self, signal_number: int, stack_frame: 'frame') -> None: daemon_context.close() _close_pools() + import threading for thread in threading.enumerate(): if thread.name == 'MainThread': @@ -495,6 +497,9 @@ def _handle_sigterm(self, signal_number: int, stack_frame: 'frame') -> None: Handle `SIGTERM` within the `Daemon` context. This method is injected into the `DaemonContext`. """ + from meerschaum.utils.process import signal_handler + signal_handler(signal_number, stack_frame) + timer = self.__dict__.get('_log_refresh_timer', None) if timer is not None: timer.cancel() diff --git a/meerschaum/utils/misc.py b/meerschaum/utils/misc.py index b56f974c..5f6edbb2 100644 --- a/meerschaum/utils/misc.py +++ b/meerschaum/utils/misc.py @@ -1103,6 +1103,12 @@ def is_docker_available() -> bool: return has_docker +def is_android() -> bool: + """Return `True` if the current platform is Android.""" + import sys + return hasattr(sys, 'getandroidapilevel') + + def is_bcp_available() -> bool: """Check if the MSSQL `bcp` utility is installed.""" import subprocess diff --git a/meerschaum/utils/packages/__init__.py b/meerschaum/utils/packages/__init__.py index 1f0f2a25..0a356642 100644 --- a/meerschaum/utils/packages/__init__.py +++ b/meerschaum/utils/packages/__init__.py @@ -46,6 +46,7 @@ def get_module_path( """ Get a module's path without importing. """ + import site if debug: from meerschaum.utils.debug import dprint if not _try_install_name_on_fail: @@ -54,33 +55,58 @@ def get_module_path( import_name_lower = install_name_lower else: import_name_lower = import_name.lower().replace('-', '_') + vtp = venv_target_path(venv, allow_nonexistent=True, debug=debug) if not vtp.exists(): if debug: - dprint(f"Venv '{venv}' does not exist, cannot import '{import_name}'.", color=False) + dprint( + ( + "Venv '{venv}' does not exist, cannot import " + + f"'{import_name}'." + ), + color = False, + ) return None + + venv_target_candidate_paths = [vtp] + if venv is None: + site_user_packages_dirs = [pathlib.Path(site.getusersitepackages())] + site_packages_dirs = [pathlib.Path(path) for path in site.getsitepackages()] + + paths_to_add = [ + path + for path in site_user_packages_dirs + site_packages_dirs + if path not in venv_target_candidate_paths + ] + venv_target_candidate_paths += paths_to_add + candidates = [] - for file_name in os.listdir(vtp): - file_name_lower = file_name.lower().replace('-', '_') - if not file_name_lower.startswith(import_name_lower): - continue - if file_name.endswith('dist_info'): + for venv_target_candidate in venv_target_candidate_paths: + try: + file_names = os.listdir(venv_target_candidate) + except FileNotFoundError: continue - file_path = vtp / file_name + for file_name in file_names: + file_name_lower = file_name.lower().replace('-', '_') + if not file_name_lower.startswith(import_name_lower): + continue + if file_name.endswith('dist_info'): + continue + file_path = venv_target_candidate / file_name - ### Most likely: Is a directory with __init__.py - if file_name_lower == import_name_lower and file_path.is_dir(): - init_path = file_path / '__init__.py' - if init_path.exists(): - candidates.append(init_path) + ### Most likely: Is a directory with __init__.py + if file_name_lower == import_name_lower and file_path.is_dir(): + init_path = file_path / '__init__.py' + if init_path.exists(): + candidates.append(init_path) - ### May be a standalone .py file. - elif file_name_lower == import_name_lower + '.py': - candidates.append(file_path) + ### May be a standalone .py file. + elif file_name_lower == import_name_lower + '.py': + candidates.append(file_path) - ### Compiled wheels (e.g. pyodbc) - elif file_name_lower.startswith(import_name_lower + '.'): - candidates.append(file_path) + ### Compiled wheels (e.g. pyodbc) + elif file_name_lower.startswith(import_name_lower + '.'): + candidates.append(file_path) if len(candidates) == 1: return candidates[0] @@ -466,12 +492,13 @@ def _get_package_metadata(import_name: str, venv: Optional[str]) -> Dict[str, st import re from meerschaum.config._paths import VIRTENV_RESOURCES_PATH install_name = _import_to_install_name(import_name) - _args = ['show', install_name] + _args = ['pip', 'show', install_name] if venv is not None: cache_dir_path = VIRTENV_RESOURCES_PATH / venv / 'cache' - _args += ['--cache-dir', str(cache_dir_path)] + _args += ['--cache-dir', cache_dir_path.as_posix()] + proc = run_python_package( - 'pip', _args, + 'uv', _args, capture_output=True, as_proc=True, venv=venv, universal_newlines=True, ) outs, errs = proc.communicate() @@ -680,12 +707,22 @@ def need_update( return False -def get_pip(venv: Optional[str] = 'mrsm', debug: bool=False) -> bool: +def get_pip( + venv: Optional[str] = 'mrsm', + color: bool = True, + debug: bool = False, + ) -> bool: """ Download and run the get-pip.py script. Parameters ---------- + venv: Optional[str], default 'mrsm' + The virtual environment into which to install `pip`. + + color: bool, default True + If `True`, force color output. + debug: bool, default False Verbosity toggle. @@ -708,7 +745,7 @@ def get_pip(venv: Optional[str] = 'mrsm', debug: bool=False) -> bool: if venv is not None: init_venv(venv=venv, debug=debug) cmd_list = [venv_executable(venv=venv), dest.as_posix()] - return subprocess.call(cmd_list, env=_get_pip_os_env()) == 0 + return subprocess.call(cmd_list, env=_get_pip_os_env(color=color)) == 0 def pip_install( @@ -721,6 +758,8 @@ def pip_install( check_pypi: bool = True, check_wheel: bool = True, _uninstall: bool = False, + _from_completely_uninstall: bool = False, + _install_uv_pip: bool = True, color: bool = True, silent: bool = False, debug: bool = False, @@ -776,7 +815,9 @@ def pip_install( """ from meerschaum.config._paths import VIRTENV_RESOURCES_PATH + from meerschaum.config import get_config from meerschaum.utils.warnings import warn + from meerschaum.utils.misc import is_android if args is None: args = ['--upgrade'] if not _uninstall else [] if color: @@ -787,10 +828,43 @@ def pip_install( have_wheel = venv_contains_package('wheel', venv=venv, debug=debug) _args = list(args) - have_pip = venv_contains_package('pip', venv=venv, debug=debug) + have_pip = venv_contains_package('pip', venv=None, debug=debug) + try: + import pip + have_pip = True + except ImportError: + have_pip = False + try: + import uv + uv_bin = uv.find_uv_bin() + have_uv_pip = True + except (ImportError, FileNotFoundError): + uv_bin = None + have_uv_pip = False + if have_pip and not have_uv_pip and _install_uv_pip and not is_android(): + if not pip_install( + 'uv', + venv = None, + debug = debug, + _install_uv_pip = False, + check_update = False, + check_pypi = False, + check_wheel = False, + ): + warn( + f"Failed to install `uv` for virtual environment '{venv}'.", + color = False, + ) + + use_uv_pip = ( + venv_contains_package('uv', venv=None, debug=debug) + and uv_bin is not None + and venv is not None + ) + import sys - if not have_pip: - if not get_pip(venv=venv, debug=debug): + if not have_pip and not use_uv_pip: + if not get_pip(venv=venv, color=color, debug=debug): import sys minor = sys.version_info.minor print( @@ -806,13 +880,18 @@ def pip_install( with Venv(venv, debug=debug): if venv is not None: - if '--ignore-installed' not in args and '-I' not in _args and not _uninstall: + if ( + '--ignore-installed' not in args + and '-I' not in _args + and not _uninstall + and not use_uv_pip + ): _args += ['--ignore-installed'] if '--cache-dir' not in args and not _uninstall: cache_dir_path = VIRTENV_RESOURCES_PATH / venv / 'cache' _args += ['--cache-dir', str(cache_dir_path)] - if 'pip' not in ' '.join(_args): + if 'pip' not in ' '.join(_args) and not use_uv_pip: if check_update and not _uninstall: pip = attempt_import('pip', venv=venv, install=False, debug=debug, lazy=False) if need_update(pip, check_pypi=check_pypi, debug=debug): @@ -820,17 +899,20 @@ def pip_install( _args = (['install'] if not _uninstall else ['uninstall']) + _args - if check_wheel and not _uninstall: + if check_wheel and not _uninstall and not use_uv_pip: if not have_wheel: if not pip_install( - 'setuptools', 'wheel', + 'setuptools', 'wheel', 'uv', venv = venv, - check_update = False, check_pypi = False, - check_wheel = False, debug = debug, + check_update = False, + check_pypi = False, + check_wheel = False, + debug = debug, + _install_uv_pip = False, ): warn( ( - "Failed to install `setuptools` and `wheel` for virtual " + "Failed to install `setuptools`, `wheel`, and `uv` for virtual " + f"environment '{venv}'." ), color = False, @@ -838,24 +920,24 @@ def pip_install( if requirements_file_path is not None: _args.append('-r') - _args.append(str(pathlib.Path(requirements_file_path).resolve())) + _args.append(pathlib.Path(requirements_file_path).resolve().as_posix()) if not ANSI and '--no-color' not in _args: _args.append('--no-color') - if '--no-input' not in _args: + if '--no-input' not in _args and not use_uv_pip: _args.append('--no-input') - if _uninstall and '-y' not in _args: + if _uninstall and '-y' not in _args and not use_uv_pip: _args.append('-y') - if '--no-warn-conflicts' not in _args and not _uninstall: + if '--no-warn-conflicts' not in _args and not _uninstall and not use_uv_pip: _args.append('--no-warn-conflicts') - if '--disable-pip-version-check' not in _args: + if '--disable-pip-version-check' not in _args and not use_uv_pip: _args.append('--disable-pip-version-check') - if '--target' not in _args and '-t' not in _args and not _uninstall: + if '--target' not in _args and '-t' not in _args and not (not use_uv_pip and _uninstall): if venv is not None: _args += ['--target', venv_target_path(venv, debug=debug)] elif ( @@ -863,12 +945,14 @@ def pip_install( and '-t' not in _args and not inside_venv() and not _uninstall + and not use_uv_pip ): _args += ['--user'] if debug: if '-v' not in _args or '-vv' not in _args or '-vvv' not in _args: - pass + if use_uv_pip: + _args.append('--verbose') else: if '-q' not in _args or '-qq' not in _args or '-qqq' not in _args: pass @@ -883,10 +967,10 @@ def pip_install( if not silent: print(msg) - if not _uninstall: + if _uninstall and not _from_completely_uninstall and not use_uv_pip: for install_name in _packages: _install_no_version = get_install_no_version(install_name) - if _install_no_version in ('pip', 'wheel'): + if _install_no_version in ('pip', 'wheel', 'uv'): continue if not completely_uninstall_package( _install_no_version, @@ -896,11 +980,17 @@ def pip_install( f"Failed to clean up package '{_install_no_version}'.", ) + ### NOTE: Only append the `--prerelease=allow` flag if we explicitly depend on a prerelease. + if use_uv_pip: + _args.insert(0, 'pip') + if not _uninstall and get_prerelease_dependencies(_packages): + _args.append('--prerelease=allow') + rc = run_python_package( - 'pip', + ('pip' if not use_uv_pip else 'uv'), _args + _packages, - venv = venv, - env = _get_pip_os_env(), + venv = None, + env = _get_pip_os_env(color=color), debug = debug, ) if debug: @@ -918,6 +1008,33 @@ def pip_install( return success +def get_prerelease_dependencies(_packages: Optional[List[str]] = None): + """ + Return a list of explicitly prerelease dependencies from a list of packages. + """ + if _packages is None: + _packages = list(all_packages.keys()) + prelrease_strings = ['dev', 'rc', 'a'] + prerelease_packages = [] + for install_name in _packages: + _install_no_version = get_install_no_version(install_name) + import_name = _install_to_import_name(install_name) + install_with_version = _import_to_install_name(import_name) + version_only = ( + install_with_version.lower().replace(_install_no_version.lower(), '') + .split(']')[-1] + ) + + is_prerelease = False + for prelrease_string in prelrease_strings: + if prelrease_string in version_only: + is_prerelease = True + + if is_prerelease: + prerelease_packages.append(install_name) + return prerelease_packages + + def completely_uninstall_package( install_name: str, venv: str = 'mrsm', @@ -944,7 +1061,7 @@ def completely_uninstall_package( continue installed_versions.append(file_name) - max_attempts = len(installed_versions) + 1 + max_attempts = len(installed_versions) while attempts < max_attempts: if not venv_contains_package( _install_to_import_name(_install_no_version), @@ -953,8 +1070,10 @@ def completely_uninstall_package( return True if not pip_uninstall( _install_no_version, - venv=venv, - silent=(not debug), debug=debug + venv = venv, + silent = (not debug), + _from_completely_uninstall = True, + debug = debug, ): return False attempts += 1 @@ -1031,6 +1150,10 @@ def run_python_package( if cwd is not None: os.chdir(cwd) executable = venv_executable(venv=venv) + venv_path = (VIRTENV_RESOURCES_PATH / venv) if venv is not None else None + env_dict = kw.get('env', os.environ).copy() + if venv_path is not None: + env_dict.update({'VIRTUAL_ENV': venv_path.as_posix()}) command = [executable, '-m', str(package_name)] + [str(a) for a in args] import traceback if debug: @@ -1055,7 +1178,7 @@ def run_python_package( command, stdout = stdout, stderr = stderr, - env = kw.get('env', os.environ), + env = env_dict, ) to_return = proc if as_proc else proc.wait() except KeyboardInterrupt: @@ -1075,9 +1198,10 @@ def attempt_import( check_update: bool = False, check_pypi: bool = False, check_is_installed: bool = True, + allow_outside_venv: bool = True, color: bool = True, debug: bool = False - ) -> Union[Any, Tuple[Any]]: + ) -> Any: """ Raise a warning if packages are not installed; otherwise import and return modules. If `lazy` is `True`, return lazy-imported modules. @@ -1120,6 +1244,15 @@ def attempt_import( check_is_installed: bool, default True If `True`, check if the package is contained in the virtual environment. + allow_outside_venv: bool, default True + If `True`, search outside of the specified virtual environment + if the package cannot be found. + Setting to `False` will reinstall the package into a virtual environment, even if it + is installed outside. + + color: bool, default True + If `False`, do not print ANSI colors. + Returns ------- The specified modules. If they're not available and `install` is `True`, it will first @@ -1201,6 +1334,7 @@ def do_import(_name: str, **kw) -> Union['ModuleType', None]: name, venv = venv, split = split, + allow_outside_venv = allow_outside_venv, debug = debug, ) _is_installed_first_check[name] = package_is_installed @@ -1346,7 +1480,9 @@ def import_rich( 'pygments', lazy=False, ) rich = attempt_import( - 'rich', lazy=lazy, **kw) + 'rich', lazy=lazy, + **kw + ) return rich @@ -1580,10 +1716,26 @@ def is_installed( import_name: str, venv: Optional[str] = 'mrsm', split: bool = True, + allow_outside_venv: bool = True, debug: bool = False, ) -> bool: """ Check whether a package is installed. + + Parameters + ---------- + import_name: str + The import name of the module. + + venv: Optional[str], default 'mrsm' + The venv in which to search for the module. + + split: bool, default True + If `True`, split on periods to determine the root module name. + + allow_outside_venv: bool, default True + If `True`, search outside of the specified virtual environment + if the package cannot be found. """ if debug: from meerschaum.utils.debug import dprint @@ -1594,7 +1746,11 @@ def is_installed( spec_path = pathlib.Path( get_module_path(root_name, venv=venv, debug=debug) or - importlib.util.find_spec(root_name).origin + ( + importlib.util.find_spec(root_name).origin + if venv is not None and allow_outside_venv + else None + ) ) except (ModuleNotFoundError, ValueError, AttributeError, TypeError) as e: spec_path = None @@ -1623,6 +1779,8 @@ def venv_contains_package( """ Search the contents of a virtual environment for a package. """ + import site + import pathlib root_name = import_name.split('.')[0] if split else import_name return get_module_path(root_name, venv=venv, debug=debug) is not None @@ -1686,7 +1844,7 @@ def _get_distribution(dist): pkg_resources.get_distribution = _get_distribution -def _get_pip_os_env(): +def _get_pip_os_env(color: bool = True): """ Return the environment variables context in which `pip` should be run. See PEP 668 for why we are overriding the environment. @@ -1695,5 +1853,6 @@ def _get_pip_os_env(): pip_os_env = os.environ.copy() pip_os_env.update({ 'PIP_BREAK_SYSTEM_PACKAGES': 'true', + ('FORCE_COLOR' if color else 'NO_COLOR'): '1', }) return pip_os_env diff --git a/meerschaum/utils/packages/_packages.py b/meerschaum/utils/packages/_packages.py index b36ebc31..7e733fd8 100644 --- a/meerschaum/utils/packages/_packages.py +++ b/meerschaum/utils/packages/_packages.py @@ -53,6 +53,7 @@ 'dill' : 'dill>=0.3.3', 'virtualenv' : 'virtualenv>=20.1.0', 'apscheduler' : 'APScheduler>=4.0.0a5', + 'uv' : 'uv>=0.2.11', }, 'drivers': { 'cryptography' : 'cryptography>=38.0.1', diff --git a/meerschaum/utils/process.py b/meerschaum/utils/process.py index 0affc62e..d9d4e96a 100644 --- a/meerschaum/utils/process.py +++ b/meerschaum/utils/process.py @@ -9,10 +9,16 @@ """ from __future__ import annotations -import os, signal, subprocess, sys, platform +import os, signal, subprocess, sys, platform, traceback from meerschaum.utils.typing import Union, Optional, Any, Callable, Dict, Tuple from meerschaum.config.static import STATIC_CONFIG +_child_processes = [] +def signal_handler(sig, frame): + for child in _child_processes: + child.send_signal(sig) + child.wait() + def run_process( *args, foreground: bool = False, @@ -73,6 +79,7 @@ def print_line(line): sys.stdout.write(line.decode('utf-8')) sys.stdout.flush() + if capture_output or line_callback is not None: kw['stdout'] = subprocess.PIPE kw['stderr'] = subprocess.STDOUT @@ -123,6 +130,7 @@ def new_pgid(): try: child = subprocess.Popen(*args, **kw) + _child_processes.append(child) # we can't set the process group id from the parent since the child # will already have exec'd. and we can't SIGSTOP it before exec, @@ -147,7 +155,9 @@ def new_pgid(): store_proc_dict[store_proc_key] = child _ret = poll_process(child, line_callback) if line_callback is not None else child.wait() ret = _ret if not as_proc else child - + except KeyboardInterrupt: + child.send_signal(signal.SIGINT) + ret = child.wait() if not as_proc else child finally: if foreground: # we have to mask SIGTTOU because tcsetpgrp diff --git a/meerschaum/utils/schedule.py b/meerschaum/utils/schedule.py index c58789a3..398ab72c 100644 --- a/meerschaum/utils/schedule.py +++ b/meerschaum/utils/schedule.py @@ -8,7 +8,7 @@ from __future__ import annotations import sys -from datetime import datetime, timezone, timedelta, timedelta +from datetime import datetime, timezone, timedelta import meerschaum as mrsm from meerschaum.utils.typing import Callable, Any, Optional, List, Dict diff --git a/meerschaum/utils/venv/__init__.py b/meerschaum/utils/venv/__init__.py index 21280dd6..30b947c6 100644 --- a/meerschaum/utils/venv/__init__.py +++ b/meerschaum/utils/venv/__init__.py @@ -15,7 +15,7 @@ 'activate_venv', 'deactivate_venv', 'init_venv', 'inside_venv', 'is_venv_active', 'venv_exec', 'venv_executable', 'venv_exists', 'venv_target_path', - 'Venv', 'get_venvs', 'verify_venv', + 'Venv', 'get_venvs', 'verify_venv', 'get_module_venv', ]) __pdoc__ = {'Venv': True} @@ -79,7 +79,7 @@ def activate_venv( else: threads_active_venvs[thread_id][venv] += 1 - target = str(venv_target_path(venv, debug=debug)) + target = venv_target_path(venv, debug=debug).as_posix() if venv in active_venvs_order: sys.path.remove(target) try: @@ -171,7 +171,7 @@ def deactivate_venv( if sys.path is None: return False - target = str(venv_target_path(venv, allow_nonexistent=force, debug=debug)) + target = venv_target_path(venv, allow_nonexistent=force, debug=debug).as_posix() with LOCKS['sys.path']: if target in sys.path: sys.path.remove(target) @@ -361,6 +361,8 @@ def init_venv( verified_venvs.add(venv) return True + import io + from contextlib import redirect_stdout, redirect_stderr import sys, platform, os, pathlib, shutil from meerschaum.config.static import STATIC_CONFIG from meerschaum.config._paths import VIRTENV_RESOURCES_PATH @@ -381,25 +383,34 @@ def init_venv( verified_venvs.add(venv) return True - from meerschaum.utils.packages import run_python_package, attempt_import + from meerschaum.utils.packages import run_python_package, attempt_import, _get_pip_os_env global tried_virtualenv try: import venv as _venv + uv = attempt_import('uv', venv=None, debug=debug) virtualenv = None except ImportError: _venv = None + uv = None virtualenv = None - _venv_success = False - if _venv is not None: - import io - from contextlib import redirect_stdout + + if uv is not None: + _venv_success = run_python_package( + 'uv', + ['venv', venv_path.as_posix(), '-q'], + venv = None, + env = _get_pip_os_env(), + debug = debug, + ) == 0 + + if _venv is not None and not _venv_success: f = io.StringIO() with redirect_stdout(f): _venv_success = run_python_package( 'venv', - [str(venv_path)] + ( + [venv_path.as_posix()] + ( ['--symlinks'] if platform.system() != 'Windows' else [] ), venv=None, debug=debug @@ -438,7 +449,7 @@ def init_venv( except Exception as e: import traceback traceback.print_exc() - virtualenv.cli_run([str(venv_path)]) + virtualenv.cli_run([venv_path.as_posix()]) if dist_packages_path.exists(): vtp.mkdir(exist_ok=True, parents=True) for file_path in dist_packages_path.glob('*'): @@ -614,7 +625,7 @@ def venv_target_path( return site_path ### Allow for dist-level paths (running as root). - for possible_dist in reversed(site.getsitepackages()): + for possible_dist in site.getsitepackages(): dist_path = pathlib.Path(possible_dist) if not dist_path.exists(): continue @@ -698,4 +709,28 @@ def get_venvs() -> List[str]: return venvs +def get_module_venv(module) -> Union[str, None]: + """ + Return the virtual environment where an imported module is installed. + + Parameters + ---------- + module: ModuleType + The imported module to inspect. + + Returns + ------- + The name of a venv or `None`. + """ + import pathlib + from meerschaum.config.paths import VIRTENV_RESOURCES_PATH + module_path = pathlib.Path(module.__file__).resolve() + try: + rel_path = module_path.relative_to(VIRTENV_RESOURCES_PATH) + except ValueError: + return None + + return rel_path.as_posix().split('/', maxsplit=1)[0] + + from meerschaum.utils.venv._Venv import Venv diff --git a/requirements/api.txt b/requirements/api.txt index 01431a39..47e20de9 100644 --- a/requirements/api.txt +++ b/requirements/api.txt @@ -45,6 +45,7 @@ watchfiles>=0.21.0 dill>=0.3.3 virtualenv>=20.1.0 APScheduler>=4.0.0a5 +uv>=0.2.11 pprintpp>=0.4.0 asciitree>=0.3.3 typing-extensions>=4.7.1 diff --git a/requirements/full.txt b/requirements/full.txt index d47c1e1b..2205e8d0 100644 --- a/requirements/full.txt +++ b/requirements/full.txt @@ -28,6 +28,7 @@ watchfiles>=0.21.0 dill>=0.3.3 virtualenv>=20.1.0 APScheduler>=4.0.0a5 +uv>=0.2.11 cryptography>=38.0.1 psycopg[binary]>=3.1.18 PyMySQL>=0.9.0 diff --git a/scripts/docker/image_setup.sh b/scripts/docker/image_setup.sh index 37e43c4a..d0d4d2d2 100755 --- a/scripts/docker/image_setup.sh +++ b/scripts/docker/image_setup.sh @@ -13,7 +13,7 @@ groupadd -r $MRSM_USER -g $MRSM_GID \ apt-get update && apt-get install sudo curl less -y --no-install-recommends ### Install user-level build tools. -sudo -u $MRSM_USER python -m pip install --user --upgrade wheel pip setuptools +sudo -u $MRSM_USER python -m pip install --user --upgrade wheel pip setuptools uv if [ "$MRSM_DEP_GROUP" != "minimal" ]; then apt-get install -y --no-install-recommends \ @@ -37,8 +37,8 @@ if [ "$MRSM_DEP_GROUP" != "minimal" ]; then || exit 1 fi - sudo -u $MRSM_USER python -m pip install --no-cache-dir --upgrade --user psycopg || exit 1 - sudo -u $MRSM_USER python -m pip install --no-cache-dir --upgrade --user pandas || exit 1 + sudo -u $MRSM_USER python -m pip install \ + --no-cache-dir --upgrade --user psycopg pandas || exit 1 fi diff --git a/tests/docker-compose.yaml b/tests/docker-compose.yaml index d9ca1e99..2ceb66ad 100644 --- a/tests/docker-compose.yaml +++ b/tests/docker-compose.yaml @@ -28,7 +28,7 @@ services: SA_PASSWORD: "supersecureSECRETPASSWORD123!" ports: - "1439:1433" - image: "mcr.microsoft.com/mssql/server:2017-latest" + image: "mcr.microsoft.com/mssql/server:2022-latest" volumes: - "mssql_volume:/var/opt/mssql" diff --git a/tests/test_sync.py b/tests/test_sync.py index 82fe1795..030c7ef9 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -3,6 +3,7 @@ # vim:fenc=utf-8 import pytest +import sys from datetime import datetime, timedelta from decimal import Decimal from tests import debug @@ -589,6 +590,7 @@ def test_nested_chunks(flavor: str): assert len(df) == num_docs +@pytest.mark.skipif(sys.version_info >= (3, 12), reason="Dask / Numpy 2.0 failing on Python 3.12") @pytest.mark.parametrize("flavor", get_flavors()) def test_sync_dask_dataframe(flavor: str): """