🔒️ v1.4.14 (#97) Security patch and add temporary to avoid automati…

…c table creation.
bmeares · Dec 5, 2022 · c3094a7 · c3094a7
2 parents edc4d23 + 78d6a7b
commit c3094a7
Show file tree

Hide file tree

Showing 22 changed files with 251 additions and 115 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,6 +4,59 @@
 
 This is the current release cycle, so stay tuned for future releases!
 
+### v1.4.14
+
+- **Added flag `temporary` to `Pipe` (and `--temporary`).**  
+  Pipes built with `temporary=True`, will not create instance tables (`pipes`, `users`, and `plugins`) or be able to modify registration. This is particularly useful when creating pipes from existing tables when automatic registration is not desired.
+
+  ```python
+  import meerschaum as mrsm
+  import pandas as pd
+  conn = mrsm.get_connector('sql:temp', uri='postgresql://user:pass@localhost:5432/db')
+
+  ### Simulating an existing table.
+  table_name = 'my_table'
+  conn.to_sql(
+      pd.DataFrame([{'id_column': 1, 'value': 1.0}]),
+      name = table_name,
+  )
+
+  ### Create a temporary pipe with the existing table as its target.
+  pipe = mrsm.Pipe(
+      'foo', 'bar',
+      target = table_name,
+      temporary = True,
+      instance = conn,
+      columns = {
+          'id': 'id_column',
+      },
+  )
+
+  docs = [
+      {
+          "id_column": 1,
+          "value": 123.456,
+          "new_column": "hello, world!",
+      },
+  ]
+
+  ### Existing table `my_table` is synced without creating other tables
+  ### or affecting pipes' registration.
+  pipe.sync(docs)
+  ```
+
+- **Fixed potential security of public instance tables.**  
+  The API now refuses to sync or serve data if the target is a protected instance table (`pipes`, `users`, or `plugins`).
+
+- **Added not-null check to `pipe.get_sync_time().`**  
+  The `datetime` column should never contain null values, but just in case, `pipe.get_sync_time()` now passes a not-null check to `params` for the datetime column.
+
+- **Removed prompt for `value` from `pipe.bootstrap()`.**  
+  The prompt for an optional `value` column has been removed from the bootstrapping wizard because `pipe.columns` is now largely used as a collection of indices rather than the original purpose of meta-columns.
+
+- **Pass `--debug` and other flags in `copy pipes`.**  
+  Command line flags are now passed to the new pipe when copying an existing pipe.
+
 ### v1.4.12 – v1.4.13
 
 - **Fixed an issue when syncing empty DataFrames [(#95)](https://github.com/bmeares/Meerschaum/issues/95).**  

diff --git a/docs/mkdocs/news/changelog.md b/docs/mkdocs/news/changelog.md
@@ -4,6 +4,59 @@
 
 This is the current release cycle, so stay tuned for future releases!
 
+### v1.4.14
+
+- **Added flag `temporary` to `Pipe` (and `--temporary`).**  
+  Pipes built with `temporary=True`, will not create instance tables (`pipes`, `users`, and `plugins`) or be able to modify registration. This is particularly useful when creating pipes from existing tables when automatic registration is not desired.
+
+  ```python
+  import meerschaum as mrsm
+  import pandas as pd
+  conn = mrsm.get_connector('sql:temp', uri='postgresql://user:pass@localhost:5432/db')
+
+  ### Simulating an existing table.
+  table_name = 'my_table'
+  conn.to_sql(
+      pd.DataFrame([{'id_column': 1, 'value': 1.0}]),
+      name = table_name,
+  )
+
+  ### Create a temporary pipe with the existing table as its target.
+  pipe = mrsm.Pipe(
+      'foo', 'bar',
+      target = table_name,
+      temporary = True,
+      instance = conn,
+      columns = {
+          'id': 'id_column',
+      },
+  )
+
+  docs = [
+      {
+          "id_column": 1,
+          "value": 123.456,
+          "new_column": "hello, world!",
+      },
+  ]
+
+  ### Existing table `my_table` is synced without creating other tables
+  ### or affecting pipes' registration.
+  pipe.sync(docs)
+  ```
+
+- **Fixed potential security of public instance tables.**  
+  The API now refuses to sync or serve data if the target is a protected instance table (`pipes`, `users`, or `plugins`).
+
+- **Added not-null check to `pipe.get_sync_time().`**  
+  The `datetime` column should never contain null values, but just in case, `pipe.get_sync_time()` now passes a not-null check to `params` for the datetime column.
+
+- **Removed prompt for `value` from `pipe.bootstrap()`.**  
+  The prompt for an optional `value` column has been removed from the bootstrapping wizard because `pipe.columns` is now largely used as a collection of indices rather than the original purpose of meta-columns.
+
+- **Pass `--debug` and other flags in `copy pipes`.**  
+  Command line flags are now passed to the new pipe when copying an existing pipe.
+
 ### v1.4.12 – v1.4.13
 
 - **Fixed an issue when syncing empty DataFrames [(#95)](https://github.com/bmeares/Meerschaum/issues/95).**  

diff --git a/meerschaum/_internal/arguments/_parser.py b/meerschaum/_internal/arguments/_parser.py
@@ -196,6 +196,7 @@ def get_arguments_triggers() -> Dict[str, Tuple[str]]:
     '-t', '--tags', nargs='+', help="Only include pipes with these tags.",
 )
 
+
 ### Sync options
 groups['sync'].add_argument(
     '--min-seconds', '--cooldown', type=float, help=(
@@ -315,7 +316,14 @@ def get_arguments_triggers() -> Dict[str, Tuple[str]]:
         "--params key1:value1,key2:value2"
     )
 )
-
+groups['misc'].add_argument(
+    '--temporary', '--temp',
+    action = 'store_true',
+    help = (
+        "Skip creating or modifying instance tables when working with pipes "
+        + "(plugins and users still trigger table creation)."
+    ),
+)
 groups['misc'].add_argument(
     '--gui', action='store_true',
     help="Open a DataFrame in an interactive pandasgui or matplotlib window."

diff --git a/meerschaum/actions/copy.py b/meerschaum/actions/copy.py
@@ -32,6 +32,7 @@ def copy(
     }
     return choose_subaction(action, options, **kw)
 
+
 def _complete_copy(
         action : Optional[List[str]] = None,
         **kw : Any
@@ -60,6 +61,7 @@ def _complete_copy(
     from meerschaum._internal.shell import default_action_completer
     return default_action_completer(action=(['copy'] + action), **kw)
 
+
 def _copy_pipes(
         yes: bool = False,
         noask: bool = False,
@@ -115,7 +117,7 @@ def _copy_pipes(
                     noask=noask, yes=yes
                 )
         ):
-            _new_pipe.sync(p.get_data(debug=debug, **kw))
+            _new_pipe.sync(p.get_data(debug=debug, **kw), debug=debug, **kw)
 
     msg = (
         "No pipes were copied." if successes == 0

diff --git a/meerschaum/api/__init__.py b/meerschaum/api/__init__.py
@@ -85,7 +85,7 @@ def get_uvicorn_config() -> Dict[str, Any]:
 _include_dash = (not no_dash)
 
 connector = None
-def get_api_connector(instance_keys : Optional[str] = None):
+def get_api_connector(instance_keys: Optional[str] = None):
     """Create the instance connector."""
     from meerschaum.utils.debug import dprint
     global connector

diff --git a/meerschaum/api/routes/_pipes.py b/meerschaum/api/routes/_pipes.py
@@ -308,6 +308,12 @@ def sync_pipe(
     if data is None:
         data = {}
     p = get_pipe(connector_keys, metric_key, location_key)
+    if p.target in ('users', 'plugins', 'pipes'):
+        raise fastapi.HTTPException(
+            status_code = 409,
+            detail = f"Cannot sync data to protected table '{p.target}'.",
+        )
+
     if not p.columns and columns is not None:
         p.columns = json.loads(columns)
     if not p.columns and not is_pipe_registered(p, pipes(refresh=True)):
@@ -363,7 +369,13 @@ def get_pipe_data(
     if not is_pipe_registered(p, pipes(refresh=True)):
         raise fastapi.HTTPException(
             status_code = 409,
-            detail = "Pipe must be registered with the datetime column specified"
+            detail = "Pipe must be registered with the datetime column specified."
+        )
+
+    if p.target in ('users', 'plugins', 'pipes'):
+        raise fastapi.HTTPException(
+            status_code = 409,
+            detail = f"Cannot retrieve data from protected table '{p.target}'.",
         )
 
     #  chunks = p.get_data(

diff --git a/meerschaum/config/_version.py b/meerschaum/config/_version.py
@@ -2,4 +2,4 @@
 Specify the Meerschaum release version.
 """
 
-__version__ = "1.4.13"
+__version__ = "1.4.14"
diff --git a/meerschaum/connectors/api/_delete.py b/meerschaum/connectors/api/_delete.py
@@ -17,27 +17,7 @@ def delete(
         debug : bool = False,
         **kw : Ahy,
     ) -> requests.Response:
-    """Wrapper for requests.delete
-
-    Parameters
-    ----------
-    r_url : str :
-        
-    headers : Optional[Dict[str :
-        
-    Any]] :
-         (Default value = None)
-    use_token : bool :
-         (Default value = True)
-    debug : bool :
-         (Default value = False)
-    **kw : Ahy :
-        
-
-    Returns
-    -------
-
-    """
+    """Wrapper for `requests.delete`."""
     if debug:
         from meerschaum.utils.debug import dprint
 
@@ -51,10 +31,7 @@ def delete(
 
     if debug:
         from meerschaum.utils.formatting import pprint
-        dprint(f"Sending DELETE request to {self.url + r_url}")
-        if headers:
-            pprint(headers)
-        pprint(kw)
+        dprint(f"Sending DELETE request to {self.url + r_url}.")
 
     return self.session.delete(
         self.url + r_url,

diff --git a/meerschaum/connectors/api/_get.py b/meerschaum/connectors/api/_get.py
@@ -11,11 +11,11 @@
 
 def get(
         self,
-        r_url : str,
-        headers : Optional[Dict[str, str]] = None,
-        use_token : bool = True,
-        debug : bool = False,
-        **kw : Any
+        r_url: str,
+        headers: Optional[Dict[str, str]] = None,
+        use_token: bool = True,
+        debug: bool = False,
+        **kw: Any
     ) -> requests.Reponse:
     """Wrapper for `requests.get`."""
     if debug:
@@ -27,14 +27,11 @@ def get(
     if use_token:
         if debug:
             dprint(f"Checking login token.")
-        headers.update({ 'Authorization': f'Bearer {self.token}' })
+        headers.update({'Authorization': f'Bearer {self.token}'})
 
     if debug:
         from meerschaum.utils.formatting import pprint
         dprint(f"Sending GET request to {self.url + r_url}.")
-        if headers:
-            pprint(headers)
-        pprint(kw)
 
     return self.session.get(
         self.url + r_url,
@@ -44,12 +41,12 @@ def get(
 
 def wget(
         self,
-        r_url : str,
-        dest : Optional[Union[str, pathlib.Path]] = None,
+        r_url: str,
+        dest: Optional[Union[str, pathlib.Path]] = None,
         headers: Optional[Dict[str, Any]] = None,
         use_token: bool = True,
         debug: bool = False,
-        **kw : Any
+        **kw: Any
     ) -> pathlib.Path:
     """Mimic wget with requests.
     """
@@ -60,5 +57,5 @@ def wget(
     if use_token:
         if debug:
             dprint(f"Checking login token.")
-        headers.update({ 'Authorization': f'Bearer {self.token}' })
+        headers.update({'Authorization': f'Bearer {self.token}'})
     return wget(self.url + r_url, dest=dest, headers=headers, **kw)
diff --git a/meerschaum/connectors/api/_patch.py b/meerschaum/connectors/api/_patch.py
@@ -11,33 +11,13 @@
 
 def patch(
         self,
-        r_url : str,
-        headers : Optional[Dict[str, Any]] = None,
-        use_token : bool = True,
-        debug : bool = False,
-        **kw : Any
+        r_url: str,
+        headers: Optional[Dict[str, Any]] = None,
+        use_token: bool = True,
+        debug: bool = False,
+        **kw: Any
     ) -> requests.Response:
-    """Wrapper for requests.patch
-
-    Parameters
-    ----------
-    r_url : str :
-        
-    headers : Optional[Dict[str :
-        
-    Any]] :
-         (Default value = None)
-    use_token : bool :
-         (Default value = True)
-    debug : bool :
-         (Default value = False)
-    **kw : Any :
-        
-
-    Returns
-    -------
-
-    """
+    """Wrapper for `requests.patch`."""
     if debug:
         from meerschaum.utils.debug import dprint
 
@@ -52,9 +32,6 @@ def patch(
     if debug:
         from meerschaum.utils.formatting import pprint
         dprint(f"Sending PATCH request to {self.url + r_url}")
-        if headers:
-            pprint(headers)
-        pprint(kw)
 
     return self.session.patch(
         self.url + r_url,

diff --git a/meerschaum/connectors/api/_post.py b/meerschaum/connectors/api/_post.py
@@ -27,14 +27,11 @@ def post(
     if use_token:
         if debug:
             dprint(f"Checking token...")
-        headers.update({ 'Authorization': f'Bearer {self.token}' })
+        headers.update({'Authorization': f'Bearer {self.token}'})
 
     if debug:
         from meerschaum.utils.formatting import pprint
         dprint(f"Sending POST request to {self.url + r_url}")
-        if headers:
-            pprint(headers)
-        pprint(kw)
 
     return self.session.post(
         self.url + r_url,