✅ v1.4.4 In-place syncing enabled by default (#89)

* 🩹 v1.4.3 Set in-place syncing to False by default, add improvements to in-place syncing. * ✅ v1.4.4 In-place syncing improvements + more test coverage.
bmeares · Nov 1, 2022 · 5e9bb3f · 5e9bb3f
1 parent 9656e5e
commit 5e9bb3f
Show file tree

Hide file tree

Showing 9 changed files with 199 additions and 63 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,7 +4,7 @@
 
 This is the current release cycle, so stay tuned for future releases!
 
-### v1.4.1
+### v1.4.0 – v1.4.4
 
 - **Added in-place syncing for SQL pipes.**  
   This feature is big (enough to warrant a new point release). When pipes with the same instance connector and data source connector are synced, the method `sync_pipe_inplace()` is invoked. For SQL pipes, this means the entire syncing process will now happen entirely in SQL, which can lead to massive performance improvements.
@@ -27,7 +27,9 @@ This is the current release cycle, so stay tuned for future releases!
   pipe.sync()
   ```
 
-  To disable this behavior, run the command `edit config system` and set the value under the keys `experimental:inplace_sync` to `false`.
+  This applies even when the source table's schema changes, just like the dynamic columns feature added in v1.3.0.
+
+  > *To disable this behavior, run the command `edit config system` and set the value under the keys `experimental:inplace_sync` to `false`.*
 
 - **Added negation to `--params`.**  
   The [`build_where()`](https://docs.meerschaum.io/utils/sql.html#meerschaum.utils.sql.build_where) function now allows you to negate certain values when prefixed with an underscore (`_`):
@@ -46,6 +48,7 @@ This is the current release cycle, so stay tuned for future releases!
 - **Fixed environment issue when starting the Web API with `gunicorn`.**
 - **Added an emoji to the SQL Query option of the web console.**
 - **Fixed an edge case with data type enforcement.**
+- **Other bugfixes**
 
 ## 1.3.x Releases
 

diff --git a/docs/mkdocs/news/changelog.md b/docs/mkdocs/news/changelog.md
@@ -4,7 +4,7 @@
 
 This is the current release cycle, so stay tuned for future releases!
 
-### v1.4.1
+### v1.4.0 – v1.4.4
 
 - **Added in-place syncing for SQL pipes.**  
   This feature is big (enough to warrant a new point release). When pipes with the same instance connector and data source connector are synced, the method `sync_pipe_inplace()` is invoked. For SQL pipes, this means the entire syncing process will now happen entirely in SQL, which can lead to massive performance improvements.
@@ -27,7 +27,9 @@ This is the current release cycle, so stay tuned for future releases!
   pipe.sync()
   ```
 
-  To disable this behavior, run the command `edit config system` and set the value under the keys `experimental:inplace_sync` to `false`.
+  This applies even when the source table's schema changes, just like the dynamic columns feature added in v1.3.0.
+
+  > *To disable this behavior, run the command `edit config system` and set the value under the keys `experimental:inplace_sync` to `false`.*
 
 - **Added negation to `--params`.**  
   The [`build_where()`](https://docs.meerschaum.io/utils/sql.html#meerschaum.utils.sql.build_where) function now allows you to negate certain values when prefixed with an underscore (`_`):
@@ -46,6 +48,7 @@ This is the current release cycle, so stay tuned for future releases!
 - **Fixed environment issue when starting the Web API with `gunicorn`.**
 - **Added an emoji to the SQL Query option of the web console.**
 - **Fixed an edge case with data type enforcement.**
+- **Other bugfixes**
 
 ## 1.3.x Releases
 

diff --git a/meerschaum/api/routes/_pipes.py b/meerschaum/api/routes/_pipes.py
@@ -332,8 +332,8 @@ def get_pipe_data(
         connector_keys: str,
         metric_key: str,
         location_key: str,
-        begin: datetime.datetime = None,
-        end: datetime.datetime = None,
+        begin: str = None,
+        end: str = None,
         params: Optional[str] = None,
         orient: str = 'records',
         curr_user = (

diff --git a/meerschaum/config/_version.py b/meerschaum/config/_version.py
@@ -2,4 +2,4 @@
 Specify the Meerschaum release version.
 """
 
-__version__ = "1.4.1"
+__version__ = "1.4.4"
diff --git a/meerschaum/config/stack/grafana/__init__.py b/meerschaum/config/stack/grafana/__init__.py
@@ -10,36 +10,36 @@
 from meerschaum.config.stack import db_host, db_user, db_pass, db_port, db_base
 
 default_datasource = {
-    'apiVersion' : 1,
-    'datasources' : [
+    'apiVersion': 1,
+    'datasources': [
         {
-            'name' : 'Meerschaum Main',
-            'type' : 'postgres',
-            'jsonData' : {
-                'sslmode' : 'disable',
-                'postgresVersion' : 1400,
-                'timescaledb' : True,
+            'name': 'Meerschaum Main',
+            'type': 'postgres',
+            'jsonData': {
+                'sslmode': 'disable',
+                'postgresVersion': 1400,
+                'timescaledb': True,
             },
-            'user' : db_user,
-            'secureJsonData' : {
-                'password' : db_pass,
+            'user': db_user,
+            'secureJsonData': {
+                'password': db_pass,
             },
-            'database' : db_base,
-            'url' : db_host + ':' + str(db_port),
-            'isDefault' : True,
-            'editable' : True,
+            'database': db_base,
+            'url': db_host + ':5432',
+            'isDefault': True,
+            'editable': True,
         }
     ],
 }
 
 default_dashboard = {
-    'apiVersion' : 1,
-    'providers' : [
+    'apiVersion': 1,
+    'providers': [
         {
-            'name' : 'Default',
-            'folder' : 'Meerschaum Dashboards',
-            'options' : {
-                'path' : '/etc/grafana/provisioning/dashboards',
+            'name': 'Default',
+            'folder': 'Meerschaum Dashboards',
+            'options': {
+                'path': '/etc/grafana/provisioning/dashboards',
             },
         }
     ],
@@ -70,4 +70,3 @@ def edit_grafana(
         'dashboard.yaml' : GRAFANA_DASHBOARD_PATH,
     }
     return general_edit_config(action=action, files=files, default='datasource', debug=debug)
-
diff --git a/meerschaum/connectors/api/_pipes.py b/meerschaum/connectors/api/_pipes.py
@@ -225,7 +225,7 @@ def get_json_str(c):
 
     for i, c in enumerate(chunks):
         if debug:
-            dprint(f"Posting chunk ({i + 1} / {_chunksize}) to {r_url}...")
+            dprint(f"Posting chunk ({i + 1} / {len(chunks)}) to {r_url}...")
         json_str = get_json_str(c)
 
         try:
@@ -301,8 +301,8 @@ def delete_pipe(
 def get_pipe_data(
         self,
         pipe: meerschaum.Pipe,
-        begin: Optional[datetime.datetime] = None,
-        end: Optional[datetime.datetime] = None,
+        begin: Union[str, datetime.datetime, None] = None,
+        end: Union[str, datetime.datetime, None] = None,
         params: Optional[Dict[str, Any]] = None,
         as_chunks: bool = False,
         debug: bool = False,