[export] Repair ggplot graph rendering by switching to plotnine

.github/workflows/tests.yml

@@ -22,7 +22,7 @@ jobs:
       fail-fast: true
       matrix:
         os: [ ubuntu-20.04 ] # , macos-latest, windows-latest ]
-        python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11" ]
+        python-version: [ "3.8", "3.9", "3.10", "3.11" ]
         mosquitto-version: [ "2.0" ]
         influxdb-version: [ "1.8" ]
         grafana-version: [ "7.5.17", "8.5.27", "9.5.7", "10.0.3" ]

CHANGES.rst

@@ -27,6 +27,8 @@ in progress
 - [docs] Refactor "decoders" section to "integrations", and improve index/overview page
 - [export] Improve export capabilities by adding parameters ``sort``, ``direction``,
   ``limit``, and ``scalar``. Thanks, @ClemensGruber.
+- [export] Repair ``ggplot`` graph rendering by switching to ``plotnine``
+- Drop support for Python 3.7
 
 
 .. _kotori-0.27.0:

Makefile

@@ -35,7 +35,7 @@ virtualenv-docs: setup-virtualenv
 # Install requirements for development.
 virtualenv-dev: setup-virtualenv
 	@$(pip) install --upgrade --prefer-binary --requirement=requirements-test.txt
-	@$(pip) install --upgrade --prefer-binary --editable=.[daq,daq_geospatial,export,scientific,firmware]
+	@$(pip) install --upgrade --prefer-binary --editable=.[daq,daq_geospatial,export,plotting,scientific,firmware]
 
 # Install requirements for releasing.
 install-releasetools: setup-virtualenv

kotori/io/export/plot.py

@@ -171,18 +171,28 @@ def render_png(self, buffer):
             """
 
 
-        elif renderer == 'ggplot':
-
-            # https://yhat.github.io/ggplot/notebook.html?page=build/docs/examples/Multiple%20Line%20Plot.html
-            # https://stackoverflow.com/questions/23541497/is-there-a-way-to-plot-a-pandas-series-in-ggplot
-            # https://stackoverflow.com/questions/24478925/is-it-possible-to-plot-multiline-chart-on-python-ggplot/24479513#24479513
-
-            # https://github.com/yhat/ggplot/blob/master/docs/how-to/Building%20Faceted%20(or%20Trellised)%20Plots.ipynb
-            # https://github.com/yhat/ggplot/blob/master/docs/how-to/Annotating%20Plots%20-%20Titles%20and%20Labels.ipynb
-            # https://github.com/yhat/ggplot/blob/master/docs/how-to/How%20to%20make%20xkcd%20style%20graphs.ipynb
+        elif renderer in ['ggplot', 'plotnine']:
+            """
+            Renderer based on ggplot, now in plotnine.
+
+            ggplot
+            ======
+            - https://yhat.github.io/ggplot/notebook.html?page=build/docs/examples/Multiple%20Line%20Plot.html
+            - https://stackoverflow.com/questions/23541497/is-there-a-way-to-plot-a-pandas-series-in-ggplot
+            - https://stackoverflow.com/questions/24478925/is-it-possible-to-plot-multiline-chart-on-python-ggplot/24479513#24479513
+
+            - https://github.com/yhat/ggplot/blob/master/docs/how-to/Building%20Faceted%20(or%20Trellised)%20Plots.ipynb
+            - https://github.com/yhat/ggplot/blob/master/docs/how-to/Annotating%20Plots%20-%20Titles%20and%20Labels.ipynb
+            - https://github.com/yhat/ggplot/blob/master/docs/how-to/How%20to%20make%20xkcd%20style%20graphs.ipynb
+
+            plotnine
+            ========
+            - https://github.com/has2k1/plotnine
+            - https://plotnine.readthedocs.io/
+            """
 
-            from ggplot import ggplot, aes, qplot, geom_line, geom_text, ggtitle, stat_smooth, scale_x_date, date_format, date_breaks
-            from ggplot import theme_538, theme_bw, theme_gray, theme_xkcd
+            from plotnine import ggplot, aes, qplot, geom_line, geom_text, ggtitle, stat_smooth, scale_x_datetime
+            from plotnine import theme_538, theme_bw, theme_gray, theme_xkcd
 
             # https://stackoverflow.com/questions/24478925/is-it-possible-to-plot-multiline-chart-on-python-ggplot/24479513#24479513
             # https://stackoverflow.com/questions/23541497/is-there-a-way-to-plot-a-pandas-series-in-ggplot
@@ -196,9 +206,12 @@ def render_png(self, buffer):
 
             plot = ggplot(df, aes(x='time', y='value', color='variable'))\
                    + geom_line()\
-                   + scale_x_date(limits=(datetime_min, datetime_max), breaks=locator, labels=formatter)\
+                   + scale_x_datetime(limits=(datetime_min, datetime_max))\
                    + ggtitle(bucket.title.human)
 
+            # FIXME: Currently croaks.
+            # + scale_x_date(limits=(datetime_min, datetime_max), breaks=locator, labels=formatter) \
+
             # Axis labels
             plot.xlab = 'Time'
             plot.ylab = 'Value'

setup.py

@@ -6,15 +6,10 @@
 here = os.path.abspath(os.path.dirname(__file__))
 README = open(os.path.join(here, 'README.rst')).read()
 
-PYTHON_35 = (3, 5) <= sys.version_info < (3, 6)
 PYTHON_GTE_310 = sys.version_info >= (3, 10)
 PYTHON_LT_311 = sys.version_info < (3, 11)
 
-# Pandas on Python 3.5 will attempt to install `numpy==1.20.1`,
-# which will bail out on aarch64 with `RuntimeError: Python version >= 3.7 required.`.
 numpy_spec = "numpy>=1.18,<1.22"
-if PYTHON_35:
-    numpy_spec = "numpy==1.18.5"
 
 pandas_spec = "pandas<1.3"
 if PYTHON_GTE_310:
@@ -102,11 +97,9 @@
 
     'plotting': [
         #'dyplot==0.8.8',
-
-        'matplotlib>=3,<3.4',
+        'matplotlib>=3.7,<3.8',
+        'plotnine<0.13',
         #'cairocffi>=0.5.4',
-        'bokeh>=1.4.0,<3.1',
-        'vincent>=0.4.4,<0.5',
     ],
 
     # Data export: Scientific data formats like HDF5 and NetCDF and plots from ggplot
@@ -124,15 +117,16 @@
         # Algorithms
         # ----------
         #'scipy>=1.4.1,<1.6',
-        'ggplot>=0.11.5,<0.12',
 
         # gfortran
         # aptitude install libatlas-base-dev lapack-dev gfortran or
         # https://gcc.gnu.org/wiki/GFortranBinaries
 
         # Visualization
         # -------------
+        'bokeh>=1.4.0,<3.1',
         #'seaborn==0.7.1',
+        'vincent>=0.4.4,<0.5',
 
     ],
 
@@ -161,7 +155,6 @@
       classifiers=[
         "Programming Language :: Python",
         "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",

test/test_export.py

@@ -246,3 +246,57 @@ def test_export_hdf5(machinery, create_influxdb, reset_influxdb):
     assert hdf["/itest_foo_bar/table"][()] == array(
         [(1583810982000000000, 51.8, 25.26)],
         dtype=[('index', '<i8'), ('humidity', '<f8'), ('temperature', '<f8')])
+
+
+@pytest_twisted.inlineCallbacks
+@pytest.mark.http
+@pytest.mark.export
+def test_export_png(machinery, create_influxdb, reset_influxdb):
+    """
+    Verify `sort` and `direction` transformation parameters of HTTP export API.
+    """
+
+    # Submit two measurements, with timestamp.
+    data = {
+        'time': 1583810982,
+        'temperature': 25.26,
+        'humidity': 51.8,
+    }
+    yield threads.deferToThread(http_json_sensor, settings.channel_path_data, data)
+
+    data = {
+        'time': 1583810993,
+        'temperature': 32.26,
+        'humidity': 64.8,
+    }
+    yield threads.deferToThread(http_json_sensor, settings.channel_path_data, data)
+
+    # Wait for some time to process the message.
+    yield sleep(PROCESS_DELAY_MQTT)
+
+    # PNG: matplotlib.
+    deferred = threads.deferToThread(http_get_data, settings.channel_path_data, format="png", params={
+        "from": ts_from,
+        "to": ts_to,
+    })
+    yield deferred
+    assert deferred.result.startswith(b"\x89PNG")
+
+    # PNG: ggplot.
+    deferred = threads.deferToThread(http_get_data, settings.channel_path_data, format="png", params={
+        "from": ts_from,
+        "to": ts_to,
+        "renderer": "ggplot",
+    })
+    yield deferred
+    assert deferred.result.startswith(b"\x89PNG")
+
+    # PNG: ggplot/xkcd.
+    deferred = threads.deferToThread(http_get_data, settings.channel_path_data, format="png", params={
+        "from": ts_from,
+        "to": ts_to,
+        "renderer": "ggplot",
+        "theme": "xkcd",
+    })
+    yield deferred
+    assert deferred.result.startswith(b"\x89PNG")