v1.2.40

- Implemented Statistics and DataProcessing API
- Added automatic parameter values history and statistics (PSRAM required)

src/HomeGenie.cpp

@@ -56,6 +56,13 @@ namespace Service {
         addIOHandler(gpioPort);
         auto homeGenieHandler = new HomeGenieHandler(gpioPort);
         addAPIHandler(homeGenieHandler);
+
+#ifndef DISABLE_DATA_PROCESSING
+        Statistics();
+        auto dataProcessingHandler = new DataProcessingHandler();
+        addAPIHandler(dataProcessingHandler);
+#endif
+
 #ifdef CONFIG_ENABLE_POWER_MANAGER
         PowerManager::setWakeUpInterval(0);
         PowerManager::init();
@@ -326,6 +333,107 @@ namespace Service {
     }
 
 
+#ifndef DISABLE_DATA_PROCESSING
+    unsigned int HomeGenie::writeParameterHistoryJSON(ModuleParameter* parameter, ResponseCallback *outputCallback, int pageNumber, int pageSize, double rangeStart, double rangeEnd, double maxWidth) {
+        bool firstItem = true;
+        auto count = Statistics::getHistorySize();
+        if (pageSize == 0) pageSize = STATS_HISTORY_RESULTS_DEFAULT_PAGE_SIZE;
+        int currentPage = 0;
+        int i = 0;
+        // normalize range start/end
+        if (rangeEnd > 0) {
+            for (; i < count; i++) {
+                auto statValue = Statistics::getHistory(i);
+                if (statValue->timestamp <= rangeEnd) {
+                    rangeEnd = statValue->timestamp;
+                    break;
+                }
+            }
+        }
+        if (rangeStart > 0) {
+            double start = rangeStart;
+            for (; i < count; i++) {
+                auto statValue = Statistics::getHistory(i);
+                if (statValue->timestamp >= rangeStart) {
+                    start = statValue->timestamp;
+                } else {
+                    break;
+                }
+            }
+            rangeStart = start;
+        }
+
+        double step = maxWidth > 0 ? (rangeEnd - rangeStart) / maxWidth : 0;
+        String out = R"({"HistoryLimit": )" + String("1440") + R"(, "HistoryLimitSize": 2000, "History": [)";
+        outputCallback->write(out.c_str());
+        int itemCount = 0;
+        for (int i = 0; i < count; i++) {
+            auto statValue = Statistics::getHistory(i);
+
+            double timestamp = statValue->timestamp;
+            double value = statValue->value;
+
+            if (statValue->parameter != parameter)
+                continue;
+
+            if (rangeStart > 0 && timestamp < rangeStart)
+                break;
+            if (rangeEnd > 0 && timestamp > rangeEnd)
+                continue;
+
+            if (step > 0) {
+                auto ts = timestamp - fmod(timestamp, step) + step;
+                int tot = 1;
+                for (int j = i + 1; j < count; j++) {
+                    statValue = Statistics::getHistory(j);
+                    if (statValue->timestamp < ts - step || statValue->timestamp < rangeStart) break;
+                    if (statValue->parameter == parameter) {
+                        value += statValue->value;
+                        tot++;
+                    }
+                    i = j;
+                }
+                // average value in the given timestamp range
+                value = (value / tot);
+            }
+
+            if (currentPage == pageNumber) {
+                if (!firstItem) {
+                    out = ",\n";
+                } else {
+                    firstItem = false;
+                    out = "\n";
+                }
+
+                int milli = (timestamp * 1000.0f) - (trunc(timestamp) * 1000);
+                auto secs = (time_t) timestamp;
+                char buf[sizeof "1990-06-11T11:11:00.000Z"];
+                strftime(buf, sizeof buf, "%FT%T", gmtime(&secs));
+                sprintf(buf, "%s.%03dZ", buf, milli);
+
+                out += R"({"Value": )" + String(value) + R"(, "Timestamp": ")" + String(buf) +
+                       R"(", "UnixTimestamp": )" + String(timestamp * 1000, 3) + R"(})";
+
+                outputCallback->write(out.c_str());
+            }
+
+            itemCount++;
+            if (pageSize != -1 && itemCount == pageSize) {
+                if (currentPage == pageNumber) {
+                    break;
+                }
+                currentPage++;
+                itemCount = 0;
+            }
+
+        }
+        out = "\n]}\n";
+
+        outputCallback->write(out.c_str());
+        return outputCallback->contentLength;
+    }
+#endif
+
     String HomeGenie::createModuleParameter(const char *name, const char *value, const char *timestamp) {
         static const char *parameterTemplate = R"({
     "Name": "%s",

src/HomeGenie.h

@@ -51,6 +51,9 @@
 #include "net/NetManager.h"
 #include "service/api/APIRequest.h"
 #include "service/api/APIHandler.h"
+#ifndef DISABLE_DATA_PROCESSING
+#include "service/api/DataProcessingHandler.h"
+#endif
 #include "service/api/HomeGenieHandler.h"
 #include "service/EventRouter.h"
 
@@ -61,6 +64,9 @@ namespace Service {
 
     using namespace IO;
     using namespace Data;
+#ifndef DISABLE_DATA_PROCESSING
+    using namespace Data::Processing;
+#endif
     using namespace Net;
     using namespace Service::API;
 #ifndef DISABLE_AUTOMATION
@@ -131,7 +137,9 @@ namespace Service {
         unsigned int writeModuleListJSON(ResponseCallback *outputCallback);
         unsigned int writeModuleJSON(ResponseCallback *outputCallback, String* domain, String* address);
         unsigned int writeGroupListJSON(ResponseCallback *outputCallback);
-
+#ifndef DISABLE_DATA_PROCESSING
+        unsigned int writeParameterHistoryJSON(ModuleParameter* parameter, ResponseCallback *outputCallback, int pageNumber = 0, int pageSize = STATS_HISTORY_RESULTS_DEFAULT_PAGE_SIZE, double rangeStart = 0, double rangeEnd = 0, double maxWidth = 0);
+#endif
         static String createModule(const char *domain, const char *address, const char *name, const char* description, const char *deviceType, const char *parameters);
         static String createModuleParameter(const char *name, const char* value, const char *timestamp);
 

src/data/Statistics.cpp

@@ -0,0 +1,38 @@
+/*
+ * HomeGenie-Mini (c) 2018-2024 G-Labs
+ *
+ *
+ * This file is part of HomeGenie-Mini (HGM).
+ *
+ *  HomeGenie-Mini is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  HomeGenie-Mini is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with HomeGenie-Mini.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *
+ * Authors:
+ * - Generoso Martello <[email protected]>
+ *
+ */
+
+#include "Statistics.h"
+
+#ifndef DISABLE_DATA_PROCESSING
+
+namespace Data { namespace Processing {
+
+    int Statistics::historyIndex = 0;
+    bool Statistics::historyIsFull = false;
+    StatValue **Statistics::History = nullptr;
+
+}}
+
+#endif

src/data/Statistics.h

@@ -0,0 +1,101 @@
+/*
+ * HomeGenie-Mini (c) 2018-2024 G-Labs
+ *
+ *
+ * This file is part of HomeGenie-Mini (HGM).
+ *
+ *  HomeGenie-Mini is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  HomeGenie-Mini is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with HomeGenie-Mini.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *
+ * Authors:
+ * - Generoso Martello <[email protected]>
+ *
+ */
+
+#ifndef HOMEGENIE_MINI_STATISTICS_H
+#define HOMEGENIE_MINI_STATISTICS_H
+
+#include "io/IOEventData.h"
+#include "Module.h"
+
+#ifndef DISABLE_DATA_PROCESSING
+
+namespace Data { namespace Processing {
+
+    using IO::IOEventDataType;
+    using Data::ModuleParameter;
+
+    class StatValue {
+    public:
+        ModuleParameter *parameter;
+        double value;
+        double timestamp;
+
+        StatValue() {
+            parameter = nullptr;
+            value = 0;
+            timestamp = 0;
+        }
+    };
+
+    const unsigned int STATS_HISTORY_LIMIT = 2000;
+    const unsigned int STATS_HISTORY_RESULTS_DEFAULT_PAGE_SIZE = 100;
+
+    class Statistics {
+    public:
+        Statistics() {
+            History = (StatValue **) ps_malloc(STATS_HISTORY_LIMIT * sizeof(StatValue *));
+        }
+
+        static size_t getHistorySize() {
+            return historyIsFull ? STATS_HISTORY_LIMIT : historyIndex;
+        }
+
+        static StatValue *getHistory(int position) {
+            auto i = (historyIndex - position - 1);
+            if (i < 0) i = STATS_HISTORY_LIMIT - i;
+            return History[i % STATS_HISTORY_LIMIT];
+        }
+
+        static unsigned int collect(ModuleParameter *parameter) {
+            auto ci = historyIndex;
+            auto sv = (StatValue *) ps_malloc(sizeof(StatValue));
+            sv->parameter = parameter;
+            sv->value = parameter->value.toDouble();
+            // unix timestamp in milliseconds
+            sv->timestamp = ((double) Config::getRTC()->getEpoch()) + ((double) (millis() % 1000) / 1000.0f);
+
+            if (historyIsFull) free(History[ci]);
+            History[ci] = sv;
+
+            // circular / overwrite old data...
+            if (++historyIndex > STATS_HISTORY_LIMIT - 1) {
+                historyIndex = 0;
+                historyIsFull = true;
+            }
+
+            return ci;
+        }
+
+    private:
+        static int historyIndex;
+        static StatValue **History;
+        static bool historyIsFull;
+    };
+
+}}
+
+#endif // DISABLE_DATA_PROCESSING
+
+#endif //HOMEGENIE_MINI_STATISTICS_H

src/defs.h

@@ -91,6 +91,10 @@
 #endif
 
 
+#ifndef BOARD_HAS_PSRAM
+#define DISABLE_DATA_PROCESSING
+#endif
+
 
 #ifdef DISABLE_BLUETOOTH
     #define DISABLE_BLUETOOTH_LE

src/io/IOEvent.h

@@ -33,12 +33,18 @@
 #include "Config.h"
 
 #include "data/Module.h"
+#ifndef DISABLE_DATA_PROCESSING
+#include "data/Statistics.h"
+#endif
 #include "IOEventData.h"
 #include "IOEventPaths.h"
 
 namespace IO {
 
     using namespace Data;
+#ifndef DISABLE_DATA_PROCESSING
+    using namespace Data::Processing;
+#endif
 
     class IIOEventSender;
 
@@ -70,6 +76,14 @@ namespace IO {
                 if (eventsDisable == nullptr || eventsDisable->value == nullptr || eventsDisable->value != "1") {
                     eventReceiver->onIOEvent(this, module->domain.c_str(), module->address.c_str(), eventPath, eventData, dataType);
                 }
+#ifndef DISABLE_DATA_PROCESSING
+                if (isNumericDataType(dataType)) {
+                    auto moduleParameter = module->getProperty(eventPath);
+                    if (moduleParameter != nullptr) {
+                        Statistics::collect(moduleParameter);
+                    }
+                }
+#endif
             }
         };
 

src/io/IOEventData.h

@@ -26,6 +26,8 @@
 #ifndef HOMEGENIE_MINI_IOEVENTDATA_H
 #define HOMEGENIE_MINI_IOEVENTDATA_H
 
+#include <cstddef>
+
 namespace IO {
 
     enum IOEventDataType {
@@ -47,6 +49,15 @@ namespace IO {
         void* data;
         size_t type_size;
     };
+
+    inline bool isNumericDataType(IOEventDataType dataType) {
+        return (dataType == IOEventDataType::SensorLight ||
+        dataType == IOEventDataType::SensorHumidity ||
+        dataType == IOEventDataType::SensorTemperature ||
+        dataType == IOEventDataType::Float ||
+        dataType == IOEventDataType::UnsignedNumber ||
+        dataType == IOEventDataType::Number);
+    }
 }
 
 #endif //HOMEGENIE_MINI_IOEVENTDATA_H

src/io/IOEventDomains.h

@@ -36,6 +36,7 @@ namespace IO {
         const char HomeAutomation_X10[] = "HomeAutomation.X10";
         const char HomeAutomation_RemoteControl[] = "HomeAutomation.RemoteControl";
         const char Automation_Components[] = "Automation.Components";
+        const char DataProcessing_Filters[] = "DataProcessing.Filters";
     };
 }
 

src/service/api/APIRequest.h

@@ -68,6 +68,7 @@ namespace Service { namespace API {
         static const char Modules_List[] = {"Modules.List"};
         static const char Modules_Get[] = {"Modules.Get"};
         static const char Modules_ParameterSet[] = {"Modules.ParameterSet"};
+        static const char Modules_StatisticsGet[] = {"Modules.StatisticsGet"};
         static const char Groups_List[] = {"Groups.List"};
         static const char WebSocket_GetToken[] = {"WebSocket.GetToken"};
         static const char System_Configure[] = {"System.Configure"};