Merge pull request #157 from GeographicaGS/dev

Dev into Master

docs/reference/devices.md

@@ -159,6 +159,8 @@ It returns a time serie of a variable (or a set of variables).
 - vars (mandatory): array of all variables ids.
 - start (mandatory): date start. Format is 'YYYY-MM-DD HH:MM'. Date time is UTC.
 - finish (mandatory): date finish. Format is 'YYYY-MM-DD HH:MM'. Date time is UTC.
+- format (optional): If its value is `csv` it will return a CSV file.
+- data_tz (optional, in conjunction with format): Will parse dates info into local user time zone.
 - bbox (optional) : [lx,ly,ux,uy]
 
 
@@ -172,6 +174,8 @@ Payload:
   },
   "filters": {
     "bbox": [-5.11170,37.24000,-5.10818,37.24303]
-  }
+  },
+  "format": "csv",
+  "data_tz": "Europe/Madrid"
 }
 ```

docs/reference/entities.md

@@ -48,6 +48,79 @@ Response:
 }]
 ```
 
+### POST /:scope/entities/search/extended
+
+Extends the functionalities of `/entities/search`
+
+Params:
+- entities (mandatory): entities to search. It's an object using the entities names as keys and the following attributes as values:
+  - select (mandatory): fields to retrieve. Array
+  - suffix (optional): suffix to append to the entity associated table_name. String
+  - filters (mandatory): the same functionalities used in other endpoints. Object
+
+Request sample
+```json
+{
+    "entities": {
+        "fire_detection.fireforestobserved": {
+            "select": [
+                "id_entity",
+                "hasfirealert"
+            ],
+            "suffix": "_lastdata",
+            "filters": {
+                "condition": {
+                    "OR": {
+                        "id_entity__icontains": "Node9",
+                        "hasfirealert__eq": 1
+                    }
+                }
+            }
+        },
+        "fire_detection.weatherobserved": {
+            "select": [
+                "id_entity",
+                "battery"
+            ],
+            "suffix": "_lastdata",
+            "filters": {
+                "condition": {
+                    "AND": {
+                        "id_entity__icontains": "3A"
+                    }
+                }
+            }
+        }
+    }
+}
+```
+
+Response:
+```json
+{
+    "fire_detection.fireforestobserved": [
+        {
+            "id_entity": "fireForestObserved:Node9",
+            "hasfirealert": 0
+        },
+        {
+            "id_entity": "fireForestObserved:Simul1",
+            "hasfirealert": 1
+        },
+        {
+            "id_entity": "fireForestObserved:Simul2",
+            "hasfirealert": 1
+        }
+    ],
+    "fire_detection.weatherobserved": [
+        {
+            "id_entity": "weatherObserved:Node3A",
+            "battery": 78.647
+        }
+    ]
+}
+```
+
 ### GET /:scope/entities/map/counters
 
 It returns the number of elements by entities. If bbox param is specified the 'filter' value is the number of elements inside the viewport.

docs/reference/variables.md

@@ -135,6 +135,7 @@ It returns a time serie of a variable (or a set of variables).
 - step (optional): time resolution. Values: 1h, 2h, 4h, 1d (default: 1d)
 - bbox (optional) : [lx,ly,ux,uy]
 - findTimes  : true | false. Default to false. If set to true and max or min aggregator are used, it returns the list of "TimeInstant" where the max or min appears.
+- noData (optional): true | false. Default to false. If true, it returns a registry for every time (aggregated by the step), even if there is no data (in that case, it's going to return null).
 
 Payload:
 ```json
@@ -147,6 +148,7 @@ Payload:
     "step": "1d"
   },
   "findTimes" : true,
+  "noData": false,
   "filters": {
     "bbox": [-5.11170,37.24000,-5.10818,37.24303]
   }

models/devicesmodel.js

@@ -427,7 +427,12 @@ DevicesModel.prototype.getDevicesRawData = function(opts, cb) {
     return Promise.all(promises);
   }).bind(this))
     .then(function(results) {
-      return new DevicesFormatter().rawData(results);
+      if (opts.data_tz && opts.format) {
+        return new DevicesFormatter().parsedData(results, opts.data_tz);
+      } else {
+        return new DevicesFormatter().rawData(results);
+      }
+
     })
     .catch(function(err) {
       return Promise.reject(err);

models/entitiesmodel.js

@@ -1,21 +1,21 @@
-// Copyright 2017 Telefónica Digital España S.L.
-//
-// This file is part of UrboCore API.
-//
-// UrboCore API is free software: you can redistribute it and/or
-// modify it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// UrboCore API is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
-// General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with UrboCore API. If not, see http://www.gnu.org/licenses/.
-//
-// For those usages not covered by this license please contact with
+// Copyright 2017 Telefónica Digital España S.L.
+//
+// This file is part of UrboCore API.
+//
+// UrboCore API is free software: you can redistribute it and/or
+// modify it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// UrboCore API is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
+// General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with UrboCore API. If not, see http://www.gnu.org/licenses/.
+//
+// For those usages not covered by this license please contact with
 // iot_support at tid dot es
 
 'use strict';
@@ -30,6 +30,7 @@ var utils = require('../utils');
 var modelutils = require('./modelutils.js');
 var log = utils.log();
 var auth = require('../auth.js');
+var QueryBuilder = require('../protools/querybuilder');
 
 function EntitiesModel(cfg) {
   PGSQLModel.call(this,cfg);
@@ -233,6 +234,47 @@ EntitiesModel.prototype.searchElements = function(opts,cb) {
   }).bind(this));
 }
 
+EntitiesModel.prototype.searchElementsExtended = function(scope, entities) {
+  const metadata = new MetadataInstanceModel();
+  return metadata
+    .getEntsForSearch(scope, Object.keys(entities))
+    .then(function(d) {
+
+      const promises = d.rows.map(((entityData) => {
+
+        const entitySelect = ['', ..._.without(entities[entityData.id_entity].select, ['id_entity', 'position'])];
+
+        const qb = new QueryBuilder(entities[entityData.id_entity]);
+        const queryFilter = `${qb.bbox()} ${qb.filter()}`;
+
+        const suffix = entities[entityData.id_entity].suffix || '';
+        return this.cachedQuery(`
+          SELECT DISTINCT ON (id_entity) id_entity, ST_AsGeoJSON(position) as geometry ${entitySelect.join(', ')}
+          FROM ${entityData.dbschema}.${entityData.entity_table_name}${suffix}
+          WHERE TRUE ${queryFilter}
+        `);
+      }).bind(this)); 
+
+      return Promise.all(promises)
+        .then(dataResult => {
+          // group result by id_entity
+          return Promise.resolve(_.reduce(dataResult, (result, r, i) => {
+            const rows = r.rows.map((v)=>{
+              return Object.assign(v, {
+                geometry: v.geometry ? JSON.parse(v.geometry).coordinates : null
+              });
+            });
+            return Object.assign(result, {
+              [d.rows[i].id_entity]: rows
+            });
+          }, {}));
+        });
+    }.bind(this))
+    .catch(function(err) {
+      return Promise.reject(err);
+    });
+}
+
 EntitiesModel.prototype._getNonRegisteredEntities = function(term,limit) {
   var qemp = ['(SELECT \'placement\' as type,park_name as name,id as element_id,BOX2D(the_geom) as pbbox',
     'FROM parques WHERE park_name ILIKE unaccent(\'%'+term+'%\')',

models/variablesmodel.js

@@ -183,7 +183,26 @@ VariablesModel.prototype.getVariablesTimeSerie = function(opts) {
               )`;
           }
 
-          var sql = `
+          if (opts.noData) {
+            var sql = `
+            WITH filtered AS (
+               SELECT
+                 DISTINCT id_entity
+               FROM ${schema}.${entityTables[i]}_lastdata
+               WHERE true
+               ${qb.bbox()}
+               ${qb.filter()}
+            )
+            SELECT
+              _timeserie AS start,
+              (_timeserie + '${step}')::timestamp AS finish,
+              ${aggs[i]}(foo."${varNames[i]}") AS "${varIds[i]}_${aggs[i]}"${ cGroupAlias }
+            FROM generate_series('${opts.start}'::timestamp, '${opts.finish}'::timestamp, '${ step }') AS _timeserie
+            LEFT JOIN ${ from } foo
+            ON foo."TimeInstant" >= _timeserie AND foo."TimeInstant" < _timeserie + '${ step }' and id_entity IN (SELECT id_entity FROM filtered)
+            GROUP BY _timeserie${ cGroupAlias } ORDER BY _timeserie`;
+          } else {
+            var sql = `
             WITH filtered AS (
                SELECT
                  DISTINCT id_entity
@@ -201,6 +220,7 @@ VariablesModel.prototype.getVariablesTimeSerie = function(opts) {
             ON foo."TimeInstant" >= _timeserie AND foo."TimeInstant" < _timeserie + '${ step }'
             WHERE id_entity IN (SELECT id_entity FROM filtered)
             GROUP BY _timeserie${ cGroupAlias } ORDER BY _timeserie`;
+          }
 
           if (opts.findTimes && (aggs[i] === 'MIN' || aggs[i] === 'MAX')) {
             var preSQL = `
@@ -221,7 +241,6 @@ VariablesModel.prototype.getVariablesTimeSerie = function(opts) {
 
             sql = `${preSQL} ${sql} ${postSQL}`;
           }
-
           return this.cachedQuery(sql);
 
         }).bind(this)());

protools/conditions.js

@@ -1,21 +1,21 @@
-// Copyright 2017 Telefónica Digital España S.L.
-// 
-// This file is part of UrboCore API.
-// 
-// UrboCore API is free software: you can redistribute it and/or
-// modify it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-// 
-// UrboCore API is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
-// General Public License for more details.
-// 
-// You should have received a copy of the GNU Affero General Public License
-// along with UrboCore API. If not, see http://www.gnu.org/licenses/.
-// 
-// For those usages not covered by this license please contact with
+// Copyright 2017 Telefónica Digital España S.L.
+// 
+// This file is part of UrboCore API.
+// 
+// UrboCore API is free software: you can redistribute it and/or
+// modify it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+// 
+// UrboCore API is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
+// General Public License for more details.
+// 
+// You should have received a copy of the GNU Affero General Public License
+// along with UrboCore API. If not, see http://www.gnu.org/licenses/.
+// 
+// For those usages not covered by this license please contact with
 // iot_support at tid dot es
 
 'use strict';
@@ -92,6 +92,8 @@ Condition.prototype.getOp = function() {
     return 'LIKE';
   } else if (this.filter === 'regex') {
     return '~';
+  } else if (this.filter === 'icontains') {
+    return 'ILIKE';
   }
 }
 
@@ -111,6 +113,8 @@ Condition.prototype.toSQL = function() {
       return ' false ';
     }
     else return '';
+  } else if (this.filter === 'icontains') {
+    return ' ' + this.getFullColumn() + ' ' + this.getOp() + ' unaccent(\'%' + this.getValue() + '%\') ';
   }
   return ' ' + this.getFullColumn() + ' ' + this.getOp() + ' \'' + this.getValue() + '\' ';
 }

protools/devicesformatter.js

@@ -1,26 +1,27 @@
-// Copyright 2017 Telefónica Digital España S.L.
-// 
-// This file is part of UrboCore API.
-// 
-// UrboCore API is free software: you can redistribute it and/or
-// modify it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-// 
-// UrboCore API is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
-// General Public License for more details.
-// 
-// You should have received a copy of the GNU Affero General Public License
-// along with UrboCore API. If not, see http://www.gnu.org/licenses/.
-// 
-// For those usages not covered by this license please contact with
+// Copyright 2017 Telefónica Digital España S.L.
+//
+// This file is part of UrboCore API.
+//
+// UrboCore API is free software: you can redistribute it and/or
+// modify it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// UrboCore API is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
+// General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with UrboCore API. If not, see http://www.gnu.org/licenses/.
+//
+// For those usages not covered by this license please contact with
 // iot_support at tid dot es
 
 'use strict';
 
 var _ = require('underscore');
+var moment = require('moment-timezone');
 var utils = require('../utils');
 var BaseFormatter = require('./baseformatter');
 var log = utils.log();
@@ -47,6 +48,25 @@ class DevicesFormatter extends BaseFormatter {
     return Promise.resolve(rslts);
   }
 
+  parsedData(results, tz) {
+
+
+    var rslts = [];
+    results.forEach(function(rslt) {
+      rslts.push(rslt.rows);
+    });
+    if (rslts.length === 1) {
+      rslts = rslts[0];
+    }
+    else {
+      rslts = _.union(_.flatten(rslts));
+    }
+    rslts.forEach(function(rsl) {
+      rsl.time = moment.tz(rsl.time, tz).format('DD/MM/YYYY HH:mm');
+    });
+    return Promise.resolve(rslts);
+  }
+
 }
 
 module.exports = DevicesFormatter;