[DATABASE] Geoloc enrichment

DBEnrichment.js

@@ -0,0 +1,229 @@
+/**
+ */
+const http = require('http');
+let querystring = require('querystring');
+const fs = require('fs');
+const os = require('os');
+
+const PATH_ALL_ENTRIES = '/api/entries/findByCountry';
+
+const HOST = os.hostname();
+const PORT = 1337;
+
+/**
+ * Retrieves all the entries of the database, or, if a country was specified as parameter, retrieves all the entries from this country.
+ * @param country
+ */
+function findAllEntries(country) {
+  return new Promise((resolve) => {
+    const options = {
+      host: HOST,
+      port: PORT,
+      path:
+        country === 'all'
+          ? PATH_ALL_ENTRIES
+          : PATH_ALL_ENTRIES + '?country=' + country,
+      method: 'GET',
+    };
+    const req = http.request(options, (res) => {
+      let chunksOfData = [];
+      res.on('data', (data) => {
+        chunksOfData.push(data);
+      });
+      res.on('end', () => {
+        let response = Buffer.concat(chunksOfData);
+        let json = JSON.parse(response);
+        resolve(json);
+      });
+    });
+    req.end();
+    req.on('error', (err) => {
+      resolve({ msg: err });
+    });
+  });
+}
+/**
+ * Updates the entry specified as parameter.
+ * @param entry
+ */
+function update(entry) {
+  return new Promise((resolve) => {
+    const data = querystring.stringify({
+      id: entry['id'],
+      country: entry['country'],
+      county: entry['county'],
+      region: entry['region'],
+      city: entry['city'],
+    });
+    const options = {
+      host: HOST,
+      port: PORT,
+      path: '/api/entries/updateOneAdministrative',
+      method: 'PUT',
+      headers: {
+        'Content-Type': 'application/x-www-form-urlencoded',
+        'Content-Length': data.length,
+      },
+    };
+    const req = http.request(options, (res) => {
+      res.setEncoding('utf8');
+    });
+    req.write(data);
+    req.end();
+    req.on('error', (err) => {
+      resolve({ msg: err });
+    });
+  });
+}
+
+/**
+ * Returns the nearest location information (country, county, region, city) based on the latitude and the longitude provided as parameters
+ * @param latitude
+ * @param longitude
+ */
+function reverseGeo(latitude, longitude) {
+  return new Promise((resolve) => {
+    const options = {
+      host: HOST,
+      port: PORT,
+      path: `/api/reverseGeocode?latitude=${latitude}&longitude=${longitude}`,
+      method: 'GET',
+    };
+    const req = http.request(options, (res) => {
+      let chunksOfData = [];
+      res.on('data', (data) => {
+        chunksOfData.push(data);
+      });
+      res.on('end', () => {
+        let response = Buffer.concat(chunksOfData);
+        let json = JSON.parse(response);
+        resolve(json);
+      });
+    });
+    req.end();
+    req.on('error', (err) => {
+      resolve({ msg: err });
+    });
+  });
+}
+/**
+ * Transforms a 3 letters country code into a 2 letters country code (iso3 -> iso2).
+ * @param codeCountry
+ */
+function countryCodeTransformer(codeCountry) {
+  let rawdata = fs.readFileSync('all.json');
+  let countries = JSON.parse(rawdata);
+  return countries
+    .filter((country) => country['alpha-3'] === codeCountry)
+    .map((country) => country['alpha-2'])[0];
+}
+
+/**
+ * Set fields to null if they are similar to avoid wrong information.
+ * @param entry
+ */
+function verifInfo(entry) {
+  if ((entry['region'] === entry['county']) === entry['city']) {
+    entry['region'] = entry['county'] = entry['city'] = null;
+  }
+  if (entry['county'] === entry['city']) {
+    entry['county'] = entry['city'] = null;
+  }
+  if (entry['region'] === entry['city']) {
+    entry['region'] = entry['city'] = null;
+  }
+  if (entry['countryName'] === entry['region']) {
+    entry['region'] = null;
+  }
+  if (entry['countryName'] === entry['county']) {
+    entry['county'] = null;
+  }
+  if (entry['countryName'] === entry['city']) {
+    entry['city'] = null;
+  }
+  return entry;
+}
+
+/**
+ * Main of the DBEnrichment script.
+ */
+async function enrichment() {
+  findAllEntries(process.argv[2]).then(async function(res) {
+    let allEntries = Array.from(res);
+    let i = 1;
+    for (let entry of allEntries) {
+      let res = await reverseGeo(entry['latitude'], entry['longitude']);
+      res = verifInfo(res);
+      // if the type parameter is completion, only fill the null fields, otherwise overwrite all the fields.
+      if (process.argv[3] === 'completion') {
+        if (res['country'] && country) {
+          res['country'] = countryCodeTransformer(res['country']);
+          if (entry['country'] === '' || entry['country'] === null) {
+            entry['country'] = res['country'];
+          }
+        }
+        if (res['region'] && region) {
+          if (entry['region'] === '' || entry['region'] === null) {
+            entry['region'] = res['region'];
+          }
+        }
+        if (res['county'] && county) {
+          if (entry['county'] === '' || entry['county'] === null) {
+            entry['county'] = res['county'];
+          }
+        }
+        if (res['city'] && city) {
+          if (entry['city'] === '' || entry['city'] === null) {
+            entry['city'] = res['city'];
+          }
+        }
+      } else {
+        if (res['country'] && country) {
+          res['country'] = countryCodeTransformer(res['country']);
+          entry['country'] = res['country'];
+        }
+        if (res['region'] && region) {
+          entry['region'] = res['region'];
+        }
+        if (res['county'] && county) {
+          entry['county'] = res['county'];
+        }
+        if (res['city'] && city) {
+          entry['city'] = res['city'];
+        }
+      }
+
+      update(entry);
+      const progress = i + ' / ' + allEntries.length;
+      // Writes the progression in a tmp txt file to show progress on server side.
+      fs.writeFileSync('tmpDBEnrichmentProgress', progress);
+      ++i;
+    }
+    // Deletes the tmp file after the script's end.
+    fs.unlinkSync('tmpDBEnrichmentProgress');
+  });
+}
+
+process.title = 'DBenrichment';
+
+// Booleans to insure that the fields used are the ones specified as parameters.
+let city = false;
+let county = false;
+let region = false;
+let country = false;
+switch (process.argv[4]) {
+  case 'city':
+    city = county = region = country = true;
+    break;
+  case 'county':
+    county = region = country = true;
+    break;
+  case 'region':
+    region = country = true;
+    break;
+  case 'country':
+    country = true;
+}
+
+// Main launch
+enrichment();

api/controllers/DBEnrichmentController.js

@@ -0,0 +1,64 @@
+/**
+ * DBEnrichmentController
+ *
+ * @description :: Server-side actions for handling incoming requests.
+ * @help        :: See https://sailsjs.com/docs/concepts/actions
+ */
+
+const SCRIPT_NAME = 'DBEnrichment';
+const fs = require('fs');
+
+module.exports = {
+  /**
+   * Checks if the script is already running, and if not, runs it with the parameters specified (country, type and detail)
+   */
+  launchDBEnrichment: (req, res) => {
+    DBEnrichmentService.isScriptRunning(SCRIPT_NAME).then((result) => {
+      if (result) {
+        let progress = 'starting...';
+        if (fs.existsSync('tmpDBEnrichmentProgress')) {
+          progress = fs.readFileSync('tmpDBEnrichmentProgress', 'utf8');
+        }
+        return res.json({
+          msg: 'script already running ...',
+          progress: progress,
+        });
+      } else {
+        const country = req.param('country');
+        const type = req.param('type');
+        const detail = req.param('detail');
+        if (type) {
+          if (type !== 'all') {
+            return res.badRequest(
+              'launchDBEnrichment : the optional parameter "type" can only be "all"',
+            );
+          }
+        }
+        if (detail) {
+          if (
+            detail !== 'country' &&
+            detail !== 'region' &&
+            detail !== 'county' &&
+            detail !== 'city'
+          ) {
+            return res.badRequest(
+              'launchDBEnrichment : the optional parameter "detail" can only be "country", "region", "county" or "city"',
+            );
+          }
+        }
+        DBEnrichmentService.runScript(
+          SCRIPT_NAME + '.js',
+          [
+            country ? country : 'all',
+            type ? type : 'completion',
+            detail ? detail : 'city',
+          ],
+          (err) => {
+            if (err) throw err;
+          },
+        );
+        return res.json({ msg: 'script starting ...' });
+      }
+    });
+  },
+};

api/controllers/EntryController.js

@@ -31,6 +31,27 @@ module.exports = {
       });
   },
 
+  findByCountry: (req, res, converter) => {
+    TEntry.find({
+      select: ['country', 'region', 'county', 'city', 'latitude', 'longitude'],
+      where: {
+        country: req.param('country'),
+      },
+      sort: 'id asc',
+    }).then(
+      (results) => {
+        if (!results) {
+          return res.notFound();
+        }
+        return res.json(results);
+      },
+      (err) => {
+        sails.log.error(err);
+        return res.serverError(`EntryController.findRandom error ${err}`);
+      },
+    );
+  },
+
   findAll: (req, res, converter) => {
     const apiControl = req.options.api;
     const parameters = {};
@@ -138,4 +159,17 @@ module.exports = {
       return ControllerService.treat(req, err, count, params, res);
     });
   },
+
+  updateEntryAdministrative: (req, res) => {
+    const { county, id, country, region, city } = req.body;
+    TEntry.updateOne({ id: id })
+      .set({ county: county, country: country, region: region, city: city })
+      .exec((err, found) => {
+        if (err) {
+          return res.json({ msg: 'Update failed for row : ' + id });
+        } else {
+          return res.json(found);
+        }
+      });
+  },
 };

api/controllers/ReverseGeocodingController.js

@@ -0,0 +1,30 @@
+/**
+ * ReverseGeocodingController
+ *
+ * @description :: Server-side actions for handling incoming requests.
+ * @help        :: See https://sailsjs.com/docs/concepts/actions
+ */
+
+module.exports = {
+  /**
+   * Returns the nearest location information (country, county, region, city) based on the latitude and the longitude provided as parameters
+   */
+  findNearBy: (req, res) => {
+    const lat = req.param('latitude');
+    const lng = req.param('longitude');
+    if (!lat) {
+      return res.badRequest('ReverseGeocodingController : latitude is missing');
+    }
+    if (!lng) {
+      return res.badRequest(
+        'ReverseGeocodingController : longitude is missing',
+      );
+    }
+    // Delegates to the reverse geocoding service
+    return ReverseGeocodingService.findNearBy(lat, lng).then(
+      (result) => res.json(result),
+      (err) =>
+        res.serverError(`ReverseGeocodingController.findNearBy error : ${err}`),
+    );
+  },
+};

api/models/TEntry.js

@@ -55,6 +55,13 @@ module.exports = {
       columnName: 'Country',
     },
 
+    county: {
+      type: 'string',
+      maxLength: 32,
+      columnName: 'County',
+      allowNull: true,
+    },
+
     region: {
       type: 'string',
       maxLength: 32,