Added localities

feeder/builders/buildLocalities.js

@@ -0,0 +1,89 @@
+/* * */
+
+const crypto = require('crypto');
+const FEEDERDB = require('../services/FEEDERDB');
+const SERVERDB = require('../services/SERVERDB');
+const timeCalc = require('../modules/timeCalc');
+const collator = require('../modules/sortCollator');
+
+/* * */
+
+module.exports = async () => {
+  //
+  // 1.
+  // Record the start time to later calculate operation duration
+  const startTime = process.hrtime();
+
+  // 2.
+  // Query Postgres for all unique localities, municipalities
+  console.log(`⤷ Querying database...`);
+  const allLocalities = await FEEDERDB.connection.query(`
+    SELECT DISTINCT ON (locality, municipality_id, municipality_name)
+        locality,
+        municipality_id,
+        municipality_name
+    FROM stops;
+  `);
+
+  // 3.
+  // Log progress
+  console.log(`⤷ Updating Localities...`);
+
+  // 4.
+  // Initate a temporary variable to hold updated Localities
+  const allLocalitiesData = [];
+  const updatedLocalityKeys = new Set();
+
+  // 5.
+  // For each locality, update its entry in the database
+  for (const localityData of allLocalities.rows) {
+    // Skip if the locality is the same as the municipality
+    if (!localityData.locality) continue;
+    else if (localityData.locality === localityData.municipality_name) continue;
+    // Setup the display string for this locality
+    const displayString = `${localityData.locality}, ${localityData.municipality_name}`;
+    // Setup a unique ID for this locality
+    const hash = crypto.createHash('sha256');
+    hash.update(displayString);
+    // Initiate a variable to hold the parsed locality
+    const parsedLocality = {
+      id: hash.digest('hex'),
+      display: displayString,
+      locality: localityData.locality,
+      municipality_id: localityData.municipality_id,
+      municipality_name: localityData.municipality_name,
+    };
+    // Update or create new document
+    allLocalitiesData.push(parsedLocality);
+    await SERVERDB.client.set(`localities:${parsedLocality.id}`, JSON.stringify(parsedLocality));
+    updatedLocalityKeys.add(`localities:${parsedLocality.id}`);
+    //
+  }
+
+  // 6.
+  // Log count of updated Localities
+  console.log(`⤷ Updated ${updatedLocalityKeys.size} Localities.`);
+
+  // 7.
+  // Add the 'all' option
+  allLocalitiesData.sort((a, b) => collator.compare(a.id, b.id));
+  await SERVERDB.client.set('localities:all', JSON.stringify(allLocalitiesData));
+  updatedLocalityKeys.add('localities:all');
+
+  // 8.
+  // Delete all Localities not present in the current update
+  const allSavedStopKeys = [];
+  for await (const key of SERVERDB.client.scanIterator({ TYPE: 'string', MATCH: 'localities:*' })) {
+    allSavedStopKeys.push(key);
+  }
+  const staleLocalityKeys = allSavedStopKeys.filter((id) => !updatedLocalityKeys.has(id));
+  if (staleLocalityKeys.length) await SERVERDB.client.del(staleLocalityKeys);
+  console.log(`⤷ Deleted ${staleLocalityKeys.length} stale Localities.`);
+
+  // 9.
+  // Log elapsed time in the current operation
+  const elapsedTime = timeCalc.getElapsedTime(startTime);
+  console.log(`⤷ Done updating Localities (${elapsedTime}).`);
+
+  //
+};

feeder/start.js

@@ -10,6 +10,7 @@ const fetchAndExtractLatestDataset = require('./modules/fetchAndExtractLatestDat
 const setupPrepareAndImportFile = require('./modules/setupPrepareAndImportFile');
 
 const buildMunicipalities = require('./builders/buildMunicipalities');
+const buildLocalities = require('./builders/buildLocalities');
 const buildPeriods = require('./builders/buildPeriods');
 const buildDates = require('./builders/buildDates');
 const buildTimetables = require('./builders/buildTimetables');
@@ -84,6 +85,10 @@ module.exports = async () => {
     console.log('STEP 2.2: Update Municipalities');
     await buildMunicipalities();
 
+    console.log();
+    console.log('STEP 2.3: Update Localities');
+    await buildLocalities();
+
     console.log();
     console.log('STEP 2.3: Update Periods');
     await buildPeriods();

server/endpoints/localities.endpoint.js

@@ -0,0 +1,21 @@
+/* * */
+
+const SERVERDB = require('../services/SERVERDB');
+
+/* * */
+
+module.exports.all = async (request, reply) => {
+  const allItems = await SERVERDB.client.get('localities:all');
+  return reply
+    .code(200)
+    .header('Content-Type', 'application/json; charset=utf-8')
+    .send(allItems || []);
+};
+
+module.exports.single = async (request, reply) => {
+  const singleItem = await SERVERDB.client.get(`localities:${request.params.id}`);
+  return reply
+    .code(200)
+    .header('Content-Type', 'application/json; charset=utf-8')
+    .send(singleItem || {});
+};

server/index.js

@@ -10,6 +10,7 @@ const SERVERDB = require('./services/SERVERDB');
 const gtfsEndpoint = require('./endpoints/gtfs.endpoint');
 const alertsEndpoint = require('./endpoints/alerts.endpoint');
 const municipalitiesEndpoint = require('./endpoints/municipalities.endpoint');
+const localitiesEndpoint = require('./endpoints/localities.endpoint');
 const periodsEndpoint = require('./endpoints/periods.endpoint');
 const datesEndpoint = require('./endpoints/dates.endpoint');
 const timetablesEndpoint = require('./endpoints/timetables.endpoint');
@@ -41,6 +42,9 @@ fastify.get('/alerts.pb', alertsEndpoint.protobuf);
 fastify.get('/municipalities', municipalitiesEndpoint.all);
 fastify.get('/municipalities/:id', municipalitiesEndpoint.single);
 
+fastify.get('/localities', localitiesEndpoint.all);
+fastify.get('/localities/:id', localitiesEndpoint.single);
+
 fastify.get('/periods', periodsEndpoint.all);
 
 fastify.get('/dates', datesEndpoint.all);