From 76ec924b764d06780384a67dd119ea2b06e53d60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20de=20Vasconcelos?= Date: Mon, 11 Dec 2023 17:15:45 +0000 Subject: [PATCH] Added localities --- feeder/builders/buildLocalities.js | 89 +++++++++++++++++++++++++ feeder/start.js | 5 ++ server/endpoints/localities.endpoint.js | 21 ++++++ server/index.js | 4 ++ 4 files changed, 119 insertions(+) create mode 100644 feeder/builders/buildLocalities.js create mode 100644 server/endpoints/localities.endpoint.js diff --git a/feeder/builders/buildLocalities.js b/feeder/builders/buildLocalities.js new file mode 100644 index 00000000..cbf0b117 --- /dev/null +++ b/feeder/builders/buildLocalities.js @@ -0,0 +1,89 @@ +/* * */ + +const crypto = require('crypto'); +const FEEDERDB = require('../services/FEEDERDB'); +const SERVERDB = require('../services/SERVERDB'); +const timeCalc = require('../modules/timeCalc'); +const collator = require('../modules/sortCollator'); + +/* * */ + +module.exports = async () => { + // + // 1. + // Record the start time to later calculate operation duration + const startTime = process.hrtime(); + + // 2. + // Query Postgres for all unique localities, municipalities + console.log(`⤷ Querying database...`); + const allLocalities = await FEEDERDB.connection.query(` + SELECT DISTINCT ON (locality, municipality_id, municipality_name) + locality, + municipality_id, + municipality_name + FROM stops; + `); + + // 3. + // Log progress + console.log(`⤷ Updating Localities...`); + + // 4. + // Initate a temporary variable to hold updated Localities + const allLocalitiesData = []; + const updatedLocalityKeys = new Set(); + + // 5. + // For each locality, update its entry in the database + for (const localityData of allLocalities.rows) { + // Skip if the locality is the same as the municipality + if (!localityData.locality) continue; + else if (localityData.locality === localityData.municipality_name) continue; + // Setup the display string for this locality + const displayString = `${localityData.locality}, ${localityData.municipality_name}`; + // Setup a unique ID for this locality + const hash = crypto.createHash('sha256'); + hash.update(displayString); + // Initiate a variable to hold the parsed locality + const parsedLocality = { + id: hash.digest('hex'), + display: displayString, + locality: localityData.locality, + municipality_id: localityData.municipality_id, + municipality_name: localityData.municipality_name, + }; + // Update or create new document + allLocalitiesData.push(parsedLocality); + await SERVERDB.client.set(`localities:${parsedLocality.id}`, JSON.stringify(parsedLocality)); + updatedLocalityKeys.add(`localities:${parsedLocality.id}`); + // + } + + // 6. + // Log count of updated Localities + console.log(`⤷ Updated ${updatedLocalityKeys.size} Localities.`); + + // 7. + // Add the 'all' option + allLocalitiesData.sort((a, b) => collator.compare(a.id, b.id)); + await SERVERDB.client.set('localities:all', JSON.stringify(allLocalitiesData)); + updatedLocalityKeys.add('localities:all'); + + // 8. + // Delete all Localities not present in the current update + const allSavedStopKeys = []; + for await (const key of SERVERDB.client.scanIterator({ TYPE: 'string', MATCH: 'localities:*' })) { + allSavedStopKeys.push(key); + } + const staleLocalityKeys = allSavedStopKeys.filter((id) => !updatedLocalityKeys.has(id)); + if (staleLocalityKeys.length) await SERVERDB.client.del(staleLocalityKeys); + console.log(`⤷ Deleted ${staleLocalityKeys.length} stale Localities.`); + + // 9. + // Log elapsed time in the current operation + const elapsedTime = timeCalc.getElapsedTime(startTime); + console.log(`⤷ Done updating Localities (${elapsedTime}).`); + + // +}; diff --git a/feeder/start.js b/feeder/start.js index 9845c7d6..ae7e32ec 100644 --- a/feeder/start.js +++ b/feeder/start.js @@ -10,6 +10,7 @@ const fetchAndExtractLatestDataset = require('./modules/fetchAndExtractLatestDat const setupPrepareAndImportFile = require('./modules/setupPrepareAndImportFile'); const buildMunicipalities = require('./builders/buildMunicipalities'); +const buildLocalities = require('./builders/buildLocalities'); const buildPeriods = require('./builders/buildPeriods'); const buildDates = require('./builders/buildDates'); const buildTimetables = require('./builders/buildTimetables'); @@ -84,6 +85,10 @@ module.exports = async () => { console.log('STEP 2.2: Update Municipalities'); await buildMunicipalities(); + console.log(); + console.log('STEP 2.3: Update Localities'); + await buildLocalities(); + console.log(); console.log('STEP 2.3: Update Periods'); await buildPeriods(); diff --git a/server/endpoints/localities.endpoint.js b/server/endpoints/localities.endpoint.js new file mode 100644 index 00000000..ef7c48f2 --- /dev/null +++ b/server/endpoints/localities.endpoint.js @@ -0,0 +1,21 @@ +/* * */ + +const SERVERDB = require('../services/SERVERDB'); + +/* * */ + +module.exports.all = async (request, reply) => { + const allItems = await SERVERDB.client.get('localities:all'); + return reply + .code(200) + .header('Content-Type', 'application/json; charset=utf-8') + .send(allItems || []); +}; + +module.exports.single = async (request, reply) => { + const singleItem = await SERVERDB.client.get(`localities:${request.params.id}`); + return reply + .code(200) + .header('Content-Type', 'application/json; charset=utf-8') + .send(singleItem || {}); +}; diff --git a/server/index.js b/server/index.js index ebd12e3b..47820e7f 100644 --- a/server/index.js +++ b/server/index.js @@ -10,6 +10,7 @@ const SERVERDB = require('./services/SERVERDB'); const gtfsEndpoint = require('./endpoints/gtfs.endpoint'); const alertsEndpoint = require('./endpoints/alerts.endpoint'); const municipalitiesEndpoint = require('./endpoints/municipalities.endpoint'); +const localitiesEndpoint = require('./endpoints/localities.endpoint'); const periodsEndpoint = require('./endpoints/periods.endpoint'); const datesEndpoint = require('./endpoints/dates.endpoint'); const timetablesEndpoint = require('./endpoints/timetables.endpoint'); @@ -41,6 +42,9 @@ fastify.get('/alerts.pb', alertsEndpoint.protobuf); fastify.get('/municipalities', municipalitiesEndpoint.all); fastify.get('/municipalities/:id', municipalitiesEndpoint.single); +fastify.get('/localities', localitiesEndpoint.all); +fastify.get('/localities/:id', localitiesEndpoint.single); + fastify.get('/periods', periodsEndpoint.all); fastify.get('/dates', datesEndpoint.all);