Skip to content

Commit

Permalink
#41 lazy loading
Browse files Browse the repository at this point in the history 
operatorInfo, wikipediaSummary, ArtistNames
  • Loading branch information
Ralf Hauser committed Feb 23, 2020
1 parent 7953dd1 commit 73fdf8d
Show file tree
Hide file tree
Showing 6 changed files with 156 additions and 127 deletions.
168 changes: 91 additions & 77 deletions server/api/controllers/controller.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,15 @@ const NodeCache = require( "node-cache" );
import {conflate} from "../services/conflate.data.service";
import applyImpliedPropertiesOsm from "../services/applyImplied.service";
import {
essenceOf, defaultCollectionEnhancement, fillInMissingWikidataFountains
essenceOf, defaultCollectionEnhancement, fillInMissingWikidataFountains,
fillWikipediaSummary
} from "../services/processing.service";
import {updateCacheWithFountain} from "../services/database.service";
import {extractProcessingErrors} from "./processing-errors.controller";
import {getImageInfo,getImgsOfCat} from "../services/wikimedia.service";
const haversine = require("haversine");
const _ = require('lodash');
import {MAX_IMG_SHOWN_IN_GALLERY} from "../../common/constants";
import {MAX_IMG_SHOWN_IN_GALLERY, LAZY_ARTIST_NAME_LOADING_i41db} from "../../common/constants";


// Configuration of Cache after https://www.npmjs.com/package/node-cache
Expand Down Expand Up @@ -256,87 +257,100 @@ function byId(req, res, dbg){
// l.info('controller.js byId fountain: '+cityS+' '+dbg+' '+new Date().toISOString());
if (null != props) {
name = props.name.value;
if (LAZY_ARTIST_NAME_LOADING_i41db) {
imgMetaPromises.push(WikidataService.fillArtistName(fountain,dbg));
}
imgMetaPromises.push(WikidataService.fillOperatorInfo(fountain,dbg));
fillWikipediaSummary(fountain, dbg, 1, imgMetaPromises);
const gal = props.gallery
// l.info('controller.js byId props: '+cityS+' '+dbg+' '+new Date().toISOString());
// l.info('controller.js byId props: '+cityS+' '+dbg+' '+new Date().toISOString());
if (null != gal && null != gal.value) {
gl = gal.value.length;
// l.info('controller.js byId gl: '+cityS+' '+dbg+' '+new Date().toISOString());
if (0 < gl) {
// l.info('controller.js byId: of '+cityS+' found gal of size '+gl+' "'+name+'" '+dbg+' '+new Date().toISOString());
let i = 0;
let lzAtt = '';
const showDetails = true;
let imgUrlSet = new Set();
let catPromises = [];
let numbOfCats = -1;
let numbOfCatsLazyAdded = 0;
// l.info('controller.js byId gl: '+cityS+' '+dbg+' '+new Date().toISOString());
if (0 < gl) {
// l.info('controller.js byId: of '+cityS+' found gal of size '+gl+' "'+name+'" '+dbg+' '+new Date().toISOString());
let i = 0;
let lzAtt = '';
const showDetails = true;
let imgUrlSet = new Set();
let catPromises = [];
let numbOfCats = -1;
let numbOfCatsLazyAdded = 0;
let imgUrlsLazyByCat = [];
if (props.wiki_commons_name && props.wiki_commons_name.value && 0 < props.wiki_commons_name.value.length) {
numbOfCats = props.wiki_commons_name.value.length;
for(const cat of props.wiki_commons_name.value) {
const add = 0 > cat.l;
if (add) {
numbOfCatsLazyAdded++;
if (0 == imgUrlSet.size) {
for(const img of gal.value) {
imgUrlSet.add(img.pgTit);
}
}
const catPromise = getImgsOfCat(cat, dbg, cityS, imgUrlSet, imgUrlsLazyByCat, "dbgIdWd", props,true);
//TODO we might prioritize categories with small number of images to have greater variety of images?
catPromises.push(catPromise);
}
}
}
Promise.all(catPromises).then(r => {
for(let k = 0; k < imgUrlsLazyByCat.length && k < MAX_IMG_SHOWN_IN_GALLERY;k++) { //between 6 && 50 imgs are on the gallery-preview
const img = imgUrlsLazyByCat[k];
let nImg = {s: img.src,pgTit: img.val,c: img.cat};
gal.value.push(nImg);
}
if (0 < imgUrlsLazyByCat.length) {
l.info('controller.js byId lazy img by lazy cat added: attempted '+imgUrlsLazyByCat.length+' in '+numbOfCatsLazyAdded+'/'+
numbOfCats+' cats, tot '+gl+' of '+cityS+' '+dbg+' "'+name+'" '+r.length+' '+new Date().toISOString());
}
for(const img of gal.value) {
let imMetaDat = img.metadata;
if (null == imMetaDat) {
lzAtt += i+',';
l.info('controller.js byId lazy getImageInfo: '+cityS+' '+i+'/'+gl+' "'+img.pgTit+'" "'+name+'" '+dbg+' '+new Date().toISOString());
imgMetaPromises.push(getImageInfo(img, i+'/'+gl+' '+dbg+' '+name+' '+cityS,showDetails, new Map()).catch(giiErr=>{
l.info('wikimedia.service.js: fillGallery getImageInfo failed for "'+img.pgTit+'" '+dbg+' '+city+' '+dbgIdWd+' "'+name+'" '+new Date().toISOString()
+ '\n'+giiErr.stack);
}));
lazyAdded++;
} else {
// l.info('controller.js byId: of '+cityS+' found imMetaDat '+i+' in gal of size '+gl+' "'+name+'" '+dbg+' '+new Date().toISOString());
}
i++;
}
if (0 < lazyAdded) {
l.info('controller.js byId lazy img metadata loading: attempted '+lazyAdded+'/'+gl+' ('+lzAtt+') of '+cityS+' '+dbg+' "'+name+'" '+new Date().toISOString());
}
Promise.all(imgMetaPromises).then(r => {
if (0 < lazyAdded) {
l.info('controller.js byId lazy img metadata loading after promise: attempted '+lazyAdded+' tot '+gl+' of '+cityS+' '+dbg+' "'+name+'" '+r.length+' '+new Date().toISOString());
}
doJson(res,fountain, 'byId '+dbg); // res.json(fountain);
l.info('controller.js byId: of '+cityS+' res.json '+dbg+' "'+name+'" '+new Date().toISOString());
resolve(fountain);
}, err => {
l.error(`controller.js: Failed on imgMetaPromises: ${err.stack} .`+dbg+' "'+name+'" '+cityS);
});
}, err => {
l.error(`controller.js: Failed on imgMetaPromises: ${err.stack} .`+dbg+' "'+name+'" '+cityS);
});
if (props.wiki_commons_name && props.wiki_commons_name.value && 0 < props.wiki_commons_name.value.length) {
numbOfCats = props.wiki_commons_name.value.length;
for(const cat of props.wiki_commons_name.value) {
const add = 0 > cat.l;
if (add) {
numbOfCatsLazyAdded++;
if (0 == imgUrlSet.size) {
for(const img of gal.value) {
imgUrlSet.add(img.pgTit);
}
}
const catPromise = getImgsOfCat(cat, dbg, cityS, imgUrlSet, imgUrlsLazyByCat, "dbgIdWd", props,true);
//TODO we might prioritize categories with small number of images to have greater variety of images?
catPromises.push(catPromise);
}
}
}
Promise.all(catPromises).then(r => {
for(let k = 0; k < imgUrlsLazyByCat.length && k < MAX_IMG_SHOWN_IN_GALLERY;k++) { //between 6 && 50 imgs are on the gallery-preview
const img = imgUrlsLazyByCat[k];
let nImg = {s: img.src,pgTit: img.val,c: img.cat};
gal.value.push(nImg);
}
if (0 < imgUrlsLazyByCat.length) {
l.info('controller.js byId lazy img by lazy cat added: attempted '+imgUrlsLazyByCat.length+' in '+numbOfCatsLazyAdded+'/'+
numbOfCats+' cats, tot '+gl+' of '+cityS+' '+dbg+' "'+name+'" '+r.length+' '+new Date().toISOString());
}
for(const img of gal.value) {
let imMetaDat = img.metadata;
if (null == imMetaDat) {
lzAtt += i+',';
l.info('controller.js byId lazy getImageInfo: '+cityS+' '+i+'/'+gl+' "'+img.pgTit+'" "'+name+'" '+dbg+' '+new Date().toISOString());
imgMetaPromises.push(getImageInfo(img, i+'/'+gl+' '+dbg+' '+name+' '+cityS,showDetails, new Map()).catch(giiErr=>{
l.info('wikimedia.service.js: fillGallery getImageInfo failed for "'+img.pgTit+'" '+dbg+' '+city+' '+dbgIdWd+' "'+name+'" '+new Date().toISOString()
+ '\n'+giiErr.stack);
}));
lazyAdded++;
} else {
// l.info('controller.js byId: of '+cityS+' found imMetaDat '+i+' in gal of size '+gl+' "'+name+'" '+dbg+' '+new Date().toISOString());
}
i++;
}
if (0 < lazyAdded) {
l.info('controller.js byId lazy img metadata loading: attempted '+lazyAdded+'/'+gl+' ('+lzAtt+') of '+cityS+' '+dbg+' "'+name+'" '+new Date().toISOString());
}
Promise.all(imgMetaPromises).then(r => {
if (0 < lazyAdded) {
l.info('controller.js byId lazy img metadata loading after promise: attempted '+lazyAdded+' tot '+gl+' of '+cityS+' '+dbg+' "'+name+'" '+r.length+' '+new Date().toISOString());
}
doJson(res,fountain, 'byId '+dbg); // res.json(fountain);
l.info('controller.js byId: of '+cityS+' res.json '+dbg+' "'+name+'" '+new Date().toISOString());
resolve(fountain);
}, err => {
l.error(`controller.js: Failed on imgMetaPromises: ${err.stack} .`+dbg+' "'+name+'" '+cityS);
});
}, err => {
l.error(`controller.js: Failed on imgMetaPromises: ${err.stack} .`+dbg+' "'+name+'" '+cityS);
});
} else {
l.info('controller.js byId: of '+cityS+' gl < 1 '+dbg+' '+new Date().toISOString());
doJson(res,fountain, 'byId '+dbg);
resolve(fountain);
}
l.info('controller.js byId: of '+cityS+' gl < 1 '+dbg+' '+new Date().toISOString());
Promise.all(imgMetaPromises).then(r => {
if (0 < lazyAdded) {
l.info('controller.js byId lazy img metadata loading after promise: attempted '+lazyAdded+' tot '+gl+' of '+cityS+' '+dbg+' "'+name+'" '+r.length+' '+new Date().toISOString());
}
doJson(res,fountain, 'byId '+dbg); // res.json(fountain);
l.info('controller.js byId: of '+cityS+' res.json '+dbg+' "'+name+'" '+new Date().toISOString());
resolve(fountain);
}, err => {
l.error(`controller.js: Failed on imgMetaPromises: ${err.stack} .`+dbg+' "'+name+'" '+cityS);
});
}
} else {
l.info('controller.js byId: of '+cityS+' gallery null || null == gal.val '+dbg+' '+new Date().toISOString());
}
l.info('controller.js byId: of '+cityS+' gallery null || null == gal.val '+dbg+' '+new Date().toISOString());
}
} else {
l.info('controller.js byId: of '+cityS+' no props '+dbg+' '+new Date().toISOString());
}
Expand Down
2 changes: 1 addition & 1 deletion server/api/services/generateLocationData.service.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ function generateLocationData(locationName){
// conflate
Promise.all([osmPromise, wikidataPromise])
// get any missing wikidata fountains for #212
.then(r=>fillInMissingWikidataFountains(r[0], r[1]))
.then(r=>fillInMissingWikidataFountains(r[0], r[1],locationName))
.then(r => conflate({
osm: r.osm,
wikidata: r.wikidata
Expand Down
88 changes: 51 additions & 37 deletions server/api/services/processing.service.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,16 +13,16 @@ import WikipediaService from './wikipedia.service';
import WikidataService from './wikidata.service';
import l from '../../common/logger';
import {fountain_property_metadata} from "../../../config/fountain.properties"
import {PROP_STATUS_INFO, PROP_STATUS_OK} from "../../common/constants";
import {PROP_STATUS_INFO, PROP_STATUS_OK,LAZY_ARTIST_NAME_LOADING_i41db, LANGS} from "../../common/constants";

export function defaultCollectionEnhancement(fountainCollection,dbg, debugAll) {
l.info('processing.service.js defaultCollectionEnhancement: '+dbg+' '+new Date().toISOString());
return new Promise((resolve, reject)=>{
fillImageGalleries(fountainCollection,dbg, debugAll)
.then(r => fillOutNames(r,dbg))
.then(r => fillWikipediaSummaries(r,dbg))
.then(r => fillArtistNames(r,dbg))
.then(r => fillOperatorInfo(r,dbg))
// .then(r => fillWikipediaSummaries(r,dbg))
// .then(r => fillArtistNames(r,dbg)) as per LAZY_ARTIST_NAME_LOADING_i41db
// .then(r => fillOperatorInfo(r,dbg))
.then(r => resolve(r))
.catch(err=>reject(err))
})
Expand Down Expand Up @@ -76,6 +76,9 @@ export function fillArtistNames(fountainCollection,dbg){
// takes a collection of fountains and returns the same collection,
// enhanced with artist names if only QID was given
l.info('processing.service.js starting fillArtistNames: '+dbg+' '+new Date().toISOString());
if (LAZY_ARTIST_NAME_LOADING_i41db) {
l.info('processing.service.js fillArtistNames LAZY_ARTIST_NAME_LOADING_i41db: '+LAZY_ARTIST_NAME_LOADING_i41db+' - '+dbg+' '+new Date().toISOString());
}
return new Promise((resolve, reject) => {
let promises = [];
let i = 0;
Expand Down Expand Up @@ -111,42 +114,49 @@ export function fillOperatorInfo(fountainCollection, dbg){
})
}

export function fillWikipediaSummary(fountain, dbg, tot, promises) {
// check all languages to see if a wikipedia page is referenced
let i = 0;
_.forEach(LANGS, lang =>{
let urlParam = `wikipedia_${lang}_url`;
i=i+1;
const dbgHere = i+'/'+tot+' '+dbg;
const props = fountain.properties;
const pU = props[urlParam];
if(null != pU && null != pU.value){
// if not Null, get summary and create new property
let dbgIdWd = null;
if (null != props.id_wikidata && null != props.id_wikidata.value) {
dbgIdWd = props.id_wikidata.value;
}
promises.push(new Promise((resolve, reject) => {
WikipediaService.getSummary(pU.value, dbgHere+' '+lang+' '+dbgIdWd)
.then(summary => {
// add summary as derived information to url property
pU.derived = {
summary: summary
};
resolve();
})
.catch(error=>{
l.error(`Error creating Wikipedia summary: ${error}`);
reject(error)
})
}));
}
});

}

export function fillWikipediaSummaries(fountainCollection, dbg){
// takes a collection of fountains and returns the same collection, enhanced with wikipedia summaries
l.info('processing.service.js starting fillWikipediaSummaries: '+dbg+' '+new Date().toISOString());
return new Promise((resolve, reject) => {
let promises = [];
// loop through fountains
let tot = fountainCollection.length;
_.forEach(fountainCollection, fountain =>{
// check all languages to see if a wikipedia page is referenced
let i = 0;
let tot = fountainCollection.length;
_.forEach(['en', 'de', 'fr', 'it', 'tr'], lang =>{
let urlParam = `wikipedia_${lang}_url`;
i=i+1;
let dbgHere = i+'/'+tot+' '+dbg;
if(!_.isNull(fountain.properties[urlParam].value)){
// if not Null, get summary and create new property
let dbgIdWd = null;
if (null != fountain.properties.id_wikidata && null != fountain.properties.id_wikidata.value) {
dbgIdWd = fountain.properties.id_wikidata.value;
}
promises.push(new Promise((resolve, reject) => {
WikipediaService.getSummary(fountain.properties[urlParam].value, dbgHere+' '+lang+' '+dbgIdWd)
.then(summary => {
// add summary as derived information to url property
fountain.properties[urlParam].derived = {
summary: summary
};
resolve();
})
.catch(error=>{
l.error(`Error creating Wikipedia summary: ${error}`);
reject(error)
})
}));
}
});
fillWikipediaSummary(fountain, dbg, tot, promises);
});

Promise.all(promises)
Expand Down Expand Up @@ -232,7 +242,6 @@ export function fillOutNames(fountainCollection,dbg) {
// takes a collection of fountains and returns the same collection, with blanks in fountain names filled from other languages or from 'name' property
l.info('processing.service.js starting fillOutNames: '+dbg+' '+new Date().toISOString());
return new Promise((resolve, reject) => {
let langs = ['en','de','fr', 'it', 'tr','sr'];
let i = 0;
for(let f of fountainCollection) {
// if the default name (aka title) if not filled, then fill it from one of the other languages
Expand All @@ -246,7 +255,7 @@ export function fillOutNames(fountainCollection,dbg) {
}
i++;
if(fProps.name.value === null){
for(let lang of langs){
for(let lang of LANGS){
let fPopLng = fProps[`name_${lang}`];
if(fPopLng != null && fPopLng.value !== null){
// take the first language-specific name that is not null and apply it to the default name
Expand All @@ -261,7 +270,7 @@ export function fillOutNames(fountainCollection,dbg) {
}
// fill lang-specific names if null and if a default name exists
if(fProps.name.value !== null) {
for (let lang of langs) {
for (let lang of LANGS) {
let fPopLng = fProps[`name_${lang}`];
if (fPopLng != null && fPopLng.value === null) {
fPopLng.value = fProps.name.value;
Expand Down Expand Up @@ -290,9 +299,14 @@ export function fillInMissingWikidataFountains(osm_fountains, wikidata_fountains
return new Promise((resolve, reject)=>{
// Get list of all Wikidata fountain qids referenced by OSM
let qid_from_osm = _.compact(_.map(osm_fountains, f=>_.get(f,['properties', 'wikidata'])));

if(process.env.NODE_ENV !== 'production') {
l.info('processing.service.js fillInMissingWikidataFountains osm_fountains '+osm_fountains.length+', qid_from_osm '+qid_from_osm.length+' ' +dbg+' '+new Date().toISOString());
}
// Get list of all Wikidata fountain qids collected
let qid_from_wikidata = _.map(wikidata_fountains, 'id');
if(process.env.NODE_ENV !== 'production') {
l.info('processing.service.js fillInMissingWikidataFountains qid_from_wikidata '+qid_from_wikidata.length+' ' +dbg+' '+new Date().toISOString());
}

// Get qids not included in wikidata collection
let missing_qids = _.difference(qid_from_osm, qid_from_wikidata);
Expand Down
5 changes: 2 additions & 3 deletions server/api/services/wikidata.service.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,11 @@
*/

import l from '../../common/logger';
import {FUNCTION_NOT_AVAILABLE, NO_FOUNTAIN_AT_LOCATION} from "./constants";
const https = require('https');
const axios = require('axios');
import { cacheAdapterEnhancer } from 'axios-extensions';
const _ = require ('lodash');
const wdk = require('wikidata-sdk');
import {LANGS} from "../../common/constants";

// Set up caching of http requests
const http = axios.create({
Expand Down Expand Up @@ -191,7 +190,7 @@ class WikidataService {
}
// Try to find a useful link
// Look for Wikipedia entry in different languages
for(let lang of ['en', 'fr', 'de', 'it', 'tr']){
for(let lang of LANGS){
if(entity.sitelinks.hasOwnProperty(lang+'wiki')){
fountain.properties.artist_name.derived.website.url = `https://${lang}.wikipedia.org/wiki/${entity.sitelinks[lang+'wiki']}`;
return fountain;
Expand Down
Loading

0 comments on commit 73fdf8d

Please sign in to comment.