Merge pull request #17 from medienhaus/intervalConfig

adding configurable fetch interval

config.js.example

@@ -1,8 +1,55 @@
+/* eslint-disable import/no-anonymous-default-export */
 export default () => ({
   matrix: {
-    homeserver_base_url: 'https://dev.medienhaus.udk-berlin.de',
-    user_id: '@rundgang-bot:dev.medienhaus.udk-berlin.de',
-    access_token: 'syt_XXXuZGdhbmctYm90_BRnkDuPAihrEnRkZJsuD_0YfBnN',
-    root_context_space_id: '!CgKzJcINbtDdglFtub:dev.medienhaus.udk-berlin.de'
+    homeserver_base_url: 'https://matrix.org',
+    user_id: '@username:matrix.org',
+    access_token: 'xxxxxxxx',
+    root_context_space_id: '!xxxxxxxxx:matrix.org'
+  },
+  fetch: {
+    depth: 500,
+    max: 10000,
+    interval: 60,
+    autoFetch: true,
+    dump: true,
+    initalyLoad: true,
+    noLog: true
+  },
+  interfaces: {
+    rest_v1: true,
+    rest_v2: true,
+    graphql: true,
+    graphql_playground: true,
+    post: true
+  },
+  application: {
+    name: 'some name',
+    api_name: 'someName-api',
+    standards: [
+      {
+        name: 'dev.medienhaus.meta',
+        version: '1.1'
+      },
+      {
+        name: 'dev.medienhaus.allocation',
+        version: '0.1'
+      },
+      {
+        name: 'dev.medienhaus.order',
+        version: '0.1'
+      }
+    ]
+  },
+  attributable: {
+    spaceTypes: {
+      item: ['item', 'studentproject', 'project', 'event'],
+      context: [
+        'context',
+        'class',
+        'faculty',
+        'centre'
+      ],
+      content: ['lang', 'headline', 'text', 'ul', 'ol', 'quote']
+    }
   }
 })

src/app.module.js

@@ -6,7 +6,7 @@ import { AppService } from './app.service'
 import { ConfigModule, ConfigService } from '@nestjs/config'
 import configuration from '../config'
 import { ItemService } from './item.service'
-import { ScheduleModule } from '@nestjs/schedule'
+import { ScheduleModule, SchedulerRegistry } from '@nestjs/schedule'
 import { HttpModule, HttpService } from '@nestjs/axios'
 import { GraphQLModule } from '@nestjs/graphql'
 import { ApolloDriver, ApolloDriverConfig } from '@nestjs/apollo'
@@ -36,10 +36,19 @@ import { join } from 'path'
     AppService,
     {
       provide: 'ITEM_PROVIDER',
-      inject: [ConfigService, HttpService],
-      useFactory: async (configService, httpService) => {
+      inject: [ConfigService, HttpService, SchedulerRegistry],
+      useFactory: async (configService, httpService, schedulerRegistry) => {
         const x = new ItemService(configService, httpService)
         if (x.configService.get('fetch.initalyLoad')) await x.fetch()
+
+        if (x.configService.get('fetch.interval')) {
+          const fetchCallback = async () => {
+            await x.fetch()
+          }
+          const fetchInterval = setInterval(fetchCallback, x.configService.get('fetch.interval') * 1000) // seconds to ms
+          schedulerRegistry.addInterval('fetchInterval', fetchInterval)
+        }
+
         return x
       }
     },

src/item.service.js

@@ -2,14 +2,16 @@ import { Dependencies, Injectable, Logger } from '@nestjs/common'
 import { createClient as createMatrixClient } from 'matrix-js-sdk'
 import { ConfigService } from '@nestjs/config'
 import * as _ from 'lodash'
-import { Interval } from '@nestjs/schedule'
 import { HttpService } from '@nestjs/axios'
 import Handlebars from 'handlebars'
 import fs from 'fs'
 import { join } from 'path'
 import moment from 'moment'
 import { isNull, template } from 'lodash'
 import { LegacyInterpreter } from './legacy-interpreter.service'
+import { Console } from 'console'
+
+export const test = 10000
 
 @Injectable()
 @Dependencies(ConfigService, HttpService)
@@ -34,6 +36,9 @@ export class ItemService {
     this.initiallyFetched = false
     this.batchCounter = 0
 
+    this.lastFetch = Date.now()
+    this.fistFetch = Date.now()
+
     this.matrixClient = createMatrixClient({
       baseUrl: this.configService.get('matrix.homeserver_base_url'),
       accessToken: this.configService.get('matrix.access_token'),
@@ -47,22 +52,21 @@ export class ItemService {
     this.legacyInterpreter = new LegacyInterpreter(this.configService, this.httpService, this.matrixClient)
   }
 
-  @Interval(120 * 60 * 1000) // Call this every 120 minutes
   async fetch () {
     if (!this.configService.get('fetch.autoFetch') && this.initiallyFetched) {
       return
     }
 
-    Logger.log('Fetching items...')
+    const fetchStart = Date.now()
 
     const allSpaces = await this.getAllSpaces(
       this.configService.get('matrix.root_context_space_id'),
       {
         max: this.configService.get('fetch.max'),
-        depth: this.configService.get('fetch.depth')
+        depth: this.configService.get('fetch.depth'),
+        noLog: this.configService.get('fetch.noLog')
       }
     )
-    Logger.log(`Found ${Object.keys(allSpaces).length} spaces`)
 
     const generatedStrucute = this.generateStructure(
       _.filter(allSpaces, (space) => {
@@ -81,7 +85,7 @@ export class ItemService {
     this.graphQlCache = {}
     structure[generatedStrucute.room_id] = generatedStrucute
     this._allRawSpaces = allSpaces
-    this.allSpaces = await this.generateAllSpaces(allSpaces)
+    this.allSpaces = await this.generateAllSpaces(allSpaces, { noLog: this.configService.get('fetch.noLog') })
     this.structure = structure
     this.contents = []
     this.batchCounter = 0
@@ -97,8 +101,6 @@ export class ItemService {
       this.items[Object.keys(ele)[0]] = ele[Object.keys(ele)[0]]
     })
 
-    Logger.log(`Found ${Object.keys(this.items).length} items`)
-
     // new for graphQL functionality
     _.forEach(this.allSpaces, (space) => {
       // fill users
@@ -123,6 +125,8 @@ export class ItemService {
       }
     })
 
+    Logger.log('Fetched ' + Object.keys(allSpaces).length + ' spaces with ' + Object.keys(this.items).length + ' items, after: ' + Math.round((Date.now() - this.fistFetch) / 10 / 60) / 100 + ' minutes,  which took: ' + (Math.round((((Date.now() - fetchStart) / 1000) * 100) / 100)) + ' seconds')
+    this.lastFetch = Date.now()
     if (!this.initiallyFetched) this.initiallyFetched = true
   }
 
@@ -147,7 +151,7 @@ export class ItemService {
     if (!hierarchyBatch?.next_batch) {
       return hirachy
     } else {
-      await new Promise((r) => setTimeout(r, 100))
+      // await new Promise((r) => setTimeout(r, 100))
 
       const getMoreRooms = await this.getBatch(
         spaceId,
@@ -179,7 +183,7 @@ export class ItemService {
       if (stateEvents?.some((state) => state.type === 'dev.medienhaus.meta')) {
         ret[space?.room_id].stateEvents = stateEvents
       }
-      await new Promise((r) => setTimeout(r, 1))
+      // await new Promise((r) => setTimeout(r, 1))
       if (!options?.noLog) {
         Logger.log('get stateEvents:\t' + i + '/' + hierarchy?.rooms.length)
       }
@@ -349,7 +353,7 @@ export class ItemService {
           Logger.log("can't get room members:/t" + spaceId)
         })
       rawSpaces[spaceId].joinedMembers = joinedMembers
-      await new Promise((r) => setTimeout(r, 1))
+      // await new Promise((r) => setTimeout(r, 1))
     } else {
       joinedMembers = rawSpaces[spaceId].joinedMembers
     }
@@ -994,7 +998,7 @@ export class ItemService {
               case 'ul':
               case 'ol':
                 return lastMessage.content.formatted_body
-              // For all other types we render the HTML using the corresponding Handlebars template in /views/contentBlocks
+                // For all other types we render the HTML using the corresponding Handlebars template in /views/contentBlocks
               default:
                 if (!this.configService.get('attributable.spaceTypes.content').some((f) => f === template)) {
                   return ''