diff --git a/lib/training/wiki_training_loader.rb b/lib/training/wiki_training_loader.rb index 61c2edaa10..65e29dab1e 100644 --- a/lib/training/wiki_training_loader.rb +++ b/lib/training/wiki_training_loader.rb @@ -43,12 +43,12 @@ def load_from_wiki def add_trainings_to_collection(wiki_page) content = new_from_wiki_page(wiki_page) unless content&.valid? - Sentry.capture_message 'Invalid wiki training content', - level: 'warning', extra: { content:, wiki_page: } - return + Sentry.capture_message 'Invalid wiki training content', + level: 'warning', extra: { content: content, wiki_page: wiki_page } + return end @collection << content - end +end def new_from_wiki_page(wiki_page) wikitext = WikiApi.new(MetaWiki.new).get_page_content(wiki_page) @@ -70,9 +70,11 @@ def new_from_json_wiki_page(json_wikitext) content = Oj.load(json_wikitext) base_page = content['wiki_page'] return content unless base_page + wikitext = WikiApi.new(MetaWiki.new).get_page_content(base_page) - training_content_and_translations(content:, base_page:, wikitext:) - end + training_content_and_translations(content: content, base_page: base_page, wikitext: wikitext) +end + # wikitext pages have the slide id and slug embedded in the page title def new_from_wikitext_page(wiki_page, wikitext) @@ -93,15 +95,24 @@ def training_content_and_translations(content:, base_page:, wikitext:) # Gets a list of page titles linked from the base page def wiki_source_pages - # To handle more than 500 pages linked from the source page, - # we'll need to update this to use 'continue'. + query_params = { prop: 'links', titles: @wiki_base_page, pllimit: 500 } - response = WikiApi.new(MetaWiki.new).query(query_params) - begin - response.data['pages'].values[0]['links'].map { |page| page['title'] } - rescue StandardError - raise InvalidWikiContentError, "could not get links from '#{@wiki_base_page}'" + links = [] + + until @continue == 'done' + response = WikiApi.new(MetaWiki.new).query(query_params.merge(plcontinue: @continue)) + + begin + current_links = response.dig('pages', @wiki_base_page, 'links') || [] + links.concat(current_links.map { |page| page['title'] }) + + @continue = response['continue']&.fetch('plcontinue', 'done') + rescue StandardError => e + raise InvalidWikiContentError, "could not get links from '#{@wiki_base_page}': #{e.message}" + end end + + links end def listed_wiki_source_pages