From e5f6bf99d959d6893eddc848bc2766960bb32878 Mon Sep 17 00:00:00 2001 From: Ewan Cahen Date: Fri, 4 Nov 2022 14:06:34 +0100 Subject: [PATCH 01/32] fix: commit scraper now retries on 202 response from GitHub --- ...ception.java => RsdResponseException.java} | 4 +- .../nl/esciencecenter/rsd/scraper/Utils.java | 37 +++++++++++++++++- .../rsd/scraper/git/GithubSI.java | 38 +++++++++++++------ .../rsd/scraper/git/MainCommits.java | 2 + 4 files changed, 66 insertions(+), 15 deletions(-) rename scrapers/src/main/java/nl/esciencecenter/rsd/scraper/{ResponseException.java => RsdResponseException.java} (82%) diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/ResponseException.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/RsdResponseException.java similarity index 82% rename from scrapers/src/main/java/nl/esciencecenter/rsd/scraper/ResponseException.java rename to scrapers/src/main/java/nl/esciencecenter/rsd/scraper/RsdResponseException.java index 808a7cbf9..3c89f5208 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/ResponseException.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/RsdResponseException.java @@ -7,10 +7,10 @@ package nl.esciencecenter.rsd.scraper; -public class ResponseException extends RuntimeException { +public class RsdResponseException extends RuntimeException { private final Integer statusCode; - public ResponseException(Integer statusCode, String message) { + public RsdResponseException(Integer statusCode, String message) { super(message); this.statusCode = statusCode; } diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/Utils.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/Utils.java index 245d203dc..8ae955f4e 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/Utils.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/Utils.java @@ -63,11 +63,46 @@ public static String get(String uri, String... headers) { throw new RuntimeException(e); } if (response.statusCode() >= 300) { - throw new ResponseException(response.statusCode(), "Error fetching data from endpoint " + uri + " with response: " + response.body()); + throw new RsdResponseException(response.statusCode(), "Error fetching data from endpoint " + uri + " with response: " + response.body()); } return response.body(); } + public static String getWithRetryOn202(int retries, long timeoutMillis, String uri, String... headers) { + if (retries < 0) retries = 0; + + String result = null; + for (int tryNumber = 0; tryNumber < retries + 1; tryNumber++) { + HttpRequest.Builder httpRequestBuilder = HttpRequest.newBuilder() + .GET() + .uri(URI.create(uri)); + if (headers != null && headers.length > 0 && headers.length % 2 == 0) { + httpRequestBuilder.headers(headers); + } + HttpRequest request = httpRequestBuilder.build(); + HttpClient client = HttpClient.newBuilder().followRedirects(HttpClient.Redirect.NORMAL).build(); + HttpResponse response; + try { + response = client.send(request, HttpResponse.BodyHandlers.ofString()); + } catch (IOException | InterruptedException e) { + throw new RuntimeException(e); + } + if (response.statusCode() == 202) { + try { + Thread.sleep(timeoutMillis); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } else if (response.statusCode() == 200) { + result = response.body(); + break; + } else { + throw new RsdResponseException(response.statusCode(), "Error fetching data from endpoint " + uri + " with response: " + response.body()); + } + } + return result; + } + /** * Retrieve data from PostgREST as an admin user and retrieve the response body. * @param uri The URI diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/GithubSI.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/GithubSI.java index 837161163..840997764 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/GithubSI.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/GithubSI.java @@ -10,10 +10,11 @@ import com.google.gson.JsonElement; import com.google.gson.JsonParser; import nl.esciencecenter.rsd.scraper.Config; -import nl.esciencecenter.rsd.scraper.ResponseException; +import nl.esciencecenter.rsd.scraper.RsdResponseException; import nl.esciencecenter.rsd.scraper.Utils; import java.util.Objects; +import java.util.Optional; public class GithubSI implements SoftwareInfo { @@ -30,9 +31,13 @@ public GithubSI(String baseApiUrl, String repo) { */ @Override public String languages() { - return Config.apiCredentialsGithub() - .map(apiCredentials -> Utils.get(baseApiUrl + "/repos/" + repo + "/languages", "Authorization", "Basic " + Utils.base64Encode(apiCredentials))) - .orElseGet(() -> Utils.get(baseApiUrl + "/repos/" + repo + "/languages")); + Optional apiCredentials = Config.apiCredentialsGithub(); + if (apiCredentials.isPresent()) { + return Utils.get(baseApiUrl + "/repos/" + repo + "/languages", "Authorization", "Basic " + Utils.base64Encode(apiCredentials.get())); + } + else { + return Utils.get(baseApiUrl + "/repos/" + repo + "/languages"); + } } /** @@ -40,9 +45,14 @@ public String languages() { */ @Override public String license() { - String repoData = Config.apiCredentialsGithub() - .map(apiCredentials -> Utils.get(baseApiUrl + "/repos/" + repo, "Authorization", "Basic " + Utils.base64Encode(apiCredentials))) - .orElseGet(() -> Utils.get(baseApiUrl + "/repos/" + repo)); + Optional apiCredentials = Config.apiCredentialsGithub(); + String repoData; + if (apiCredentials.isPresent()) { + repoData = Utils.get(baseApiUrl + "/repos/" + repo, "Authorization", "Basic " + Utils.base64Encode(apiCredentials.get())); + } + else { + repoData = Utils.get(baseApiUrl + "/repos/" + repo); + } JsonElement jsonLicense = JsonParser.parseString(repoData).getAsJsonObject().get("license"); return jsonLicense.isJsonNull() ? null : jsonLicense.getAsJsonObject().getAsJsonPrimitive("spdx_id").getAsString(); } @@ -72,14 +82,18 @@ public String license() { public String contributions() { String contributions; try { - contributions = Config.apiCredentialsGithub() - .map(apiCredentials -> Utils.get(baseApiUrl + "/repos/" + repo + "/stats/contributors", "Authorization", "Basic " + Utils.base64Encode(apiCredentials))) - .orElseGet(() -> Utils.get(baseApiUrl + "/repos/" + repo + "/stats/contributors")); - if (contributions.equals("")) { + Optional apiCredentials = Config.apiCredentialsGithub(); + if (apiCredentials.isPresent()) { + contributions = Utils.getWithRetryOn202(1, 3000, baseApiUrl + "/repos/" + repo + "/stats/contributors", "Authorization", "Basic " + Utils.base64Encode(apiCredentials.get())); + } else { + contributions = Utils.getWithRetryOn202(1, 3000, baseApiUrl + "/repos/" + repo + "/stats/contributors"); + } + + if (contributions != null && contributions.equals("")) { // Repository exists, but no contributions yet, empty list is more appropriate contributions = "[]"; } - } catch (ResponseException e) { + } catch (RsdResponseException e) { if (e.getStatusCode() == 404) { // Repository does not exist contributions = null; diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/MainCommits.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/MainCommits.java index 3bd0ae6aa..f91a08bb6 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/MainCommits.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/MainCommits.java @@ -67,6 +67,8 @@ private static void scrapeGitHub() { if (repo.endsWith("/")) repo = repo.substring(0, repo.length() - 1); String scrapedCommits = new AggregateContributionsPerWeekSIDecorator(new GithubSI("https://api.github.com", repo), CodePlatformProvider.GITHUB).contributions(); +// this happens on 202 response, keep the old value + if (scrapedCommits == null) scrapedCommits = commitData.commitHistory(); RepositoryUrlData updatedData = new RepositoryUrlData( commitData.software(), commitData.url(), CodePlatformProvider.GITHUB, commitData.license(), commitData.licenseScrapedAt(), From 8097e2749dbf9cc0bfc74c4c6096da6c299dbcfe Mon Sep 17 00:00:00 2001 From: Ewan Cahen Date: Fri, 4 Nov 2022 16:56:19 +0100 Subject: [PATCH 02/32] build: bump version number --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index bb26c3a4e..8d70373cc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -130,7 +130,7 @@ services: scrapers: build: ./scrapers - image: rsd/scrapers:1.2.0 + image: rsd/scrapers:1.2.1 environment: # it uses values from .env file - POSTGREST_URL From d70e07f9c021c22b44a0195c7195f2ecd26ca876 Mon Sep 17 00:00:00 2001 From: "Dusan Mijatovic (PC2022)" Date: Sat, 5 Nov 2022 22:37:23 +0100 Subject: [PATCH 03/32] fix: initial import keywords from doi --- .../information/ImportKeywordsFromDoi.tsx | 90 +++++++------------ frontend/utils/editKeywords.ts | 4 +- 2 files changed, 33 insertions(+), 61 deletions(-) diff --git a/frontend/components/software/edit/information/ImportKeywordsFromDoi.tsx b/frontend/components/software/edit/information/ImportKeywordsFromDoi.tsx index 376f22cb6..3a69f0156 100644 --- a/frontend/components/software/edit/information/ImportKeywordsFromDoi.tsx +++ b/frontend/components/software/edit/information/ImportKeywordsFromDoi.tsx @@ -14,11 +14,9 @@ import {KeywordForSoftware} from '~/types/SoftwareTypes' import {useState} from 'react' import {getKeywordsFromDoi} from '~/utils/getInfoFromDatacite' import useSnackbar from '~/components/snackbar/useSnackbar' -import {searchForSoftwareKeywordExact} from './searchForSoftwareKeyword' -import {addKeywordsToSoftware, createKeyword} from '~/utils/editKeywords' +import {addKeywordsToSoftware, createOrGetKeyword, KeywordItem} from '~/utils/editKeywords' import {useSession} from '~/auth' import {sortOnStrProp} from '~/utils/sortFn' -import {Keyword} from '~/components/keyword/FindKeyword' import logger from '~/utils/logger' type ImportKeywordsFromDoiProps = { @@ -33,42 +31,6 @@ export default function ImportKeywordsFromDoi({software_id, concept_doi, keyword const [loading, setLoading] = useState(false) const {showInfoMessage,showSuccessMessage} = useSnackbar() - async function addKeyword(selected: Keyword) { - const resp = await addKeywordsToSoftware({ - data:{software:software_id, keyword: selected.id}, - token - }) - return { - ...resp, - keyword: { - ...selected, - software: software_id - } - } - } - - async function createAndAdd(selected: string) { - // create keyword - const resp = await createKeyword({ - keyword: selected, - token - }) - if (resp.status === 201) { - const keyword = { - id: resp.message as string, - keyword: selected, - cnt: null - } - // add keyword after created - return addKeyword(keyword) - } else { - return { - ...resp, - keyword: null - } - } - } - async function importKeywordsFromDoi() { const importedKeywords:KeywordForSoftware[] = [] const failedKeywords:string[] = [] @@ -88,28 +50,40 @@ export default function ImportKeywordsFromDoi({software_id, concept_doi, keyword continue } // is it already added to this software? - const find = keywords.filter(item => item.keyword.trim().toLowerCase() === kw.trim().toLowerCase()) - if (find.length > 0) { + const find = keywords.find(item => item.keyword.trim().toLowerCase() === kw.trim().toLowerCase()) + if (find) { // skip to next item in the loop continue } - // is it already in RSD? - const findDb = await searchForSoftwareKeywordExact({searchFor: kw.trim()}) - if (findDb.length === 1) { - const {status, keyword} = await addKeyword(findDb[0]) - if (status === 200) { - importedKeywords.push(keyword) + // create new or get reference to existing + const create = await createOrGetKeyword({ + keyword: kw.trim(), + token + }) + if (create.status === 201) { + const newKeyword: KeywordItem = create.message + const add = await addKeywordsToSoftware({ + data: { + software: software_id, + keyword: newKeyword.id + }, + token + }) + // if added properly + if (add.status === 200) { + // add to list of imported + importedKeywords.push({ + id: newKeyword.id, + keyword: newKeyword.value, + software: software_id + }) } else { + // add to list of failed failedKeywords.push(kw) } } else { - // create new keyword - const {status, keyword} = await createAndAdd(kw) - if (status === 200 && keyword!==null) { - importedKeywords.push(keyword) - } else { - failedKeywords.push(kw) - } + // add to list of failed + failedKeywords.push(kw) } } if (importedKeywords.length > 0) { @@ -121,13 +95,13 @@ export default function ImportKeywordsFromDoi({software_id, concept_doi, keyword // update collection onSetKeywords(items) showSuccessMessage(`${importedKeywords.length} keywords imported from DOI ${concept_doi}`) + } else if (failedKeywords.length > 0) { + showInfoMessage(`Failed to import keywords [${failedKeywords.toString()}] from DOI [${concept_doi}]`) + // log failure + logger(`importKeywordsFromDoi: Failed to import keywords [${failedKeywords.toString()}] from DOI [${concept_doi}]`,'warn') } else { showInfoMessage(`No (additional) keywords imported from DOI ${concept_doi}`) } - // we only log failed for now - if (failedKeywords.length > 0) { - logger(`Failed to import keywords [${failedKeywords.toString()}] from DOI [${concept_doi}]`,'warn') - } setLoading(false) } diff --git a/frontend/utils/editKeywords.ts b/frontend/utils/editKeywords.ts index 20ac20a16..29d0e452b 100644 --- a/frontend/utils/editKeywords.ts +++ b/frontend/utils/editKeywords.ts @@ -3,8 +3,6 @@ // // SPDX-License-Identifier: Apache-2.0 -// import {SoftwareKeyword} from '~/components/software/edit/information/softwareKeywordsChanges' -import {Keyword} from '~/components/keyword/FindKeyword' import {createJsonHeaders, extractReturnMessage} from './fetchHelpers' import logger from './logger' @@ -15,7 +13,7 @@ export type ProjectKeyword = { keyword: string } -type KeywordItem = { +export type KeywordItem = { id: string, value: string } From 8772fd29f749f0dced33c468a308af57027a40bc Mon Sep 17 00:00:00 2001 From: "Dusan Mijatovic (PC2022)" Date: Sat, 5 Nov 2022 22:56:04 +0100 Subject: [PATCH 04/32] fix: rsd_admin role can edit highligted item. --- .../mention/MentionEditFeatured.tsx | 28 ++++++++++--------- .../components/mention/MentionEditSection.tsx | 3 +- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/frontend/components/mention/MentionEditFeatured.tsx b/frontend/components/mention/MentionEditFeatured.tsx index 496ec02c9..bb8e041dd 100644 --- a/frontend/components/mention/MentionEditFeatured.tsx +++ b/frontend/components/mention/MentionEditFeatured.tsx @@ -6,7 +6,8 @@ import {IconButton} from '@mui/material' import EditIcon from '@mui/icons-material/Edit' import DeleteIcon from '@mui/icons-material/Delete' -import UpdateIcon from '@mui/icons-material/Update' +// import UpdateIcon from '@mui/icons-material/Update' +import {useSession} from '~/auth' import {MentionTitle} from './MentionItemBase' import {MentionItemProps} from '~/types/Mention' import useEditMentionReducer from './useEditMentionReducer' @@ -19,6 +20,7 @@ type MentionListItem = { } export default function MentionEditFeatured({item}: MentionListItem) { + const {user} = useSession() // use context methods to pass btn action // const {onUpdate, confirmDelete, setEditModal} = useContext(EditMentionContext) const {setEditModal,onUpdate,confirmDelete} = useEditMentionReducer() @@ -30,18 +32,18 @@ export default function MentionEditFeatured({item}: MentionListItem) { function renderButtons() { const html = [] - if (item.source.toLowerCase() === 'manual' && - item.doi) { - // we only update items with DOI - html.push( - onUpdate(item)}> - - - ) - } else if (item.source.toLowerCase() === 'manual') { + if (item.doi) { + // WE do not allow update of mentions with DOI from FE + // Dusan 2022-10-19 + // html.push( + // onUpdate(item)}> + // + // + // ) + } else if (user?.role==='rsd_admin') { // manual items without DOI can be edited html.push( sortOnNumProp(a, b, 'publication_year', 'desc')) - .map((item, pos) => { - + .map((item) => { return ( Date: Sat, 5 Nov 2022 23:04:48 +0100 Subject: [PATCH 05/32] fix: organisation logo svg scaling --- .../components/organisation/metadata/OrganisationLogo.tsx | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/frontend/components/organisation/metadata/OrganisationLogo.tsx b/frontend/components/organisation/metadata/OrganisationLogo.tsx index a5a1d9ed2..157392728 100644 --- a/frontend/components/organisation/metadata/OrganisationLogo.tsx +++ b/frontend/components/organisation/metadata/OrganisationLogo.tsx @@ -149,9 +149,7 @@ export default function OrganisationLogo({id,name,logo_id,isMaintainer}: if (isMaintainer) { return (
-
- {renderAvatar()} -
+ {renderAvatar()}
Date: Wed, 26 Oct 2022 21:00:46 +0200 Subject: [PATCH 06/32] fix: show logo on mobile --- frontend/components/organisation/OrganisationCard.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/components/organisation/OrganisationCard.tsx b/frontend/components/organisation/OrganisationCard.tsx index ea43a0403..4bbba9e54 100644 --- a/frontend/components/organisation/OrganisationCard.tsx +++ b/frontend/components/organisation/OrganisationCard.tsx @@ -61,8 +61,8 @@ export default function OrganisationCard(organisation: OrganisationForOverview) /> }
-
-
+
+
Date: Sun, 30 Oct 2022 18:05:00 +0100 Subject: [PATCH 07/32] refactor: align header breakpoints with main and footer --- frontend/components/AppFooter/index.tsx | 2 +- frontend/components/AppHeader/index.tsx | 2 +- frontend/components/layout/MainContent.tsx | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/frontend/components/AppFooter/index.tsx b/frontend/components/AppFooter/index.tsx index fd862c25c..b11758a12 100644 --- a/frontend/components/AppFooter/index.tsx +++ b/frontend/components/AppFooter/index.tsx @@ -18,7 +18,7 @@ export default function AppFooter () { return (