Skip to content

Commit

Permalink
new webin get client
Browse files Browse the repository at this point in the history
  • Loading branch information
dipayan1985 committed Jan 13, 2025
1 parent af9befa commit b9bbcd2
Show file tree
Hide file tree
Showing 12 changed files with 235 additions and 45 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,197 @@
/*
* Copyright 2021 EMBL - European Bioinformatics Institute
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this
* file except in compliance with the License. You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software distributed under the
* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
* CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*/
package uk.ac.ebi.biosamples.client;

import java.io.IOException;
import java.net.URI;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import javax.annotation.PreDestroy;
import lombok.Getter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.web.client.RestTemplateBuilder;
import org.springframework.hateoas.MediaTypes;
import org.springframework.hateoas.client.Traverson;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpRequest;
import org.springframework.http.client.ClientHttpRequestExecution;
import org.springframework.http.client.ClientHttpRequestInterceptor;
import org.springframework.http.client.ClientHttpResponse;
import org.springframework.web.client.RestClientException;
import org.springframework.web.client.RestTemplate;
import org.springframework.web.util.UriComponentsBuilder;
import uk.ac.ebi.biosamples.client.service.ClientService;
import uk.ac.ebi.biosamples.client.service.SampleRetrievalServiceV2;
import uk.ac.ebi.biosamples.client.service.WebinAuthClientService;
import uk.ac.ebi.biosamples.client.utils.ClientProperties;
import uk.ac.ebi.biosamples.model.Sample;
import uk.ac.ebi.biosamples.service.SampleValidator;

public class BioSamplesWebinGetClient implements AutoCloseable {
private final Logger log = LoggerFactory.getLogger(getClass());
private final SampleRetrievalServiceV2 sampleRetrievalServiceV2;
/** -- GETTER -- Gets the public client. */
@Getter private final Optional<BioSamplesClient> publicClient;

/**
* Constructs a BioSamplesClient with the given parameters.
*
* @param uri the URI for BioSamples
* @param uriV2 the URI for BioSamples V2
* @param restTemplateBuilder the RestTemplateBuilder
* @param sampleValidator the SampleValidator
* @param clientService the ClientService
* @param clientProperties the BioSamplesProperties
*/
public BioSamplesWebinGetClient(
final URI uri,
URI uriV2,
final RestTemplateBuilder restTemplateBuilder,
final SampleValidator sampleValidator,
final ClientService clientService,
final ClientProperties clientProperties) {
if (uriV2 == null) {
uriV2 = UriComponentsBuilder.fromUri(URI.create(uri + "/v2")).build().toUri();
}

final RestTemplate restOperations = restTemplateBuilder.build();

if (clientService != null) {
if (clientService instanceof WebinAuthClientService) {
log.trace("Adding WebinClientHttpRequestInterceptor");
restOperations
.getInterceptors()
.add(new BioSamplesWebinGetClient.BsdClientHttpRequestInterceptor(clientService));
} else {
log.trace("No ClientService available");
}
}

final Traverson traverson = new Traverson(uri, MediaTypes.HAL_JSON);
traverson.setRestOperations(restOperations);

sampleRetrievalServiceV2 = new SampleRetrievalServiceV2(restOperations, uriV2);

if (clientService == null) {
publicClient = Optional.empty();
} else {
publicClient =
Optional.of(
new BioSamplesClient(
uri, uriV2, restTemplateBuilder, sampleValidator, null, clientProperties));
}
}

private static class BsdClientHttpRequestInterceptor implements ClientHttpRequestInterceptor {
private final ClientService clientService;

BsdClientHttpRequestInterceptor(final ClientService clientService) {
this.clientService = clientService;
}

@Override
public ClientHttpResponse intercept(
final HttpRequest request, final byte[] body, final ClientHttpRequestExecution execution)
throws IOException {
if (clientService != null && !request.getHeaders().containsKey(HttpHeaders.AUTHORIZATION)) {
final String jwt = clientService.getJwt();
if (jwt != null) {
request.getHeaders().set(HttpHeaders.AUTHORIZATION, "Bearer " + jwt);
}
}

// pass along to the next interceptor
return execution.execute(request, body);
}
}

// TODO: we can think of using an interceptor to remove the cache from the biosamples client
@Override
@PreDestroy
public void close() {
// close down public client if present
publicClient.ifPresent(BioSamplesClient::close);
// close down our own thread pools
// threadPoolExecutor.shutdownNow();
try {
// threadPoolExecutor.awaitTermination(1, TimeUnit.MINUTES);
} catch (final Exception e) {
throw new RuntimeException(e);
}
}

/**
* Fetches sample resources by accessions using BioSamples V2.
*
* @param accessions the list of accessions
* @return a map of sample accessions to samples
* @throws RestClientException if there is an error while fetching samples
*/
public Map<String, Sample> fetchSampleResourcesByAccessionsV2(final List<String> accessions)
throws RestClientException {
try {
return sampleRetrievalServiceV2.fetchSamplesByAccessions(accessions);
} catch (final Exception e) {
throw new RuntimeException(e);
}
}

/**
* Fetches sample resources by accessions using BioSamples V2 with JWT authentication.
*
* @param accessions the list of accessions
* @param jwt the JWT token
* @return a map of sample accessions to samples
* @throws RestClientException if there is an error while fetching samples
*/
public Map<String, Sample> fetchSampleResourcesByAccessionsV2(
final List<String> accessions, final String jwt) throws RestClientException {
try {
return sampleRetrievalServiceV2.fetchSamplesByAccessions(accessions, jwt);
} catch (final Exception e) {
throw new RuntimeException(e);
}
}

/**
* Fetches a sample resource by accession using BioSamples V2.
*
* @param accession the accession of the sample
* @return the sample
* @throws RestClientException if there is an error while fetching the sample
*/
public Sample fetchSampleResourceV2(final String accession) throws RestClientException {
try {
return sampleRetrievalServiceV2.fetchSampleByAccession(accession);
} catch (final Exception e) {
throw new RuntimeException(e);
}
}

/**
* Fetches a sample resource by accession using BioSamples V2 with JWT authentication.
*
* @param accession the accession of the sample
* @param jwt the JWT token
* @return the sample
* @throws RestClientException if there is an error while fetching the sample
*/
public Sample fetchSampleResourceV2(final String accession, final String jwt)
throws RestClientException {
try {
return sampleRetrievalServiceV2.fetchSampleByAccession(accession, jwt);
} catch (final Exception e) {
throw new RuntimeException(e);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
public class NcbiElementCallable implements Callable<Void> {
private static final int MAX_RETRIES = 5;
private final Logger log = LoggerFactory.getLogger(getClass());
private final Element sampleElem;
private final Element sampleElement;
private final String webinId;
private final BioSamplesClient bioSamplesClient;
private final NcbiSampleConversionService ncbiSampleConversionService;
Expand All @@ -39,19 +39,19 @@ public class NcbiElementCallable implements Callable<Void> {
NcbiElementCallable(
final NcbiSampleConversionService ncbiSampleConversionService,
final BioSamplesClient bioSamplesClient,
final Element sampleElem,
final Element sampleElement,
final String webinId,
final Map<String, Set<StructuredDataTable>> sampleToAmrMap) {
this.ncbiSampleConversionService = ncbiSampleConversionService;
this.bioSamplesClient = bioSamplesClient;
this.sampleElem = sampleElem;
this.sampleElement = sampleElement;
this.webinId = webinId;
this.sampleToAmrMap = sampleToAmrMap;
}

@Override
public Void call() {
final String accession = sampleElem.attributeValue("accession");
final String accession = sampleElement.attributeValue("accession");

try {
boolean success = false;
Expand All @@ -67,7 +67,7 @@ public Void call() {

// Generate the sample without the domain
final Sample sampleWithoutAuthInfo =
ncbiSampleConversionService.convertNcbiXmlElementToSample(sampleElem);
ncbiSampleConversionService.convertNcbiXmlElementToSample(sampleElement);
final Sample sample =
Sample.Builder.fromSample(sampleWithoutAuthInfo)
.withWebinSubmissionAccountId(webinId)
Expand All @@ -81,12 +81,8 @@ public Void call() {
try {
bioSamplesClient.persistSampleResource(sample);

success = true;

try {
bioSamplesClient.persistCuration(accession, curation, webinId);
} catch (final Exception e) {
log.info("Failed to curate NCBI sample with ENA link " + accession);
if (bioSamplesClient.fetchSampleResource(sample.getAccession()).isPresent()) {
success = true;
}
} catch (final Exception e) {
if (++numRetry == MAX_RETRIES) {
Expand All @@ -95,8 +91,14 @@ public Void call() {
}
}

try {
bioSamplesClient.persistCuration(accession, curation, webinId);
} catch (final Exception e) {
log.info("Failed to curate NCBI sample with ENA link " + accession);
}

final Set<StructuredDataTable> structuredDataTableSet =
ncbiSampleConversionService.convertNcbiXmlElementToStructuredData(sampleElem, amrData);
ncbiSampleConversionService.convertNcbiXmlElementToStructuredData(sampleElement, amrData);

if (!structuredDataTableSet.isEmpty()) {
final StructuredData structuredData =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,7 @@ public boolean isBlockStart(

// check the date compared to window
final LocalDate updateDate;

if (attributes.getValue("", "last_update") != null) {
updateDate =
LocalDate.parse(
Expand All @@ -156,6 +157,7 @@ public boolean isBlockStart(
}

LocalDate latestDate = updateDate;

if (releaseDate.isAfter(latestDate)) {
latestDate = releaseDate;
}
Expand All @@ -165,7 +167,6 @@ public boolean isBlockStart(
}

return toDate == null || !latestDate.isAfter(toDate);

// hasn't failed, so we must be interested in it
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ public class PipelineFutureCallback implements ThreadUtils.Callback<PipelineResu
@Override
public void call(final PipelineResult pipelineResult) {
totalCount = totalCount + pipelineResult.getModifiedRecords();

if (!pipelineResult.isSuccess()) {
failedSamples.add(pipelineResult.getAccession());
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,9 @@
*/
package uk.ac.ebi.biosamples;

import lombok.Getter;

@Getter
public class PipelineResult {
private final long modifiedRecords;
private final boolean success;
Expand All @@ -20,16 +23,4 @@ public PipelineResult(String accession, long modifiedRecords, boolean success) {
this.modifiedRecords = modifiedRecords;
this.success = success;
}

public String getAccession() {
return accession;
}

public long getModifiedRecords() {
return modifiedRecords;
}

public boolean isSuccess() {
return success;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -43,14 +43,11 @@ private XmlFragmenter() {}
public void handleStream(
final InputStream inputStream, final String encoding, final ElementCallback... callback)
throws ParserConfigurationException, SAXException, IOException {

final InputSource isource = new InputSource(inputStream);

isource.setEncoding(encoding);

final DefaultHandler handler = new FragmentationHandler(callback);
final SAXParser saxParser = factory.newSAXParser();

isource.setEncoding(encoding);
saxParser.parse(isource, handler);
}

Expand Down Expand Up @@ -87,7 +84,9 @@ public void startElement(
}
if (inRegion.get(i)) {
addTextIfNeeded(i);

final Element el;

if (elementStack.get(i).isEmpty()) {
el = doc.get(i).addElement(qName);
} else {
Expand All @@ -96,6 +95,7 @@ public void startElement(
for (int j = 0; j < attributes.getLength(); j++) {
el.addAttribute(attributes.getQName(j), attributes.getValue(j));
}

elementStack.get(i).push(el);
}
}
Expand All @@ -106,6 +106,7 @@ public void endElement(final String uri, final String localName, final String qN
for (int i = 0; i < callbacks.size(); i++) {
if (inRegion.get(i)) {
addTextIfNeeded(i);

elementStack.get(i).pop();

if (elementStack.get(i).isEmpty()) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,8 @@ public static <T> void checkAndCallbackFutures(
}
}

public static interface Callback<T> {
public void call(T t);
public interface Callback<T> {
void call(T t);
}

public static <T, U> void checkAndRetryFutures(
Expand Down
Loading

0 comments on commit b9bbcd2

Please sign in to comment.