From 5ff27438df019ee59b03537d5deab94942f49c81 Mon Sep 17 00:00:00 2001 From: Maarten Plieger Date: Wed, 29 Aug 2018 14:46:02 +0200 Subject: [PATCH 1/7] Content type is now forwared to adaguc-server opendap handler --- .settings/org.eclipse.wst.common.component | 2 +- pom.xml | 2 +- .../knmi/adaguc/services/adagucserver/ADAGUCRequestMapper.java | 2 ++ .../java/nl/knmi/adaguc/services/adagucserver/ADAGUCServer.java | 1 + 4 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.settings/org.eclipse.wst.common.component b/.settings/org.eclipse.wst.common.component index 0f9c906..9f3dbfb 100644 --- a/.settings/org.eclipse.wst.common.component +++ b/.settings/org.eclipse.wst.common.component @@ -1,5 +1,5 @@ - + diff --git a/pom.xml b/pom.xml index e73e6cb..7c84b84 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ nl.knmi.adagucservices adaguc-services - 1.0.5 + 1.0.6 war adaguc-services diff --git a/src/main/java/nl/knmi/adaguc/services/adagucserver/ADAGUCRequestMapper.java b/src/main/java/nl/knmi/adaguc/services/adagucserver/ADAGUCRequestMapper.java index 58752ff..15a5509 100644 --- a/src/main/java/nl/knmi/adaguc/services/adagucserver/ADAGUCRequestMapper.java +++ b/src/main/java/nl/knmi/adaguc/services/adagucserver/ADAGUCRequestMapper.java @@ -62,6 +62,7 @@ public void ADAGUCSERVER(HttpServletResponse response, HttpServletRequest reques } @ResponseBody + @CrossOrigin @RequestMapping("wcs") public void ADAGUCSERVERWCS(HttpServletResponse response, HttpServletRequest request){ @@ -79,6 +80,7 @@ public void ADAGUCSERVERWCS(HttpServletResponse response, HttpServletRequest req } @ResponseBody + @CrossOrigin @RequestMapping("adagucopendap/**") public void ADAGUCSERVEROPENDAP(HttpServletResponse response, HttpServletRequest request){ diff --git a/src/main/java/nl/knmi/adaguc/services/adagucserver/ADAGUCServer.java b/src/main/java/nl/knmi/adaguc/services/adagucserver/ADAGUCServer.java index 1a0a292..df3ffdb 100644 --- a/src/main/java/nl/knmi/adaguc/services/adagucserver/ADAGUCServer.java +++ b/src/main/java/nl/knmi/adaguc/services/adagucserver/ADAGUCServer.java @@ -118,6 +118,7 @@ public static void runADAGUC(HttpServletRequest request,HttpServletResponse resp environmentVariables.add("HOME="+userHomeDir); environmentVariables.add("QUERY_STRING="+queryString); + environmentVariables.add("CONTENT_TYPE="+request.getHeader("Content-Type")); if(serviceType == ADAGUCServiceType.WMS){ environmentVariables.add("ADAGUC_ONLINERESOURCE="+homeURL+"/wms?"); } From a634e4e55800562aff3bae1b38b50cd8b877088a Mon Sep 17 00:00:00 2001 From: Maarten Plieger Date: Wed, 29 Aug 2018 15:51:33 +0200 Subject: [PATCH 2/7] Fixed docker build --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index e5bfe7d..fdd13ee 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,5 @@ -FROM centos:7 +FROM centos/devtoolset-7-toolchain-centos7:7 +USER root MAINTAINER Adaguc Team at KNMI From 14093b3dbc0e45d567894414e2b96d5429c3486a Mon Sep 17 00:00:00 2001 From: Maarten Plieger Date: Fri, 31 Aug 2018 14:00:01 +0200 Subject: [PATCH 3/7] Reduced unneeded logging, authentication can now use headers --- .settings/org.eclipse.wst.common.component | 2 +- pom.xml | 9 +- .../adaguc/security/AuthenticatorImpl.java | 21 +- .../nl/knmi/adaguc/security/PemX509Tools.java | 107 +- .../adaguc/security/SecurityConfigurator.java | 21 +- .../adaguc/security/token/TokenManager.java | 1 - .../adagucserver/ADAGUCRequestMapper.java | 9 +- .../services/adagucserver/ADAGUCServer.java | 17 +- .../services/basket/BasketRequestMapper.java | 33 +- .../adaguc/services/oauth2/OAuth2Handler.java | 1589 +++++++++-------- .../services/oauth2/OAuth2RequestMapper.java | 113 +- .../pywpsserver/PyWPSConfigurator.java | 2 +- .../pywpsserver/PyWPSRequestMapper.java | 2 + .../TinyDapRequestMapper.java | 1 + src/main/resources/application.properties | 4 + 15 files changed, 996 insertions(+), 935 deletions(-) diff --git a/.settings/org.eclipse.wst.common.component b/.settings/org.eclipse.wst.common.component index 9f3dbfb..8bead68 100644 --- a/.settings/org.eclipse.wst.common.component +++ b/.settings/org.eclipse.wst.common.component @@ -1,5 +1,5 @@ - + diff --git a/pom.xml b/pom.xml index 7c84b84..fe29c2a 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ nl.knmi.adagucservices adaguc-services - 1.0.6 + 1.0.7 war adaguc-services @@ -148,12 +148,15 @@ com.github.maartenplieger nl.knmi.adaguc.tools - 1.0.4 + 1.0.10 com.github.maartenplieger nl.knmi.adaguc.config - 1.0.8 + + 1.0.10 com.github.maartenplieger diff --git a/src/main/java/nl/knmi/adaguc/security/AuthenticatorImpl.java b/src/main/java/nl/knmi/adaguc/security/AuthenticatorImpl.java index 38ed3ae..96ce708 100644 --- a/src/main/java/nl/knmi/adaguc/security/AuthenticatorImpl.java +++ b/src/main/java/nl/knmi/adaguc/security/AuthenticatorImpl.java @@ -49,20 +49,31 @@ public synchronized void init(HttpServletRequest request) { if (request == null ) { return; } - // Debug.println("Init"); + /* Get user from session */ String sessionId = null; HttpSession session = request.getSession(); if (session!=null) { sessionId = (String) session.getAttribute("user_identifier"); } - - if (sessionId!=null) { x509 = new PemX509Tools().new X509Info(sessionId, sessionId); Debug.println("Got userid from session"); return; - } else { - Debug.println("No userinfo from session"); + } + + /* Get user from header (Set by SSL client cert verification in NGINX)*/ + try { + String userHeader = SecurityConfigurator.getUserHeader(); + if (userHeader != null) { + String userIdFromHeader = request.getHeader(userHeader); + if (userIdFromHeader != null && userIdFromHeader.length() > 4) { + String userID = new PemX509Tools().getUserIdFromSubjectDN(userIdFromHeader); + Debug.println("Found user from header: " + userID); + x509 = new PemX509Tools().new X509Info(userID, userID); + return; + } + } + } catch (ElementNotFoundException e) { } x509 = new PemX509Tools().getUserIdFromCertificate(request); diff --git a/src/main/java/nl/knmi/adaguc/security/PemX509Tools.java b/src/main/java/nl/knmi/adaguc/security/PemX509Tools.java index 4740e59..7048913 100644 --- a/src/main/java/nl/knmi/adaguc/security/PemX509Tools.java +++ b/src/main/java/nl/knmi/adaguc/security/PemX509Tools.java @@ -146,7 +146,7 @@ public static PrivateKey readPrivateKeyFromPEM (String fileName) throws IOExcept PrivateKeyInfo ukp = (PrivateKeyInfo) object; return converter.getPrivateKey(ukp); } - + } /** @@ -182,6 +182,17 @@ public X509Info getUserIdFromCertificate(HttpServletRequest request){ return null; } + public String getUserIdFromSubjectDN (String subjectDN) { + String[] dnItems = subjectDN.split(", "); + for (int j = 0; j < dnItems.length; j++) { + int CNIndex = dnItems[j].indexOf("CN"); + if (CNIndex != -1) { + return dnItems[j].substring("CN=".length() + + CNIndex); + } + } + return null; + } /** * Returns information about the given certificate, like CN and serial number. This method does not verify * the certificate against trustroots. @@ -194,15 +205,7 @@ public X509Info getUserIdFromCertificate(X509Certificate cert){ String uniqueId = null; uniqueId = "x509_"+cert.getSerialNumber(); String subjectDN = cert.getSubjectDN().toString(); - //Debug.println("getSubjectDN: " + subjectDN); - String[] dnItems = subjectDN.split(", "); - for (int j = 0; j < dnItems.length; j++) { - int CNIndex = dnItems[j].indexOf("CN"); - if (CNIndex != -1) { - CertOpenIdIdentifier = dnItems[j].substring("CN=".length() - + CNIndex); - } - } + CertOpenIdIdentifier = getUserIdFromSubjectDN(subjectDN); if(CertOpenIdIdentifier == null || uniqueId == null){ return null; } @@ -230,33 +233,33 @@ public static X509Certificate signCSR(PKCS10CertificationRequest csr, X509Certif Calendar notBefore = Calendar.getInstance(); //notBefore.add(Calendar., -1); - + JcaPKCS10CertificationRequest jcaRequest = new JcaPKCS10CertificationRequest(csr); X509v3CertificateBuilder certificateBuilder = new JcaX509v3CertificateBuilder(caCert, BigInteger.valueOf(System.currentTimeMillis()), notBefore.getTime(), notAfter.getTime(), jcaRequest.getSubject(), jcaRequest.getPublicKey()); -// JcaX509ExtensionUtils extUtils = new JcaX509ExtensionUtils(); -// certificateBuilder.addExtension(Extension.authorityKeyIdentifier, false, extUtils.createAuthorityKeyIdentifier(caCert)) -// .addExtension(Extension.subjectKeyIdentifier, false, extUtils.createSubjectKeyIdentifier(jcaRequest.getPublicKey())) -// .addExtension(Extension.basicConstraints, true, new BasicConstraints(0)) -// .addExtension(Extension.keyUsage, true, new KeyUsage(KeyUsage.digitalSignature | KeyUsage.keyEncipherment)) -// .addExtension(Extension.extendedKeyUsage, true, new ExtendedKeyUsage(KeyPurposeId.id_kp_serverAuth)); - -// // add pkcs extensions -// Attribute[] attributes = csr.getAttributes(); -// for (Attribute attr : attributes) { -// // process extension request -// if (attr.getAttrType().equals(PKCSObjectIdentifiers.pkcs_9_at_extensionRequest)) { -// Extensions extensions = Extensions.getInstance(attr.getAttrValues().getObjectAt(0)); -// @SuppressWarnings("unchecked") -// Enumeration e = (Enumeration )extensions.oids(); -// while (e.hasMoreElements()) { -// ASN1ObjectIdentifier oid = e.nextElement(); -// Extension ext = extensions.getExtension(oid); -// certificateBuilder.addExtension(oid, ext.isCritical(), ext.getParsedValue()); -// } -// } -// } + // JcaX509ExtensionUtils extUtils = new JcaX509ExtensionUtils(); + // certificateBuilder.addExtension(Extension.authorityKeyIdentifier, false, extUtils.createAuthorityKeyIdentifier(caCert)) + // .addExtension(Extension.subjectKeyIdentifier, false, extUtils.createSubjectKeyIdentifier(jcaRequest.getPublicKey())) + // .addExtension(Extension.basicConstraints, true, new BasicConstraints(0)) + // .addExtension(Extension.keyUsage, true, new KeyUsage(KeyUsage.digitalSignature | KeyUsage.keyEncipherment)) + // .addExtension(Extension.extendedKeyUsage, true, new ExtendedKeyUsage(KeyPurposeId.id_kp_serverAuth)); + + // // add pkcs extensions + // Attribute[] attributes = csr.getAttributes(); + // for (Attribute attr : attributes) { + // // process extension request + // if (attr.getAttrType().equals(PKCSObjectIdentifiers.pkcs_9_at_extensionRequest)) { + // Extensions extensions = Extensions.getInstance(attr.getAttrValues().getObjectAt(0)); + // @SuppressWarnings("unchecked") + // Enumeration e = (Enumeration )extensions.oids(); + // while (e.hasMoreElements()) { + // ASN1ObjectIdentifier oid = e.nextElement(); + // Extension ext = extensions.getExtension(oid); + // certificateBuilder.addExtension(oid, ext.isCritical(), ext.getParsedValue()); + // } + // } + // } ContentSigner signer = new JcaContentSignerBuilder("SHA256withRSA").setProvider("BC").build(caPrivateKey); return new JcaX509CertificateConverter().setProvider("BC").getCertificate(certificateBuilder.build(signer)); @@ -286,7 +289,7 @@ public static String certificateToPemString(Object certHolder) throws IOExceptio public static void writeCertificateToPemFile(Object certHolder, String fileName) throws IOException { Tools.writeFile(fileName, certificateToPemString(certHolder)); } - + public static void writePrivateKeyToPemFile(PrivateKey certHolder, String fileName) throws IOException { Tools.writeFile(fileName, privateKeyToPemString(certHolder)); } @@ -436,27 +439,27 @@ public X509UserCertAndKey setupSLCertificateUser(String clientId, X509Certificat /* Step 5 - Generate CSR */ PKCS10CertificationRequest csr = PemX509Tools.createCSR("CN="+clientId, keyPairCSR); - -// try { -// PemX509Tools.writeCertificateToPemFile(csr, "/tmp/_usercsr.csr"); -// PemX509Tools.writeCertificateToPemFile(caCertificate, "/tmp/_ca.pem"); -// PemX509Tools.writePrivateKeyToPemFile(privateKey, "/tmp/_ca.key"); -// } catch (IOException e) { -// // TODO Auto-generated catch block -// e.printStackTrace(); -// } + + // try { + // PemX509Tools.writeCertificateToPemFile(csr, "/tmp/_usercsr.csr"); + // PemX509Tools.writeCertificateToPemFile(caCertificate, "/tmp/_ca.pem"); + // PemX509Tools.writePrivateKeyToPemFile(privateKey, "/tmp/_ca.key"); + // } catch (IOException e) { + // // TODO Auto-generated catch block + // e.printStackTrace(); + // } /* Step 6 - Sign CSR with CA */ X509Certificate signedCrt = PemX509Tools.signCSR(csr, caCertificate, privateKey); -// try { -// PemX509Tools.writeCertificateToPemFile(signedCrt, "/tmp/_user.crt"); -// PemX509Tools.writePrivateKeyToPemFile(keyPairCSR.getPrivate(), "/tmp/_user.key"); -// } catch (IOException e) { -// // TODO Auto-generated catch block -// e.printStackTrace(); -// } - + // try { + // PemX509Tools.writeCertificateToPemFile(signedCrt, "/tmp/_user.crt"); + // PemX509Tools.writePrivateKeyToPemFile(keyPairCSR.getPrivate(), "/tmp/_user.key"); + // } catch (IOException e) { + // // TODO Auto-generated catch block + // e.printStackTrace(); + // } + return new X509UserCertAndKey(signedCrt, keyPairCSR.getPrivate()); } @@ -495,7 +498,7 @@ public CloseableHttpClient getHTTPClientForPEMBasedClientAuthPEM( } return getHTTPClientForPEMBasedClientAuth(trustStoreLocation,trustStorePassword,certAndKey); } - + /** * * @param trustStoreLocation diff --git a/src/main/java/nl/knmi/adaguc/security/SecurityConfigurator.java b/src/main/java/nl/knmi/adaguc/security/SecurityConfigurator.java index f5202e8..41faf30 100644 --- a/src/main/java/nl/knmi/adaguc/security/SecurityConfigurator.java +++ b/src/main/java/nl/knmi/adaguc/security/SecurityConfigurator.java @@ -2,6 +2,7 @@ import java.util.Vector; +import lombok.Synchronized; import nl.knmi.adaguc.config.ConfigurationReader; import nl.knmi.adaguc.services.oauth2.OAuthConfigurator.Oauth2Settings; import nl.knmi.adaguc.tools.Debug; @@ -39,20 +40,23 @@ public class SecurityConfigurator implements nl.knmi.adaguc.config.ConfiguratorI private static String keyStorePassword=null; private static String keyStoreType="JKS"; private static String keyAlias="tomcat"; + private static String userHeader=null; private static String caCertificate = null; private static String caPrivateKey = null; public static class ComputeNode { public String url = null; + public String name = null; }; static Vector computeNodes = new Vector(); static ConfigurationReader configurationReader = new ConfigurationReader (); + @Synchronized @Override public void doConfig(XMLElement configReader){ - + if(configReader.getNodeValue ("adaguc-services.security")==null){ Debug.println("adaguc-services.security is not configured"); return; @@ -63,20 +67,24 @@ public void doConfig(XMLElement configReader){ keyStore=configReader.getNodeValue("adaguc-services.security.keystore"); keyStorePassword=configReader.getNodeValue("adaguc-services.security.keystorepassword"); keyStoreType=configReader.getNodeValue("adaguc-services.security.keystoretype"); + computeNodes.clear(); keyAlias=configReader.getNodeValue("adaguc-services.security.keyalias"); + userHeader=configReader.getNodeValue("adaguc-services.security.userheader"); + if (configReader.getNodeValue("adaguc-services.security.tokenapi")!=null){ caCertificate=configReader.getNodeValue("adaguc-services.security.tokenapi.cacertificate"); caPrivateKey=configReader.getNodeValue("adaguc-services.security.tokenapi.caprivatekey"); if (configReader.getNodeValue("adaguc-services.security.tokenapi.remote-instances")!=null){ try { - Vector computeNodeElements = configReader.get("adaguc-services").get("security").get("tokenapi").getList("remote-instances"); + Vector computeNodeElements = configReader.get("adaguc-services").get("security").get("tokenapi").get("remote-instances").getList("adaguc-service"); for(int j=0;j headerNames = request.getHeaderNames(); +// while (headerNames.hasMoreElements()) { +// String headerName = headerNames.nextElement(); +// String headerValue = request.getHeader(headerName); +// Debug.println(headerName + ":" + headerValue); +// } +// List environmentVariables = new ArrayList(); String userHomeDir="/tmp/"; @@ -113,8 +123,9 @@ public static void runADAGUC(HttpServletRequest request,HttpServletResponse resp if(queryString == null){ queryString = request.getQueryString(); } - - + Debug.println("[ADAGUC-Server]" + queryString); + + environmentVariables.add("HOME="+userHomeDir); environmentVariables.add("QUERY_STRING="+queryString); diff --git a/src/main/java/nl/knmi/adaguc/services/basket/BasketRequestMapper.java b/src/main/java/nl/knmi/adaguc/services/basket/BasketRequestMapper.java index 0ac7ccb..bf30087 100644 --- a/src/main/java/nl/knmi/adaguc/services/basket/BasketRequestMapper.java +++ b/src/main/java/nl/knmi/adaguc/services/basket/BasketRequestMapper.java @@ -75,21 +75,23 @@ public void listBasket(HttpServletResponse response, HttpServletRequest request) try { boolean enabled = BasketConfigurator.getEnabled(); if(!enabled){ - /* Make a basket of the remote instance */ - Vector computeNodes = SecurityConfigurator.getComputeNodes(); - if (computeNodes.size() == 0) { - jsonResponse.setMessage(new JSONObject().put("error","ADAGUC basket is not enabled and no computenodes are available")); - } else { - String url = computeNodes.get(0).url + "/basket/list"; - Debug.println("Getting basket from " + url); - String basketResponse = UserManager.makeGetRequestWithUserFromServletRequest(request, url); - Debug.println(basketResponse); - jsonResponse.setMessage(new JSONObject(). - put("type","ROOT"). - put("name",computeNodes.get(0).url.replace("https://","")). - put("children", new JSONObject(basketResponse))); - } - }else{ + jsonResponse.setMessage(new JSONObject().put("error", "ADAGUC basket is not enabled")); +// /* Make a basket of the remote instance */ +// Vector computeNodes = SecurityConfigurator.getComputeNodes(); +// if (computeNodes.size() == 0) { +// jsonResponse.setMessage(new JSONObject().put("error","ADAGUC basket is not enabled and no computenodes are available")); +// } else { +// String url = computeNodes.get(0).url + "/basket/list"; +// Debug.println("Getting basket from " + url); +// String basketResponse = UserManager.makeGetRequestWithUserFromServletRequest(request, url); +// Debug.println(basketResponse); +// jsonResponse.setMessage(new JSONObject(). +// put("type","ROOT"). +// put("name",computeNodes.get(0).url.replace("https://","")). +// put("children", new JSONObject(basketResponse))); +// } + }else + { /*Try to use the basket locally available */ Debug.println("getoverview"); String tokenStr=null; @@ -156,6 +158,7 @@ public void uploadBasket(HttpServletResponse response, HttpServletRequest reques try { boolean enabled = BasketConfigurator.getEnabled(); if(!enabled){ + Debug.errprintln("Basket is not enabled"); jsonResponse.setMessage(new JSONObject().put("error","ADAGUC basket is not enabled")); }else{ Debug.println("uploadToBasket"); diff --git a/src/main/java/nl/knmi/adaguc/services/oauth2/OAuth2Handler.java b/src/main/java/nl/knmi/adaguc/services/oauth2/OAuth2Handler.java index 0a5635f..5316e9e 100644 --- a/src/main/java/nl/knmi/adaguc/services/oauth2/OAuth2Handler.java +++ b/src/main/java/nl/knmi/adaguc/services/oauth2/OAuth2Handler.java @@ -27,8 +27,6 @@ of this software and associated documentation files (the "Software"), to deal package nl.knmi.adaguc.services.oauth2; - - import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.StringWriter; @@ -91,12 +89,16 @@ of this software and associated documentation files (the "Software"), to deal import org.json.JSONObject; import org.json.JSONTokener; +import com.fasterxml.jackson.databind.ObjectMapper; + import nl.knmi.adaguc.config.MainServicesConfigurator; import nl.knmi.adaguc.security.CertificateVerificationException; import nl.knmi.adaguc.security.PemX509Tools; import nl.knmi.adaguc.security.PemX509Tools.X509UserCertAndKey; import nl.knmi.adaguc.security.SecurityConfigurator; import nl.knmi.adaguc.security.SecurityConfigurator.ComputeNode; +import nl.knmi.adaguc.security.token.Token; +import nl.knmi.adaguc.security.token.TokenManager; import nl.knmi.adaguc.security.user.User; import nl.knmi.adaguc.security.user.UserManager; import nl.knmi.adaguc.services.oauth2.OAuthConfigurator.Oauth2Settings; @@ -115,800 +117,807 @@ of this software and associated documentation files (the "Software"), to deal * * @author Maarten Plieger and Ernst de Vreede, KNMI * - * If you use parts of this code, please let us know :). + * If you use parts of this code, please let us know :). * */ public class OAuth2Handler { - /* Documentation: - * - * === First of all: === - * !!! Remember to add accounts.google ssl certificate to truststore !!! - * And add other SSL certificates from configured Oauth2 providers like CEDA - * - * === Adding an SSL cert to the truststore can be done like: === - * echo | openssl s_client -connect accounts.google.com:443 2>&1 | sed -ne '/-BEGIN CERTIFICATE-/,/-END CERTIFICATE-/p' > accounts.google.com - * echo | openssl s_client -connect github.com:443 2>&1 | sed -ne '/-BEGIN - * CERTIFICATE-/,/-END CERTIFICATE-/p' > github.com - * keytool -import -v -trustcacerts -alias accounts.google.com -file - * accounts.google.com -keystore esg-truststore2.ts - * keytool -import -v -trustcacerts -alias github.com -file github.com - * -keystore esg-truststore2.ts - * -echo | openssl s_client -connect slcs.ceda.ac.uk:443 2>&1 | sed -ne '/-BEGIN CERTIFICATE-/,/-END CERTIFICATE-/p' > slcs.ceda.ac.uk -keytool -import -v -trustcacerts -alias slcs.ceda.ac.uk -file slcs.ceda.ac.uk -keystore /usr/people/plieger/impactportal/esg-truststore2.ts - - * - * - * === Test URLs to check which are restricted === - * /impactportal/ImpactService?&source=http://vesg.ipsl.fr/thredds/dodsC/esg_dataroot/CMIP5/output1/IPSL/IPSL-CM5A-LR/1pctCO2/day/atmos/cfDay/r1i1p1/v20110427/albisccp/albisccp_cfDay_IPSL-CM5A-LR_1pctCO2_r1i1p1_19700101-19891231.nc&SERVICE=WMS&&SERVICE=WMS&VERSION=1.3.0&REQUEST=GetMap&LAYERS=albisccp&WIDTH=1635&HEIGHT=955&CRS=EPSG:4326&BBOX=-105.13761467889908,-180,105.13761467889908,180&STYLES=auto/nearest&FORMAT=image/png&TRANSPARENT=TRUE&&time=1989-11-27T12:00:00Z - * - * === wget example to climate4impact with an OAuth2 access_token used as bearer in the headers === - * "http://climate4impact.eu/impactportal/ImpactService?&service=basket&request=getoverview&_dc=1424696174221&node=root" - * --header="Authorization: Bearer " - * -O info.txt --no-check-certificate - * - * === wget example with a JWT ID Token to climate4impact === - * wget "http://climate4impact.eu/impactportal/ImpactService?&service=basket&request=getoverview&_dc=1424696174221&node=root" - * --header="Authorization: JWT " - * - * === wget example with an access_token to Google OpenID connect services === - * wget "https://www.googleapis.com/plus/v1/people/me/openIdConnect?" - * --header="Authorization: Bearer " - * -O info.txt --no-check-certificate - * - * === Useful links: === - * - http://self-issued.info/docs/draft-ietf-oauth-v2-bearer.html#authz-header - * - * - http://self-issued.info/docs/draft-jones-json-web-token-01.html#DefiningRSA - * - https://www.googleapis.com/oauth2/v2/certs - * - https://console.developers.google.com/project - * - */ - - static Map oauthStatesMapper = new ConcurrentHashMap();//Remembered states - - public static class StateObject{ - StateObject(String redirectURL){ - this.returnURL = redirectURL; - creationTimeMillies = DateFunctions.getCurrentDateInMillis(); - } - public String returnURL =""; - long creationTimeMillies; - } - - private static void cleanStateObjects(){ - long currentTimeMillis = DateFunctions.getCurrentDateInMillis(); - for (Map.Entry entry : oauthStatesMapper.entrySet()){ - StateObject stateObject = entry.getValue(); - if(currentTimeMillis-stateObject.creationTimeMillies>1000*60){ - Debug.println("Removing unused state with key" +entry.getKey()); - oauthStatesMapper.remove(entry.getKey()); - } - } - } - - static String oAuthCallbackURL = "/oauth"; // The external Servlet location - - /** - * UserInfo object used to share multiple userinfo attributes over functions. - * - * @author plieger - * - */ - public static class UserInfo { - public String user_openid = null; - public String user_identifier = null; - public String user_email = null; - public String certificate; - public String oauth_access_token; - public String certificate_notafter; - } - - /** - * Endpoint which should directly be called by the servlet. - * - * @param request - * @param response - * @throws ElementNotFoundException - */ - public static void doGet(HttpServletRequest request, - HttpServletResponse response) throws ElementNotFoundException { - - // Check if we are dealing with getting JSON request for building up the - // login form - String makeform = null; - try { - makeform = HTTPTools.getHTTPParam(request, "makeform"); - } catch (Exception e) { - } - if (makeform != null) { - makeForm(request, response); - return; - } - - // Check if we are dealing with step1 or step2 in the OAuth process. - String code = null; - try { - code = HTTPTools.getHTTPParam(request, "code"); - } catch (Exception e) { - } - - if (code == null) { - // Step 1 - Debug.println("Step 1: start GetCode request for " - + request.getQueryString()); - try { - getCode(request, response); - } catch (OAuthSystemException e1) { - e1.printStackTrace(); - } catch (IOException e) { - e.printStackTrace(); - } - } else { - // Step 2 - Debug.println("Step 2: start makeOauthzResponse for " - + request.getQueryString()); - makeOauthzResponse(request, response); - - } - }; - - /** - * Step 1: Starts Oauth2 authentication request. It retrieves a one time - * usable code which can be used to retrieve an access token or id token - * - * @param httpRequest - * @return - * @throws OAuthSystemException - * @throws IOException - * @throws ElementNotFoundException - */ - static void getCode(HttpServletRequest httpRequest, - HttpServletResponse response) throws OAuthSystemException, IOException, ElementNotFoundException { - - - Debug.println("getQueryString:"+httpRequest.getQueryString()); - - String returnURL = ""; - try { - returnURL=HTTPTools.getHTTPParam(httpRequest, "returnurl"); - } catch (Exception e1) { - Debug.println("Note: No redir URL given"); - } - - cleanStateObjects(); - - String stateID = UUID.randomUUID().toString(); - - Debug.println("Putting info in stateID ["+stateID+"]"); - oauthStatesMapper.put(stateID, new StateObject(returnURL)); - - String provider = null; - try { - provider = HTTPTools.getHTTPParam(httpRequest, "provider"); - } catch (Exception e) { - } - Debug.println(" OAuth2 Step 1 getCode: Provider is " + provider); - - OAuthConfigurator.Oauth2Settings settings = OAuthConfigurator - .getOAuthSettings(provider); - if (settings == null) { - Debug.errprintln(" OAuth2 Step 1 getCode: No Oauth settings set"); - return; - } - Debug.println(" OAuth2 Step 1 getCode: Using " + settings.id); - - JSONObject state = new JSONObject(); - try { - state.put("provider", provider); - state.put("state_id", stateID); - } catch (JSONException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - - - OAuthClientRequest oauth2ClientRequest = OAuthClientRequest - .authorizationLocation(settings.OAuthAuthLoc) - .setClientId(settings.OAuthClientId) - .setRedirectURI(MainServicesConfigurator.getServerExternalURL() + oAuthCallbackURL) - .setScope(settings.OAuthClientScope).setResponseType("code") - .setState(state.toString()).buildQueryMessage(); - - Debug.println(" OAuth2 Step 1 getCode: locationuri = " - + oauth2ClientRequest.getLocationUri()); - response.sendRedirect(oauth2ClientRequest.getLocationUri()); - } - - /** - * Step 2: Get authorization response. Here the access_tokens and possibly - * id_tokens are retrieved with the previously retrieved code. - * - * @param request - * @param response - */ - public static void makeOauthzResponse(HttpServletRequest request, - HttpServletResponse response) { - try { - OAuthAuthzResponse oar = OAuthAuthzResponse - .oauthCodeAuthzResponse(request); - - String stateResponseAsString = oar.getState(); - if (stateResponseAsString == null) { - stateResponseAsString = ""; - } - if (stateResponseAsString.equals("")) { - Debug.errprintln(" OAuth2 Step 2 OAuthz: FAILED"); - return; - } - - Debug - .println(" OAuth2 Step 2 OAuthz: State is " - + stateResponseAsString); - - JSONObject stateResponseAsJSONObject = (JSONObject) new JSONTokener(stateResponseAsString) - .nextValue(); - - String stateID= stateResponseAsJSONObject.getString("state_id"); - - Debug.println(" OAuth2 Step 2 OAuthz: stateID="+stateID); - - if (request.getParameter("r") != null) { - Debug - .println(" OAuth2 Step 2 OAuthz: Token request already done, stopping"); - return; - } - - String currentProvider = stateResponseAsJSONObject.getString("provider"); - Debug.println(" OAuth2 Step 2 OAuthz: Provider="+currentProvider); - - Debug.println(" OAuth2 Step 2 OAuthz: Starting token request"); - - - - - OAuthConfigurator.Oauth2Settings settings = OAuthConfigurator - .getOAuthSettings(currentProvider); - Debug.println(" OAuth2 Step 2 OAuthz: Using " + settings.id); - // Debug.println(" OAuth2 Step 2 OAuthz: OAuthTokenLoc " + settings.OAuthTokenLoc); - - -// Debug.println(settings.OAuthTokenLoc); -// Debug.println(settings.OAuthClientSecret); - - OAuthClientRequest tokenRequest = OAuthClientRequest - .tokenLocation(settings.OAuthTokenLoc) - .setGrantType(GrantType.AUTHORIZATION_CODE) - .setRedirectURI(MainServicesConfigurator.getServerExternalURL() + oAuthCallbackURL) - .setCode(oar.getCode()) - .setScope(settings.OAuthClientScope) - .setClientId(settings.OAuthClientId) - .setClientSecret(settings.OAuthClientSecret) - - .buildBodyMessage(); - - OAuthClient oauthclient = new OAuthClient(new URLConnectionClient()); - -// Debug.println(tokenRequest.getBody()); -// -// Map map = tokenRequest.getHeaders(); -// -// for (Map.Entry entry : map.entrySet()) { -// System.out.println("Key : " + entry.getKey() + -// " ,Value : " + entry.getValue()); -// } - - - OAuthAccessTokenResponse oauth2Response = oauthclient - .accessToken(tokenRequest); - - Debug.println(" OAuth2 Step 2 OAuthz: Token request succeeded"); - - Debug.println(" OAuth2 Step 2 OAuthz: oauth2Response.getBody():" - + oauth2Response.getBody()); - - Debug.println(" OAuth2 Step 2 OAuthz: ACCESS TOKEN:" - + oauth2Response.getAccessToken()); - - - Debug.println("Found stateID ["+stateID+"]"); - StateObject stateObject = oauthStatesMapper.get(stateID); - - if( oauthStatesMapper.remove(stateID) == null){ - Debug.println("Could not remove stateID ["+stateID+"]"); - }else{ - Debug.println("removeD stateID ["+stateID+"]"); - } - - - if(stateObject == null){ - throw new Exception(" OAuth2 Step 2 OAuthz: Given STATE parameter is not matching, incorrect!!!"); - }else{ - Debug.println(" OAuth2 Step 2 OAuthz: Found state object with key "+stateID); - } - - handleSpecificProviderCharacteristics(request, settings, oauth2Response); - - - - String returnURL = stateObject.returnURL; - if(returnURL.equals("")==false){ - Debug.println("Returning to "+returnURL); - response.sendRedirect(returnURL); - }else{ - response.sendRedirect("http://returnURLNotDefined.knmi.nl:1"); - } - - } catch (Exception e) { - request.getSession().setAttribute("message", "Error in OAuth2 service:\n"+e.getMessage()); - try { - Debug.errprintln("Error in OAuth2 service"); - response.sendRedirect("/impactportal/exception.jsp"); - } catch (IOException e1) { - // TODO Auto-generated catch block - e1.printStackTrace(); - } - e.printStackTrace(); - } - }; - - /** - * All providers are handled a bit different. One of them is CEDA, which - * offers a certificate issuing service for ESGF. - * - * @param request - * @param settings - * @param oauth2Response - * @throws Exception - */ - private static void handleSpecificProviderCharacteristics( - HttpServletRequest request, Oauth2Settings settings, - OAuthAccessTokenResponse oauth2Response) throws Exception { - - if (settings.id.equals("google")) { - try { - /* Google */ - String id_token = oauth2Response.getParam("id_token"); - - if (id_token == null) { - Debug.errprintln("ID TOKEN == NULL!"); - } - if (id_token != null) { - if (id_token.indexOf(".") != -1) { - UserInfo userInfo = getIdentifierFromJWTPayload(TokenDecoder - .base64Decode(id_token.split("\\.")[1])); - -// userInfo.user_openid =userInfo.user_identifier; -// userInfo.user_openid = userInfo.user_openid.replaceAll("\\/", "_"); -// userInfo.user_openid = userInfo.user_openid.replaceAll("\\.", "_"); -// userInfo.user_openid = "https://climate4impact.eu/"+userInfo.user_openid ; -// Debug.println("Setting openid to ["+ userInfo.user_openid+"]"); - if(userInfo == null){ - Debug.errprintln("Error in OAuth2 service getIdentifierFromJWTPayload failed. Check logs!!!"); - return ; - } - setSessionInfo(request, userInfo); - - String accessToken = oauth2Response.getAccessToken(); - Debug.println("ACCESS TOKEN:" + accessToken); - Debug.println("EXPIRESIN:" + oauth2Response.getExpiresIn()); - - } - } - } catch (Exception e) { - e.printStackTrace(); - } - } - }; - - /** - * Step 3 - Make SLCS certificate request to external OAuth2 service A - - * generate keypair B - generate certificate signing request (CSR) C - Request - * certificate from CEDA service using CSR and access_token D - Retrieve user - * identifier from retrieved SLCS - * - * @param currentProvider - * @param accessToken - * @return - * @throws Exception - */ - - /** - * Sets session parameters for the impactportal - * - * @param request - * @param userInfo - * @throws JSONException - */ - public static void setSessionInfo(HttpServletRequest request, - UserInfo userInfo) throws JSONException { - request.getSession() - .setAttribute("openid_identifier", userInfo.user_openid); - request.getSession().setAttribute("user_identifier", - userInfo.user_identifier); - request.getSession().setAttribute("emailaddress", userInfo.user_email); - request.getSession().setAttribute("certificate", userInfo.certificate); - request.getSession().setAttribute("oauth_access_token", userInfo.oauth_access_token); - request.getSession().setAttribute("login_method", "oauth2"); - - try { - JSONObject accessToken = makeUserCertificate(User.makePosixUserId(userInfo.user_identifier)); - if ( accessToken.has("error")){ - Debug.errprintln("Error getting user cert: " + accessToken.toString()); - request.getSession().setAttribute("services_access_token", accessToken.toString()); - } else { - Debug.println("makeUserCertificate succeeded: "+accessToken.toString()); - request.getSession().setAttribute("services_access_token", accessToken.get("token")); - request.getSession().setAttribute("domain", accessToken.get("domain")); - Debug.println("makeUserCertificate succeeded: "+accessToken); - } - - } catch (Exception e) { + /* + * Documentation: + * + * === First of all: === !!! Remember to add accounts.google ssl certificate + * to truststore !!! And add other SSL certificates from configured Oauth2 + * providers like CEDA + * + * === Adding an SSL cert to the truststore can be done like: === echo | + * openssl s_client -connect accounts.google.com:443 2>&1 | sed -ne '/-BEGIN + * CERTIFICATE-/,/-END CERTIFICATE-/p' > accounts.google.com echo | openssl + * s_client -connect github.com:443 2>&1 | sed -ne '/-BEGIN + * CERTIFICATE-/,/-END CERTIFICATE-/p' > github.com keytool -import -v + * -trustcacerts -alias accounts.google.com -file accounts.google.com + * -keystore esg-truststore2.ts keytool -import -v -trustcacerts -alias + * github.com -file github.com -keystore esg-truststore2.ts + * + * echo | openssl s_client -connect slcs.ceda.ac.uk:443 2>&1 | sed -ne + * '/-BEGIN CERTIFICATE-/,/-END CERTIFICATE-/p' > slcs.ceda.ac.uk keytool + * -import -v -trustcacerts -alias slcs.ceda.ac.uk -file slcs.ceda.ac.uk + * -keystore /usr/people/plieger/impactportal/esg-truststore2.ts + * + * + * + * === Test URLs to check which are restricted === + * /impactportal/ImpactService?&source=http://vesg.ipsl.fr/thredds/dodsC/ + * esg_dataroot/CMIP5/output1/IPSL/IPSL-CM5A-LR/1pctCO2/day/atmos/cfDay/ + * r1i1p1/v20110427/albisccp/albisccp_cfDay_IPSL-CM5A- + * LR_1pctCO2_r1i1p1_19700101-19891231.nc&SERVICE=WMS&&SERVICE=WMS&VERSION=1 + * .3.0&REQUEST=GetMap&LAYERS=albisccp&WIDTH=1635&HEIGHT=955&CRS=EPSG:4326& + * BBOX=-105.13761467889908,-180,105.13761467889908,180&STYLES=auto/nearest& + * FORMAT=image/png&TRANSPARENT=TRUE&&time=1989-11-27T12:00:00Z + * + * === wget example to climate4impact with an OAuth2 access_token used as + * bearer in the headers === + * "http://climate4impact.eu/impactportal/ImpactService?&service=basket&request=getoverview&_dc=1424696174221&node=root" + * --header="Authorization: Bearer " -O info.txt + * --no-check-certificate + * + * === wget example with a JWT ID Token to climate4impact === wget + * "http://climate4impact.eu/impactportal/ImpactService?&service=basket&request=getoverview&_dc=1424696174221&node=root" + * --header="Authorization: JWT " + * + * === wget example with an access_token to Google OpenID connect services + * === wget "https://www.googleapis.com/plus/v1/people/me/openIdConnect?" + * --header="Authorization: Bearer " -O info.txt + * --no-check-certificate + * + * === Useful links: === - + * http://self-issued.info/docs/draft-ietf-oauth-v2-bearer.html#authz-header + * + * - http://self-issued.info/docs/draft-jones-json-web-token-01.html# + * DefiningRSA - https://www.googleapis.com/oauth2/v2/certs - + * https://console.developers.google.com/project + * + */ + + static Map oauthStatesMapper = new ConcurrentHashMap();// Remembered + // states + + public static class StateObject { + StateObject(String redirectURL) { + this.returnURL = redirectURL; + creationTimeMillies = DateFunctions.getCurrentDateInMillis(); + } + + public String returnURL = ""; + long creationTimeMillies; + } + + private static void cleanStateObjects() { + long currentTimeMillis = DateFunctions.getCurrentDateInMillis(); + for (Map.Entry entry : oauthStatesMapper.entrySet()) { + StateObject stateObject = entry.getValue(); + if (currentTimeMillis - stateObject.creationTimeMillies > 1000 * 60) { + Debug.println("Removing unused state with key" + entry.getKey()); + oauthStatesMapper.remove(entry.getKey()); + } + } + } + + static String oAuthCallbackURL = "/oauth"; // The external Servlet location + + /** + * UserInfo object used to share multiple userinfo attributes over + * functions. + * + * @author plieger + * + */ + public static class UserInfo { + public String user_openid = null; + public String user_identifier = null; + public String user_email = null; + public String certificate; + public String oauth_access_token; + public String certificate_notafter; + } + + /** + * Endpoint which should directly be called by the servlet. + * + * @param request + * @param response + * @throws ElementNotFoundException + */ + public static void doGet(HttpServletRequest request, HttpServletResponse response) throws ElementNotFoundException { + + // Check if we are dealing with getting JSON request for building up the + // login form + String makeform = null; + try { + makeform = HTTPTools.getHTTPParam(request, "makeform"); + } catch (Exception e) { + } + if (makeform != null) { + makeForm(request, response); + return; + } + + // Check if we are dealing with step1 or step2 in the OAuth process. + String code = null; + try { + code = HTTPTools.getHTTPParam(request, "code"); + } catch (Exception e) { + } + + if (code == null) { + // Step 1 + Debug.println("Step 1: start GetCode request for " + request.getQueryString()); + try { + getCode(request, response); + } catch (OAuthSystemException e1) { + e1.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); + } + } else { + // Step 2 + Debug.println("Step 2: start makeOauthzResponse for " + request.getQueryString()); + makeOauthzResponse(request, response); + + } + }; + + /** + * Step 1: Starts Oauth2 authentication request. It retrieves a one time + * usable code which can be used to retrieve an access token or id token + * + * @param httpRequest + * @return + * @throws OAuthSystemException + * @throws IOException + * @throws ElementNotFoundException + */ + static void getCode(HttpServletRequest httpRequest, HttpServletResponse response) + throws OAuthSystemException, IOException, ElementNotFoundException { + + Debug.println("getQueryString:" + httpRequest.getQueryString()); + + String returnURL = ""; + try { + returnURL = HTTPTools.getHTTPParam(httpRequest, "returnurl"); + } catch (Exception e1) { + Debug.println("Note: No redir URL given"); + } + + cleanStateObjects(); + + String stateID = UUID.randomUUID().toString(); + + Debug.println("Putting info in stateID [" + stateID + "]"); + oauthStatesMapper.put(stateID, new StateObject(returnURL)); + + String provider = null; + try { + provider = HTTPTools.getHTTPParam(httpRequest, "provider"); + } catch (Exception e) { + } + Debug.println(" OAuth2 Step 1 getCode: Provider is " + provider); + + OAuthConfigurator.Oauth2Settings settings = OAuthConfigurator.getOAuthSettings(provider); + if (settings == null) { + Debug.errprintln(" OAuth2 Step 1 getCode: No Oauth settings set"); + return; + } + Debug.println(" OAuth2 Step 1 getCode: Using " + settings.id); + + JSONObject state = new JSONObject(); + try { + state.put("provider", provider); + state.put("state_id", stateID); + } catch (JSONException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + + OAuthClientRequest oauth2ClientRequest = OAuthClientRequest.authorizationLocation(settings.OAuthAuthLoc) + .setClientId(settings.OAuthClientId) + .setRedirectURI(MainServicesConfigurator.getServerExternalURL() + oAuthCallbackURL) + .setScope(settings.OAuthClientScope).setResponseType("code").setState(state.toString()) + .buildQueryMessage(); + + Debug.println(" OAuth2 Step 1 getCode: locationuri = " + oauth2ClientRequest.getLocationUri()); + response.sendRedirect(oauth2ClientRequest.getLocationUri()); + } + + /** + * Step 2: Get authorization response. Here the access_tokens and possibly + * id_tokens are retrieved with the previously retrieved code. + * + * @param request + * @param response + */ + public static void makeOauthzResponse(HttpServletRequest request, HttpServletResponse response) { + try { + OAuthAuthzResponse oar = OAuthAuthzResponse.oauthCodeAuthzResponse(request); + + String stateResponseAsString = oar.getState(); + if (stateResponseAsString == null) { + stateResponseAsString = ""; + } + if (stateResponseAsString.equals("")) { + Debug.errprintln(" OAuth2 Step 2 OAuthz: FAILED"); + return; + } + + Debug.println(" OAuth2 Step 2 OAuthz: State is " + stateResponseAsString); + + JSONObject stateResponseAsJSONObject = (JSONObject) new JSONTokener(stateResponseAsString).nextValue(); + + String stateID = stateResponseAsJSONObject.getString("state_id"); + + Debug.println(" OAuth2 Step 2 OAuthz: stateID=" + stateID); + + if (request.getParameter("r") != null) { + Debug.println(" OAuth2 Step 2 OAuthz: Token request already done, stopping"); + return; + } + + String currentProvider = stateResponseAsJSONObject.getString("provider"); + Debug.println(" OAuth2 Step 2 OAuthz: Provider=" + currentProvider); + + Debug.println(" OAuth2 Step 2 OAuthz: Starting token request"); + + OAuthConfigurator.Oauth2Settings settings = OAuthConfigurator.getOAuthSettings(currentProvider); + Debug.println(" OAuth2 Step 2 OAuthz: Using " + settings.id); + // Debug.println(" OAuth2 Step 2 OAuthz: OAuthTokenLoc " + + // settings.OAuthTokenLoc); + + // Debug.println(settings.OAuthTokenLoc); + // Debug.println(settings.OAuthClientSecret); + + OAuthClientRequest tokenRequest = OAuthClientRequest.tokenLocation(settings.OAuthTokenLoc) + .setGrantType(GrantType.AUTHORIZATION_CODE) + .setRedirectURI(MainServicesConfigurator.getServerExternalURL() + oAuthCallbackURL) + .setCode(oar.getCode()).setScope(settings.OAuthClientScope).setClientId(settings.OAuthClientId) + .setClientSecret(settings.OAuthClientSecret) + + .buildBodyMessage(); + + OAuthClient oauthclient = new OAuthClient(new URLConnectionClient()); + + // Debug.println(tokenRequest.getBody()); + // + // Map map = tokenRequest.getHeaders(); + // + // for (Map.Entry entry : map.entrySet()) { + // System.out.println("Key : " + entry.getKey() + + // " ,Value : " + entry.getValue()); + // } + + OAuthAccessTokenResponse oauth2Response = oauthclient.accessToken(tokenRequest); + + Debug.println(" OAuth2 Step 2 OAuthz: Token request succeeded"); + + Debug.println(" OAuth2 Step 2 OAuthz: oauth2Response.getBody():" + oauth2Response.getBody()); + + Debug.println(" OAuth2 Step 2 OAuthz: ACCESS TOKEN:" + oauth2Response.getAccessToken()); + + Debug.println("Found stateID [" + stateID + "]"); + StateObject stateObject = oauthStatesMapper.get(stateID); + + if (oauthStatesMapper.remove(stateID) == null) { + Debug.println("Could not remove stateID [" + stateID + "]"); + } else { + Debug.println("removeD stateID [" + stateID + "]"); + } + + if (stateObject == null) { + throw new Exception(" OAuth2 Step 2 OAuthz: Given STATE parameter is not matching, incorrect!!!"); + } else { + Debug.println(" OAuth2 Step 2 OAuthz: Found state object with key " + stateID); + } + + handleSpecificProviderCharacteristics(request, settings, oauth2Response); + + String returnURL = stateObject.returnURL; + if (returnURL.equals("") == false) { + Debug.println("Returning to " + returnURL); + response.sendRedirect(returnURL); + } else { + response.sendRedirect("http://returnURLNotDefined.knmi.nl:1"); + } + + } catch (Exception e) { + request.getSession().setAttribute("message", "Error in OAuth2 service:\n" + e.getMessage()); + try { + Debug.errprintln("Error in OAuth2 service"); + response.sendRedirect("/impactportal/exception.jsp"); + } catch (IOException e1) { + // TODO Auto-generated catch block + e1.printStackTrace(); + } + e.printStackTrace(); + } + }; + + /** + * All providers are handled a bit different. One of them is CEDA, which + * offers a certificate issuing service for ESGF. + * + * @param request + * @param settings + * @param oauth2Response + * @throws Exception + */ + private static void handleSpecificProviderCharacteristics(HttpServletRequest request, Oauth2Settings settings, + OAuthAccessTokenResponse oauth2Response) throws Exception { + + if (settings.id.equals("google")) { + try { + /* Google */ + String id_token = oauth2Response.getParam("id_token"); + + if (id_token == null) { + Debug.errprintln("ID TOKEN == NULL!"); + } + if (id_token != null) { + if (id_token.indexOf(".") != -1) { + UserInfo userInfo = getIdentifierFromJWTPayload( + TokenDecoder.base64Decode(id_token.split("\\.")[1])); + + // userInfo.user_openid =userInfo.user_identifier; + // userInfo.user_openid = + // userInfo.user_openid.replaceAll("\\/", "_"); + // userInfo.user_openid = + // userInfo.user_openid.replaceAll("\\.", "_"); + // userInfo.user_openid = + // "https://climate4impact.eu/"+userInfo.user_openid ; + // Debug.println("Setting openid to ["+ + // userInfo.user_openid+"]"); + if (userInfo == null) { + Debug.errprintln( + "Error in OAuth2 service getIdentifierFromJWTPayload failed. Check logs!!!"); + return; + } + setSessionInfo(request, userInfo); + + String accessToken = oauth2Response.getAccessToken(); + Debug.println("ACCESS TOKEN:" + accessToken); + Debug.println("EXPIRESIN:" + oauth2Response.getExpiresIn()); + + } + } + } catch (Exception e) { + e.printStackTrace(); + } + } + }; + + /** + * Step 3 - Make SLCS certificate request to external OAuth2 service A - + * generate keypair B - generate certificate signing request (CSR) C - + * Request certificate from CEDA service using CSR and access_token D - + * Retrieve user identifier from retrieved SLCS + * + * @param currentProvider + * @param accessToken + * @return + * @throws Exception + */ + + /** + * Sets session parameters for the impactportal + * + * @param request + * @param userInfo + * @throws JSONException + */ + public static void setSessionInfo(HttpServletRequest request, UserInfo userInfo) throws JSONException { + request.getSession().setAttribute("openid_identifier", userInfo.user_openid); + request.getSession().setAttribute("user_identifier", userInfo.user_identifier); + request.getSession().setAttribute("emailaddress", userInfo.user_email); + request.getSession().setAttribute("certificate", userInfo.certificate); + request.getSession().setAttribute("oauth_access_token", userInfo.oauth_access_token); + request.getSession().setAttribute("login_method", "oauth2"); + + try { + makeUserCertificate(User.makePosixUserId(userInfo.user_identifier)); + Token token = TokenManager.registerToken(UserManager.getUser(userInfo.user_identifier)); + ObjectMapper om = new ObjectMapper(); + String result = om.writeValueAsString(token); + Debug.println(result); + JSONObject accessToken = new JSONObject(result); + // accessToken.put("domain", + // url.substring(0,url.lastIndexOf("/")).replaceAll("https://", + // "")); + if (accessToken.has("error")) { + Debug.errprintln("Error getting user cert: " + accessToken.toString()); + request.getSession().setAttribute("services_access_token", accessToken.toString()); + } else { + Debug.println("makeUserCertificate succeeded: " + accessToken.toString()); + request.getSession().setAttribute("services_access_token", accessToken.get("token")); + // request.getSession().setAttribute("backend", MainServicesConfigurator.getServerExternalURL()); + Debug.println("makeUserCertificate succeeded: " + accessToken); + } + + } catch (Exception e) { request.getSession().setAttribute("services_access_token", null); Debug.errprintln("makeUserCertificate Failed"); Debug.printStackTrace(e); - } - }; - - public static JSONObject makeUserCertificate(String clientId) throws CertificateException, IOException, InvalidKeyException, NoSuchAlgorithmException, OperatorCreationException, KeyManagementException, UnrecoverableKeyException, KeyStoreException, NoSuchProviderException, SignatureException, GSSException, ElementNotFoundException, CertificateVerificationException, JSONException { - - - User user = UserManager.getUser(clientId); - Debug.println("Making user cert for "+clientId); - X509Certificate caCertificate = PemX509Tools.readCertificateFromPEM(SecurityConfigurator.getCACertificate()); - PrivateKey privateKey = PemX509Tools.readPrivateKeyFromPEM(SecurityConfigurator.getCAPrivateKey()); - X509UserCertAndKey userCert = new PemX509Tools().setupSLCertificateUser(clientId, caCertificate, privateKey); - user.setCertificate(userCert); - - - CloseableHttpClient httpClient = new PemX509Tools().getHTTPClientForPEMBasedClientAuth( - SecurityConfigurator.getTrustStore(), - SecurityConfigurator.getTrustStorePassword().toCharArray(), - userCert); - Debug.println("Created user cert"); - Vector computeNodes = SecurityConfigurator.getComputeNodes(); - - String url = computeNodes.get(0).url + "/registertoken"; - Debug.println("Requesting token from " + url); - CloseableHttpResponse httpResponse = httpClient.execute(new HttpGet(url)); - String result = EntityUtils.toString(httpResponse.getEntity()); - JSONObject resultAsJSON = new JSONObject(result); - resultAsJSON.put("domain", url.substring(0,url.lastIndexOf("/")).replaceAll("https://", "")); - httpResponse.close(); - Debug.println("resultAsJSON:["+resultAsJSON+"]"); - return resultAsJSON; - - } - - /** - * Verifies a signed JWT Id_token with RSA SHA-256 - * - * @param id_token - * @return true if verified - * @throws JSONException - * @throws WebRequestBadStatusException - * @throws IOException - * @throws SignatureException - * @throws InvalidKeyException - * @throws NoSuchAlgorithmException - * @throws InvalidKeySpecException - */ - @SuppressWarnings("unused") - private static boolean verify_JWT_IdToken(String id_token) - throws JSONException, WebRequestBadStatusException, IOException, - SignatureException, InvalidKeyException, NoSuchAlgorithmException, - InvalidKeySpecException { - // http://self-issued.info/docs/draft-jones-json-web-token-01.html#DefiningRSA - // The JWT Signing Input is always the concatenation of a JWT Header - // Segment, a period ('.') character, and the JWT Payload Segment - // RSASSA-PKCS1-V1_5-VERIFY - - // 8.2. Signing a JWT with RSA SHA-256 - // - // This section defines the use of the RSASSA-PKCS1-v1_5 signature algorithm - // as defined in RFC 3447 [RFC3447], Section 8.2 (commonly known as PKCS#1), - // using SHA-256 as the hash function. Note that the use of the - // RSASSA-PKCS1-v1_5 algorithm is described in FIPS 186-3 [FIPS.186‑3], - // Section 5.5, as is the SHA-256 cryptographic hash function, which is - // defined in FIPS 180-3 [FIPS.180‑3]. The reserved "alg" header parameter - // value "RS256" is used in the JWT Header Segment to indicate that the JWT - // Crypto Segment contains an RSA SHA-256 signature. - // - // A 2048-bit or longer key length MUST be used with this algorithm. - // - // The RSA SHA-256 signature is generated as follows: - // - // Let K be the signer's RSA private key and let M be the UTF-8 - // representation of the JWT Signing Input. - // Compute the octet string S = RSASSA-PKCS1-V1_5-SIGN (K, M) using SHA-256 - // as the hash function. - // Base64url encode the octet string S, as defined in this document. - // - // The output is placed in the JWT Crypto Segment for that JWT. - // - // The RSA SHA-256 signature on a JWT is validated as follows: - // - // Take the JWT Crypto Segment and base64url decode it into an octet string - // S. If decoding fails, then the token MUST be rejected. - // Let M be the UTF-8 representation of the JWT Signing Input and let (n, e) - // be the public key corresponding to the private key used by the signer. - // Validate the signature with RSASSA-PKCS1-V1_5-VERIFY ((n, e), M, S) using - // SHA-256 as the hash function. - // If the validation fails, the token MUST be rejected. - // - // Signing with the RSA SHA-384 and RSA SHA-512 algorithms is performed - // identically to the procedure for RSA SHA-256 - just with correspondingly - // longer key and result values. - - Debug.println("Starting verification of id_token"); - Debug.println("[" + id_token + "]"); - String JWTHeader = TokenDecoder.base64Decode(id_token.split("\\.")[0]); - String JWTPayload = TokenDecoder.base64Decode(id_token.split("\\.")[1]); - String JWTSigningInput = id_token.split("\\.")[0] + "." - + id_token.split("\\.")[1]; - String JWTSignature = id_token.split("\\.")[2]; - - Debug.println("Decoded JWT IDToken Header:" + JWTHeader); - Debug.println("Decoded JWT IDToken Payload:" + JWTPayload); - - // Find the discovery page - JSONObject JWTPayLoadObject = (JSONObject) new JSONTokener(JWTPayload) - .nextValue(); - String iss = JWTPayLoadObject.getString("iss"); - Debug.println("iss=" + iss); - - // Load the OpenId discovery page - String discoveryURL = "https://" + iss - + "/.well-known/openid-configuration"; - JSONObject openid_configuration = (JSONObject) new JSONTokener( - HTTPTools.makeHTTPGetRequest(discoveryURL)).nextValue(); - String jwks_uri = openid_configuration.getString("jwks_uri"); - Debug.println("jwks_uri:" + jwks_uri); - - // Load the jwks uri - JSONObject certs = (JSONObject) new JSONTokener( - HTTPTools.makeHTTPGetRequest(jwks_uri)).nextValue(); - JSONArray jwks_keys = certs.getJSONArray("keys"); - Debug.println("jwks_keys:" + jwks_keys.length()); - - JSONObject JWTHeaderObject = (JSONObject) new JSONTokener(JWTHeader) - .nextValue(); - String kid = JWTHeaderObject.getString("kid"); - Debug.println("kid=" + kid); - - String modulus = null; - String exponent = null; - - for (int j = 0; j < jwks_keys.length(); j++) { - if (jwks_keys.getJSONObject(j).getString("kid").equals(kid)) { - Debug.println("Found kid in jwks"); - modulus = jwks_keys.getJSONObject(j).getString("n"); - exponent = jwks_keys.getJSONObject(j).getString("e"); - break; - } - } - return RSASSA_PKCS1_V1_5_VERIFY(modulus, exponent, JWTSigningInput, - JWTSignature); - }; - - /** - * RSASSA-PKCS1-V1_5-VERIFY ((n, e), M, S) using SHA-256 - * - * @param modulus_n - * @param exponent_e - * @param signinInput_M - * @param signature_S - * @return - * @throws SignatureException - * @throws InvalidKeyException - * @throws NoSuchAlgorithmException - * @throws InvalidKeySpecException - */ - static boolean RSASSA_PKCS1_V1_5_VERIFY(String modulus_n, String exponent_e, - String signinInput_M, String signature_S) throws SignatureException, - InvalidKeyException, NoSuchAlgorithmException, InvalidKeySpecException { - Debug.println("Starting verification"); - /* RSA SHA-256 RSASSA-PKCS1-V1_5-VERIFY */ - // Modulus (n from https://www.googleapis.com/oauth2/v2/certs) - String n = modulus_n; - // Exponent (e from https://www.googleapis.com/oauth2/v2/certs) - String e = exponent_e; - // The JWT Signing Input (JWT Header and JWT Payload concatenated with ".") - byte[] M = signinInput_M.getBytes(); - // Signature (JWT Crypto) - byte[] S = Base64.decodeBase64(signature_S); - - byte[] modulusBytes = Base64.decodeBase64(n); - byte[] exponentBytes = Base64.decodeBase64(e); - BigInteger modulusInteger = new BigInteger(1, modulusBytes); - BigInteger exponentInteger = new BigInteger(1, exponentBytes); - - RSAPublicKeySpec rsaPubKey = new RSAPublicKeySpec(modulusInteger, - exponentInteger); - KeyFactory fact = KeyFactory.getInstance("RSA"); - PublicKey pubKey = fact.generatePublic(rsaPubKey); - Signature signature = Signature.getInstance("SHA256withRSA"); - signature.initVerify(pubKey); - signature.update(M); - boolean isVerified = signature.verify(S); - Debug.println("Verify result [" + isVerified + "]"); - return isVerified; - } - - - /** - * Returns unique user identifier from id_token (JWTPayload). The JWT token is - * *NOT* verified. Several impact portal session attributes are set: - - * user_identifier - emailaddress - * - * @param request - * @param JWT - * @return - * @throws ElementNotFoundException - */ - private static UserInfo getIdentifierFromJWTPayload(String JWT) throws ElementNotFoundException { - JSONObject id_token_json = null; - try { - id_token_json = (JSONObject) new JSONTokener(JWT).nextValue(); - } catch (JSONException e1) { - Debug.errprintln("Unable to convert JWT Token to JSON"); - return null; - } - - String email = "null"; - String userSubject = null; - String aud = ""; - try { - email = id_token_json.get("email").toString(); - } catch (JSONException e) { - } - try { - userSubject = id_token_json.get("sub").toString(); - } catch (JSONException e) { - } - - try { - aud = id_token_json.get("aud").toString(); - } catch (JSONException e) { - } - - if (aud == null){ - Debug.errprintln("Error: aud == null"); - return null; - } - if (userSubject == null){ - Debug.errprintln("Error: userSubject == null"); - return null; - } - - //Get ID based on aud (client id) - String clientId = null; - - Vector providernames = OAuthConfigurator.getProviders(); - - for (int j = 0; j < providernames.size(); j++) { - OAuthConfigurator.Oauth2Settings settings = OAuthConfigurator - .getOAuthSettings(providernames.get(j)); - if(settings.OAuthClientId.equals(aud)){ - clientId = settings.id; - } - } - - if(clientId == null){ - Debug.errprintln("Error: could not match OAuthClientId to aud"); - return null; - - } - - String user_identifier = clientId + "/" + userSubject; - String user_openid = null; - UserInfo userInfo = new UserInfo(); - userInfo.user_identifier = user_identifier; - userInfo.user_openid = user_openid; - userInfo.user_email = email; - - Debug.println("getIdentifierFromJWTPayload (id_token): Found unique ID " - + user_identifier); - - return userInfo; - - } - - /** - * Makes a JSON object and sends it to response with information needed for - * building the OAuth2 login form. - * - * @param request - * @param response - * @throws ElementNotFoundException - */ - private static void makeForm(HttpServletRequest request, - HttpServletResponse response) throws ElementNotFoundException { - JSONResponse jsonResponse = new JSONResponse(request); - - JSONObject form = new JSONObject(); - try { - - JSONArray providers = new JSONArray(); - form.put("providers", providers); - Vector providernames = OAuthConfigurator.getProviders(); - - for (int j = 0; j < providernames.size(); j++) { - OAuthConfigurator.Oauth2Settings settings = OAuthConfigurator - .getOAuthSettings(providernames.get(j)); - JSONObject provider = new JSONObject(); - provider.put("id", providernames.get(j)); - provider.put("description", settings.description); - provider.put("logo", settings.logo); - provider.put("registerlink", settings.registerlink); - providers.put(provider); - - } - } catch (JSONException e) { - } - jsonResponse.setMessage(form); - - try { - jsonResponse.print(response); - } catch (Exception e1) { - - } - - }; - - /* Check if an access token was provided in the HttpServletRequest object - and return a user identifier on success. - * - * It returns the unique user identifier. It does this by calling the - userinfo_endpoint using the access_token. - * All endpoints are discovered by reading the open-id Discovery service. - * This is one of the OpenId-Connect extensions on OAuth2 - * - * @param request - * @return - * @throws JSONException - * @throws WebRequestBadStatusException - * @throws IOException - */ - public static UserInfo verifyAndReturnUserIdentifier(HttpServletRequest request) - throws JSONException, WebRequestBadStatusException, IOException, ElementNotFoundException { - - //1) Find the Authorization header containing the access_token - String access_token = request.getHeader("Authorization"); - if (access_token == null){ - //No access token, probably not an OAuth2 request, skip. - return null; - } - Debug.println("Authorization : " + access_token); - - //2) Find the Discovery service, it might have been passed in the request headers: - String discoveryURL = request.getHeader("Discovery"); - if (discoveryURL == null) { - discoveryURL = - "https://accounts.google.com/.well-known/openid-configuration"; - } - Debug.println("Discovery : " + discoveryURL); - - //3 Retrieve the Discovery service, so we get all service endpoints - String discoveryData = HTTPTools.makeHTTPGetRequest(discoveryURL); - JSONObject jsonObject = (JSONObject) new JSONTokener(discoveryData) - .nextValue(); - - //4) Retrieve userinfo endpoint - String userInfoEndpoint = jsonObject.getString("userinfo_endpoint"); - Debug.println("userInfoEndpoint:" + userInfoEndpoint); - - //5) Make a get request with Authorization headers set, the access_token is used here as Bearer. - KVPKey key = new KVPKey(); - key.addKVP("Authorization", access_token); - Debug.println("Starting request"); - String id_token = HTTPTools.makeHTTPGetRequestWithHeaders(userInfoEndpoint,key);// ,"Authorization: Bearer "+access_token); - Debug.println("Finished request"); - - //6) The ID token is retrieved, now return the identifier from this token. - Debug.println("Retrieved id_token=" + id_token); - return getIdentifierFromJWTPayload(id_token); - }; + } + }; + + public static int makeUserCertificate(String clientId) throws CertificateException, IOException, + InvalidKeyException, NoSuchAlgorithmException, OperatorCreationException, KeyManagementException, + UnrecoverableKeyException, KeyStoreException, NoSuchProviderException, SignatureException, GSSException, + ElementNotFoundException, CertificateVerificationException, JSONException { + + User user = UserManager.getUser(clientId); + Debug.println("Making user cert for " + clientId); + X509Certificate caCertificate = PemX509Tools.readCertificateFromPEM(SecurityConfigurator.getCACertificate()); + PrivateKey privateKey = PemX509Tools.readPrivateKeyFromPEM(SecurityConfigurator.getCAPrivateKey()); + X509UserCertAndKey userCert = new PemX509Tools().setupSLCertificateUser(clientId, caCertificate, privateKey); + user.setCertificate(userCert); + + Debug.println("Created user cert"); + return 0; + + // CloseableHttpClient httpClient = new + // PemX509Tools().getHTTPClientForPEMBasedClientAuth( + // SecurityConfigurator.getTrustStore(), + // SecurityConfigurator.getTrustStorePassword().toCharArray(), + // userCert); + // + // + // Vector computeNodes = + // SecurityConfigurator.getComputeNodes(); + // String url = computeNodes.get(0).url + "/registertoken"; + // Debug.println("Requesting token from " + url); + // CloseableHttpResponse httpResponse = httpClient.execute(new + // HttpGet(url)); + // String result = EntityUtils.toString(httpResponse.getEntity()); + // JSONObject resultAsJSON = new JSONObject(result); + // resultAsJSON.put("domain", + // url.substring(0,url.lastIndexOf("/")).replaceAll("https://", "")); + // httpResponse.close(); + // Debug.println("resultAsJSON:["+resultAsJSON+"]"); + // return resultAsJSON; + + } + + /** + * Verifies a signed JWT Id_token with RSA SHA-256 + * + * @param id_token + * @return true if verified + * @throws JSONException + * @throws WebRequestBadStatusException + * @throws IOException + * @throws SignatureException + * @throws InvalidKeyException + * @throws NoSuchAlgorithmException + * @throws InvalidKeySpecException + */ + @SuppressWarnings("unused") + private static boolean verify_JWT_IdToken(String id_token) throws JSONException, WebRequestBadStatusException, + IOException, SignatureException, InvalidKeyException, NoSuchAlgorithmException, InvalidKeySpecException { + // http://self-issued.info/docs/draft-jones-json-web-token-01.html#DefiningRSA + // The JWT Signing Input is always the concatenation of a JWT Header + // Segment, a period ('.') character, and the JWT Payload Segment + // RSASSA-PKCS1-V1_5-VERIFY + + // 8.2. Signing a JWT with RSA SHA-256 + // + // This section defines the use of the RSASSA-PKCS1-v1_5 signature + // algorithm + // as defined in RFC 3447 [RFC3447], Section 8.2 (commonly known as + // PKCS#1), + // using SHA-256 as the hash function. Note that the use of the + // RSASSA-PKCS1-v1_5 algorithm is described in FIPS 186-3 [FIPS.186‑3], + // Section 5.5, as is the SHA-256 cryptographic hash function, which is + // defined in FIPS 180-3 [FIPS.180‑3]. The reserved "alg" header + // parameter + // value "RS256" is used in the JWT Header Segment to indicate that the + // JWT + // Crypto Segment contains an RSA SHA-256 signature. + // + // A 2048-bit or longer key length MUST be used with this algorithm. + // + // The RSA SHA-256 signature is generated as follows: + // + // Let K be the signer's RSA private key and let M be the UTF-8 + // representation of the JWT Signing Input. + // Compute the octet string S = RSASSA-PKCS1-V1_5-SIGN (K, M) using + // SHA-256 + // as the hash function. + // Base64url encode the octet string S, as defined in this document. + // + // The output is placed in the JWT Crypto Segment for that JWT. + // + // The RSA SHA-256 signature on a JWT is validated as follows: + // + // Take the JWT Crypto Segment and base64url decode it into an octet + // string + // S. If decoding fails, then the token MUST be rejected. + // Let M be the UTF-8 representation of the JWT Signing Input and let + // (n, e) + // be the public key corresponding to the private key used by the + // signer. + // Validate the signature with RSASSA-PKCS1-V1_5-VERIFY ((n, e), M, S) + // using + // SHA-256 as the hash function. + // If the validation fails, the token MUST be rejected. + // + // Signing with the RSA SHA-384 and RSA SHA-512 algorithms is performed + // identically to the procedure for RSA SHA-256 - just with + // correspondingly + // longer key and result values. + + Debug.println("Starting verification of id_token"); + Debug.println("[" + id_token + "]"); + String JWTHeader = TokenDecoder.base64Decode(id_token.split("\\.")[0]); + String JWTPayload = TokenDecoder.base64Decode(id_token.split("\\.")[1]); + String JWTSigningInput = id_token.split("\\.")[0] + "." + id_token.split("\\.")[1]; + String JWTSignature = id_token.split("\\.")[2]; + + Debug.println("Decoded JWT IDToken Header:" + JWTHeader); + Debug.println("Decoded JWT IDToken Payload:" + JWTPayload); + + // Find the discovery page + JSONObject JWTPayLoadObject = (JSONObject) new JSONTokener(JWTPayload).nextValue(); + String iss = JWTPayLoadObject.getString("iss"); + Debug.println("iss=" + iss); + + // Load the OpenId discovery page + String discoveryURL = "https://" + iss + "/.well-known/openid-configuration"; + JSONObject openid_configuration = (JSONObject) new JSONTokener(HTTPTools.makeHTTPGetRequest(discoveryURL)) + .nextValue(); + String jwks_uri = openid_configuration.getString("jwks_uri"); + Debug.println("jwks_uri:" + jwks_uri); + + // Load the jwks uri + JSONObject certs = (JSONObject) new JSONTokener(HTTPTools.makeHTTPGetRequest(jwks_uri)).nextValue(); + JSONArray jwks_keys = certs.getJSONArray("keys"); + Debug.println("jwks_keys:" + jwks_keys.length()); + + JSONObject JWTHeaderObject = (JSONObject) new JSONTokener(JWTHeader).nextValue(); + String kid = JWTHeaderObject.getString("kid"); + Debug.println("kid=" + kid); + + String modulus = null; + String exponent = null; + + for (int j = 0; j < jwks_keys.length(); j++) { + if (jwks_keys.getJSONObject(j).getString("kid").equals(kid)) { + Debug.println("Found kid in jwks"); + modulus = jwks_keys.getJSONObject(j).getString("n"); + exponent = jwks_keys.getJSONObject(j).getString("e"); + break; + } + } + return RSASSA_PKCS1_V1_5_VERIFY(modulus, exponent, JWTSigningInput, JWTSignature); + }; + + /** + * RSASSA-PKCS1-V1_5-VERIFY ((n, e), M, S) using SHA-256 + * + * @param modulus_n + * @param exponent_e + * @param signinInput_M + * @param signature_S + * @return + * @throws SignatureException + * @throws InvalidKeyException + * @throws NoSuchAlgorithmException + * @throws InvalidKeySpecException + */ + static boolean RSASSA_PKCS1_V1_5_VERIFY(String modulus_n, String exponent_e, String signinInput_M, + String signature_S) + throws SignatureException, InvalidKeyException, NoSuchAlgorithmException, InvalidKeySpecException { + Debug.println("Starting verification"); + /* RSA SHA-256 RSASSA-PKCS1-V1_5-VERIFY */ + // Modulus (n from https://www.googleapis.com/oauth2/v2/certs) + String n = modulus_n; + // Exponent (e from https://www.googleapis.com/oauth2/v2/certs) + String e = exponent_e; + // The JWT Signing Input (JWT Header and JWT Payload concatenated with + // ".") + byte[] M = signinInput_M.getBytes(); + // Signature (JWT Crypto) + byte[] S = Base64.decodeBase64(signature_S); + + byte[] modulusBytes = Base64.decodeBase64(n); + byte[] exponentBytes = Base64.decodeBase64(e); + BigInteger modulusInteger = new BigInteger(1, modulusBytes); + BigInteger exponentInteger = new BigInteger(1, exponentBytes); + + RSAPublicKeySpec rsaPubKey = new RSAPublicKeySpec(modulusInteger, exponentInteger); + KeyFactory fact = KeyFactory.getInstance("RSA"); + PublicKey pubKey = fact.generatePublic(rsaPubKey); + Signature signature = Signature.getInstance("SHA256withRSA"); + signature.initVerify(pubKey); + signature.update(M); + boolean isVerified = signature.verify(S); + Debug.println("Verify result [" + isVerified + "]"); + return isVerified; + } + + /** + * Returns unique user identifier from id_token (JWTPayload). The JWT token + * is *NOT* verified. Several impact portal session attributes are set: - + * user_identifier - emailaddress + * + * @param request + * @param JWT + * @return + * @throws ElementNotFoundException + */ + private static UserInfo getIdentifierFromJWTPayload(String JWT) throws ElementNotFoundException { + JSONObject id_token_json = null; + try { + id_token_json = (JSONObject) new JSONTokener(JWT).nextValue(); + } catch (JSONException e1) { + Debug.errprintln("Unable to convert JWT Token to JSON"); + return null; + } + + String email = "null"; + String userSubject = null; + String aud = ""; + try { + email = id_token_json.get("email").toString(); + } catch (JSONException e) { + } + try { + userSubject = id_token_json.get("sub").toString(); + } catch (JSONException e) { + } + + try { + aud = id_token_json.get("aud").toString(); + } catch (JSONException e) { + } + + if (aud == null) { + Debug.errprintln("Error: aud == null"); + return null; + } + if (userSubject == null) { + Debug.errprintln("Error: userSubject == null"); + return null; + } + + // Get ID based on aud (client id) + String clientId = null; + + Vector providernames = OAuthConfigurator.getProviders(); + + for (int j = 0; j < providernames.size(); j++) { + OAuthConfigurator.Oauth2Settings settings = OAuthConfigurator.getOAuthSettings(providernames.get(j)); + if (settings.OAuthClientId.equals(aud)) { + clientId = settings.id; + } + } + + if (clientId == null) { + Debug.errprintln("Error: could not match OAuthClientId to aud"); + return null; + + } + + String user_identifier = clientId + "/" + userSubject; + String user_openid = null; + UserInfo userInfo = new UserInfo(); + userInfo.user_identifier = user_identifier; + userInfo.user_openid = user_openid; + userInfo.user_email = email; + + Debug.println("getIdentifierFromJWTPayload (id_token): Found unique ID " + user_identifier); + + return userInfo; + + } + + /** + * Makes a JSON object and sends it to response with information needed for + * building the OAuth2 login form. + * + * @param request + * @param response + * @throws ElementNotFoundException + */ + private static void makeForm(HttpServletRequest request, HttpServletResponse response) + throws ElementNotFoundException { + JSONResponse jsonResponse = new JSONResponse(request); + + JSONObject form = new JSONObject(); + try { + + JSONArray providers = new JSONArray(); + form.put("providers", providers); + Vector providernames = OAuthConfigurator.getProviders(); + + for (int j = 0; j < providernames.size(); j++) { + OAuthConfigurator.Oauth2Settings settings = OAuthConfigurator.getOAuthSettings(providernames.get(j)); + JSONObject provider = new JSONObject(); + provider.put("id", providernames.get(j)); + provider.put("description", settings.description); + provider.put("logo", settings.logo); + provider.put("registerlink", settings.registerlink); + providers.put(provider); + + } + } catch (JSONException e) { + } + jsonResponse.setMessage(form); + + try { + jsonResponse.print(response); + } catch (Exception e1) { + + } + + }; + + /* + * Check if an access token was provided in the HttpServletRequest object + * and return a user identifier on success. + * + * It returns the unique user identifier. It does this by calling the + * userinfo_endpoint using the access_token. All endpoints are discovered by + * reading the open-id Discovery service. This is one of the OpenId-Connect + * extensions on OAuth2 + * + * @param request + * + * @return + * + * @throws JSONException + * + * @throws WebRequestBadStatusException + * + * @throws IOException + */ + public static UserInfo verifyAndReturnUserIdentifier(HttpServletRequest request) + throws JSONException, WebRequestBadStatusException, IOException, ElementNotFoundException { + + // 1) Find the Authorization header containing the access_token + String access_token = request.getHeader("Authorization"); + if (access_token == null) { + // No access token, probably not an OAuth2 request, skip. + return null; + } + Debug.println("Authorization : " + access_token); + + // 2) Find the Discovery service, it might have been passed in the + // request headers: + String discoveryURL = request.getHeader("Discovery"); + if (discoveryURL == null) { + discoveryURL = "https://accounts.google.com/.well-known/openid-configuration"; + } + Debug.println("Discovery : " + discoveryURL); + + // 3 Retrieve the Discovery service, so we get all service endpoints + String discoveryData = HTTPTools.makeHTTPGetRequest(discoveryURL); + JSONObject jsonObject = (JSONObject) new JSONTokener(discoveryData).nextValue(); + + // 4) Retrieve userinfo endpoint + String userInfoEndpoint = jsonObject.getString("userinfo_endpoint"); + Debug.println("userInfoEndpoint:" + userInfoEndpoint); + + // 5) Make a get request with Authorization headers set, the + // access_token is used here as Bearer. + KVPKey key = new KVPKey(); + key.addKVP("Authorization", access_token); + Debug.println("Starting request"); + String id_token = HTTPTools.makeHTTPGetRequestWithHeaders(userInfoEndpoint, key);// ,"Authorization: + // Bearer + // "+access_token); + Debug.println("Finished request"); + + // 6) The ID token is retrieved, now return the identifier from this + // token. + Debug.println("Retrieved id_token=" + id_token); + return getIdentifierFromJWTPayload(id_token); + }; } diff --git a/src/main/java/nl/knmi/adaguc/services/oauth2/OAuth2RequestMapper.java b/src/main/java/nl/knmi/adaguc/services/oauth2/OAuth2RequestMapper.java index 367770b..cdabe81 100644 --- a/src/main/java/nl/knmi/adaguc/services/oauth2/OAuth2RequestMapper.java +++ b/src/main/java/nl/knmi/adaguc/services/oauth2/OAuth2RequestMapper.java @@ -16,6 +16,7 @@ import org.bouncycastle.operator.OperatorCreationException; import org.ietf.jgss.GSSException; +import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; import org.springframework.context.annotation.Bean; @@ -56,56 +57,56 @@ public MappingJackson2HttpMessageConverter mappingJackson2HttpMessageConverter() method = RequestMethod.GET ) public void doOauth(HttpServletResponse response, HttpServletRequest request) throws JSONException, IOException, ElementNotFoundException{ - boolean useDev = false; - if (useDev) { - request.getSession().setAttribute("user_identifier","maarten"); - Vector computeNodes = SecurityConfigurator.getComputeNodes(); - request.getSession().setAttribute("domain",computeNodes.get(0).url.replace("https://", "")); - try { - OAuth2Handler.makeUserCertificate("maarten"); - } catch (InvalidKeyException e1) { - // TODO Auto-generated catch block - e1.printStackTrace(); - } catch (KeyManagementException e1) { - // TODO Auto-generated catch block - e1.printStackTrace(); - } catch (UnrecoverableKeyException e1) { - // TODO Auto-generated catch block - e1.printStackTrace(); - } catch (CertificateException e1) { - // TODO Auto-generated catch block - e1.printStackTrace(); - } catch (NoSuchAlgorithmException e1) { - // TODO Auto-generated catch block - e1.printStackTrace(); - } catch (OperatorCreationException e1) { - // TODO Auto-generated catch block - e1.printStackTrace(); - } catch (KeyStoreException e1) { - // TODO Auto-generated catch block - e1.printStackTrace(); - } catch (NoSuchProviderException e1) { - // TODO Auto-generated catch block - e1.printStackTrace(); - } catch (SignatureException e1) { - // TODO Auto-generated catch block - e1.printStackTrace(); - } catch (GSSException e1) { - // TODO Auto-generated catch block - e1.printStackTrace(); - } catch (CertificateVerificationException e1) { - // TODO Auto-generated catch block - e1.printStackTrace(); - } -; try { - response.sendRedirect(HTTPTools.getHTTPParam(request, "returnurl")); - } catch (Exception e) { - e.printStackTrace(); - response.sendRedirect(MainServicesConfigurator.getServerExternalURL()); - } - return; - } - +// boolean useDev = false; // TODO +// if (useDev) { +// request.getSession().setAttribute("user_identifier","maarten"); +// Vector computeNodes = SecurityConfigurator.getComputeNodes(); +// request.getSession().setAttribute("domain",computeNodes.get(0).url.replace("https://", "")); +// try { +// OAuth2Handler.makeUserCertificate("maarten");// TODO +// } catch (InvalidKeyException e1) { +// // TODO Auto-generated catch block +// e1.printStackTrace(); +// } catch (KeyManagementException e1) { +// // TODO Auto-generated catch block +// e1.printStackTrace(); +// } catch (UnrecoverableKeyException e1) { +// // TODO Auto-generated catch block +// e1.printStackTrace(); +// } catch (CertificateException e1) { +// // TODO Auto-generated catch block +// e1.printStackTrace(); +// } catch (NoSuchAlgorithmException e1) { +// // TODO Auto-generated catch block +// e1.printStackTrace(); +// } catch (OperatorCreationException e1) { +// // TODO Auto-generated catch block +// e1.printStackTrace(); +// } catch (KeyStoreException e1) { +// // TODO Auto-generated catch block +// e1.printStackTrace(); +// } catch (NoSuchProviderException e1) { +// // TODO Auto-generated catch block +// e1.printStackTrace(); +// } catch (SignatureException e1) { +// // TODO Auto-generated catch block +// e1.printStackTrace(); +// } catch (GSSException e1) { +// // TODO Auto-generated catch block +// e1.printStackTrace(); +// } catch (CertificateVerificationException e1) { +// // TODO Auto-generated catch block +// e1.printStackTrace(); +// } +//; try { +// response.sendRedirect(HTTPTools.getHTTPParam(request, "returnurl")); +// } catch (Exception e) { +// e.printStackTrace(); +// response.sendRedirect(MainServicesConfigurator.getServerExternalURL()); +// } +// return; +// } +// OAuth2Handler.doGet(request, response); } /** @@ -128,7 +129,6 @@ public void getId(HttpServletResponse response, HttpServletRequest request) thro String id = (String) request.getSession().getAttribute("user_identifier"); String servicesAccessToken = (String) request.getSession().getAttribute("services_access_token"); String emailAddress = (String) request.getSession().getAttribute("emailaddress"); - String domain = (String) request.getSession().getAttribute("domain"); JSONObject jsonObj = new JSONObject(); @@ -152,12 +152,13 @@ public void getId(HttpServletResponse response, HttpServletRequest request) thro jsonObj.put("email_address",emailAddress); } - if(domain == null || domain.length() == 0 - || isIdUnknown(id)){ - jsonObj.put("domain","undefined"); - }else{ - jsonObj.put("domain",domain); + jsonObj.put("backend",MainServicesConfigurator.getServerExternalURL()); + Vector computeNodes = SecurityConfigurator.getComputeNodes(); + JSONArray jsonArray = new JSONArray(); + for (final ComputeNode computeNode : computeNodes) { + jsonArray.put(new JSONObject().put("url", computeNode.url).put("name", computeNode.name)); } + jsonObj.put("compute", jsonArray); jsonResponse.setMessage(jsonObj); jsonResponse.print(response); diff --git a/src/main/java/nl/knmi/adaguc/services/pywpsserver/PyWPSConfigurator.java b/src/main/java/nl/knmi/adaguc/services/pywpsserver/PyWPSConfigurator.java index 78e33ca..3a324cf 100644 --- a/src/main/java/nl/knmi/adaguc/services/pywpsserver/PyWPSConfigurator.java +++ b/src/main/java/nl/knmi/adaguc/services/pywpsserver/PyWPSConfigurator.java @@ -38,7 +38,7 @@ public class PyWPSConfigurator implements ConfiguratorInterface { static ConfigurationReader configurationReader = new ConfigurationReader (); public void doConfig(XMLElement configReader) throws ElementNotFoundException { if(configReader.getNodeValue ("adaguc-services.pywps-server")==null){ - Debug.println("adaguc-services.pywps-server is not configured"); +// Debug.println("adaguc-services.pywps-server is not configured"); return; } PyWPSExecutable = configReader.getNodeValueMustNotBeUndefined ("adaguc-services.pywps-server.pywpsexecutable"); diff --git a/src/main/java/nl/knmi/adaguc/services/pywpsserver/PyWPSRequestMapper.java b/src/main/java/nl/knmi/adaguc/services/pywpsserver/PyWPSRequestMapper.java index 51b86de..82eb054 100644 --- a/src/main/java/nl/knmi/adaguc/services/pywpsserver/PyWPSRequestMapper.java +++ b/src/main/java/nl/knmi/adaguc/services/pywpsserver/PyWPSRequestMapper.java @@ -8,6 +8,7 @@ import org.springframework.context.annotation.Bean; import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter; import org.springframework.security.core.AuthenticationException; +import org.springframework.web.bind.annotation.CrossOrigin; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.RestController; @@ -30,6 +31,7 @@ public MappingJackson2HttpMessageConverter mappingJackson2HttpMessageConverter() return converter; } @ResponseBody + @CrossOrigin @RequestMapping("wps") public void PyWPSServer(HttpServletResponse response, HttpServletRequest request) throws IOException{ diff --git a/src/main/java/nl/knmi/adaguc/services/tinyopendapserver/TinyDapRequestMapper.java b/src/main/java/nl/knmi/adaguc/services/tinyopendapserver/TinyDapRequestMapper.java index c8f9c06..971bcc8 100644 --- a/src/main/java/nl/knmi/adaguc/services/tinyopendapserver/TinyDapRequestMapper.java +++ b/src/main/java/nl/knmi/adaguc/services/tinyopendapserver/TinyDapRequestMapper.java @@ -37,6 +37,7 @@ public MappingJackson2HttpMessageConverter mappingJackson2HttpMessageConverter() @ResponseBody @RequestMapping("opendap/**") public void runTinyDap(HttpServletResponse response, HttpServletRequest request) throws IOException{ + // Debug.println("/opendap"); /* Three ways of authentication are possible: * 1) - browser session based * 2) - X509, for command line diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index e615564..2642bcd 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -20,3 +20,7 @@ # + +tomcat.ajp.port=9090 +tomcat.ajp.remoteauthentication=false +tomcat.ajp.enabled=true \ No newline at end of file From 8271c97caf1c18e0cc7d7da377a36887a7fa8dbe Mon Sep 17 00:00:00 2001 From: Maarten Plieger Date: Fri, 7 Sep 2018 16:59:55 +0200 Subject: [PATCH 4/7] Added ESGF Search support --- .settings/org.eclipse.wst.common.component | 2 +- pom.xml | 13 +- .../adaguc/services/esgfsearch/DiskCache.java | 88 ++ .../esgfsearch/ESGFSearchConfigurator.java | 54 ++ .../esgfsearch/ESGFSearchRequestMapper.java | 131 +++ .../services/esgfsearch/LockOnQuery.java | 52 ++ .../adaguc/services/esgfsearch/NetCDFC.java | 81 ++ .../services/esgfsearch/OpendapViewer.java | 391 ++++++++ .../adaguc/services/esgfsearch/Search.java | 874 ++++++++++++++++++ .../services/esgfsearch/SessionManager.java | 74 ++ .../esgfsearch/THREDDSCatalogBrowser.java | 504 ++++++++++ .../esgfsearch/THREDDSCatalogToHTML.java | 373 ++++++++ 12 files changed, 2630 insertions(+), 7 deletions(-) create mode 100644 src/main/java/nl/knmi/adaguc/services/esgfsearch/DiskCache.java create mode 100644 src/main/java/nl/knmi/adaguc/services/esgfsearch/ESGFSearchConfigurator.java create mode 100644 src/main/java/nl/knmi/adaguc/services/esgfsearch/ESGFSearchRequestMapper.java create mode 100644 src/main/java/nl/knmi/adaguc/services/esgfsearch/LockOnQuery.java create mode 100644 src/main/java/nl/knmi/adaguc/services/esgfsearch/NetCDFC.java create mode 100644 src/main/java/nl/knmi/adaguc/services/esgfsearch/OpendapViewer.java create mode 100644 src/main/java/nl/knmi/adaguc/services/esgfsearch/Search.java create mode 100644 src/main/java/nl/knmi/adaguc/services/esgfsearch/SessionManager.java create mode 100755 src/main/java/nl/knmi/adaguc/services/esgfsearch/THREDDSCatalogBrowser.java create mode 100644 src/main/java/nl/knmi/adaguc/services/esgfsearch/THREDDSCatalogToHTML.java diff --git a/.settings/org.eclipse.wst.common.component b/.settings/org.eclipse.wst.common.component index 8bead68..3e8bd1b 100644 --- a/.settings/org.eclipse.wst.common.component +++ b/.settings/org.eclipse.wst.common.component @@ -1,5 +1,5 @@ - + diff --git a/pom.xml b/pom.xml index fe29c2a..ab38c96 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ nl.knmi.adagucservices adaguc-services - 1.0.7 + 1.0.3 war adaguc-services @@ -146,17 +146,18 @@ 4.2.2.RELEASE + com.github.maartenplieger nl.knmi.adaguc.tools - 1.0.10 + 1.0.11 com.github.maartenplieger nl.knmi.adaguc.config - - 1.0.10 + + 1.0.11 com.github.maartenplieger diff --git a/src/main/java/nl/knmi/adaguc/services/esgfsearch/DiskCache.java b/src/main/java/nl/knmi/adaguc/services/esgfsearch/DiskCache.java new file mode 100644 index 0000000..8df3697 --- /dev/null +++ b/src/main/java/nl/knmi/adaguc/services/esgfsearch/DiskCache.java @@ -0,0 +1,88 @@ +package nl.knmi.adaguc.services.esgfsearch; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.BasicFileAttributes; +import java.nio.file.attribute.FileTime; +import java.util.Calendar; + +import nl.knmi.adaguc.tools.Tools; +import nl.knmi.adaguc.tools.Debug; + +public class DiskCache { + + /** + * Returns the stored message, null if not available, null if too old + * @param diskCacheLocation + * @param uniqueId + * @param mustbeYoungerThanNSeconds + * @return + */ + public static String get( String diskCacheLocation,String uniqueId, int mustbeYoungerThanNSeconds) { + uniqueId = uniqueId.replaceAll("\\?", "_"); + uniqueId = uniqueId.replaceAll("&", "_"); + uniqueId = uniqueId.replaceAll(":", "_"); + uniqueId = uniqueId.replaceAll("/", "_"); + uniqueId = uniqueId.replaceAll("=", "_"); + + + //Debug.println("Getting from diskcache: "+uniqueId); + + try { + + if(mustbeYoungerThanNSeconds!=0){ + Path fileCacheId = new File(diskCacheLocation+uniqueId).toPath(); + + + BasicFileAttributes attributes = Files.readAttributes(fileCacheId, BasicFileAttributes.class); + FileTime creationTime = attributes.creationTime(); + long createdHowManySecondsAgo = ( Calendar.getInstance().getTimeInMillis()-creationTime.toMillis())/1000; + //DebugConsole.println("Created:"+createdHowManySecondsAgo); + if(createdHowManySecondsAgo>mustbeYoungerThanNSeconds) + { +// Debug.println("Ignoring "+uniqueId+"Because too old."); + Tools.rm(diskCacheLocation+"/"+uniqueId); + return null; + } + //else{ +// Debug.println(fileCacheId.toString()+"("+createdHowManySecondsAgo+"<"+mustbeYoungerThanNSeconds+")"); +// } + } +// Debug.println("Found "+diskCacheLocation+"/"+uniqueId); + return Tools.readFile(diskCacheLocation+"/"+uniqueId); + } catch (IOException e) { + } + return null; + } + + /** + * Store a string in the diskcache system identified with an id + * @param data The data to store + * @param identifier The identifier of this string + */ + public static void set(String diskCacheLocation ,String identifier, String data){ + identifier = identifier.replaceAll("\\?", "_"); + identifier = identifier.replaceAll("&", "_"); + identifier = identifier.replaceAll(":", "_"); + identifier = identifier.replaceAll("/", "_"); + identifier = identifier.replaceAll("=", "_"); + + //Debug.println("Storing to diskcache: "+identifier); + try{ + try { + Tools.mksubdirs(diskCacheLocation); + Tools.writeFile(diskCacheLocation+"/"+identifier, data); + } catch (IOException e) { + + e.printStackTrace(); + } + }catch(Exception e){ + Debug.errprintln("Unable to write to cachelocation "+diskCacheLocation+" with identifier "+identifier); + + } + } + + +} diff --git a/src/main/java/nl/knmi/adaguc/services/esgfsearch/ESGFSearchConfigurator.java b/src/main/java/nl/knmi/adaguc/services/esgfsearch/ESGFSearchConfigurator.java new file mode 100644 index 0000000..1319595 --- /dev/null +++ b/src/main/java/nl/knmi/adaguc/services/esgfsearch/ESGFSearchConfigurator.java @@ -0,0 +1,54 @@ +package nl.knmi.adaguc.services.esgfsearch; + +import lombok.Synchronized; +import nl.knmi.adaguc.config.ConfigurationReader; +import nl.knmi.adaguc.tools.ElementNotFoundException; +import nl.knmi.adaguc.tools.MyXMLParser.XMLElement; + +/** + * + * @author maartenplieger + * + + + + changeit + + + */ + + +public class ESGFSearchConfigurator implements nl.knmi.adaguc.config.ConfiguratorInterface { + + private static String esgfSearchURL=null; + private static String cacheLocation="/tmp/"; + private static boolean enabled = false; + + static ConfigurationReader configurationReader = new ConfigurationReader (); + + @Synchronized + @Override + public void doConfig(XMLElement configReader){ + if(configReader.getNodeValue ("adaguc-services.esgfsearch")==null){ + return; + } + esgfSearchURL=configReader.getNodeValue("adaguc-services.esgfsearch.searchurl"); + enabled="true".equals(configReader.getNodeValue("adaguc-services.esgfsearch.enabled")); + cacheLocation=configReader.getNodeValue("adaguc-services.esgfsearch.cachelocation"); + } + public static String getEsgfSearchURL() throws ElementNotFoundException { + configurationReader.readConfig(); + return esgfSearchURL; + } + public static boolean getEnabled() throws ElementNotFoundException { + configurationReader.readConfig(); + return enabled; + } + public static String getCacheLocation() throws ElementNotFoundException { + configurationReader.readConfig(); + return cacheLocation; + } + +} + + diff --git a/src/main/java/nl/knmi/adaguc/services/esgfsearch/ESGFSearchRequestMapper.java b/src/main/java/nl/knmi/adaguc/services/esgfsearch/ESGFSearchRequestMapper.java new file mode 100644 index 0000000..90b59c9 --- /dev/null +++ b/src/main/java/nl/knmi/adaguc/services/esgfsearch/ESGFSearchRequestMapper.java @@ -0,0 +1,131 @@ +package nl.knmi.adaguc.services.esgfsearch; + +import java.io.IOException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.json.JSONObject; +import org.springframework.beans.factory.DisposableBean; +import org.springframework.context.annotation.Bean; +import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter; +import org.springframework.web.bind.annotation.CrossOrigin; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.ResponseBody; +import org.springframework.web.bind.annotation.RestController; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; + +import nl.knmi.adaguc.config.MainServicesConfigurator; +import nl.knmi.adaguc.tools.Debug; +import nl.knmi.adaguc.tools.ElementNotFoundException; +import nl.knmi.adaguc.tools.JSONResponse; + +@RestController +@RequestMapping("esgfsearch") +@CrossOrigin +public class ESGFSearchRequestMapper implements DisposableBean { + private static ExecutorService threadPool = null; + private static Search esgfSearch = null; + + + public ESGFSearchRequestMapper() throws ElementNotFoundException { + super(); + esgfSearch=getESGFSearchInstance(); + } + + public void destroy() { + Debug.println("Shutting down"); + threadPool.shutdown(); + esgfSearch = null; + } + + + + @Bean + public MappingJackson2HttpMessageConverter mappingJackson2HttpMessageConverter() { + ObjectMapper mapper = new ObjectMapper(); + mapper.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, true); + MappingJackson2HttpMessageConverter converter = + new MappingJackson2HttpMessageConverter(mapper); + return converter; + } + + + @ResponseBody + @RequestMapping("/search") + public void search(HttpServletResponse response, HttpServletRequest request) throws IOException{ + JSONResponse jsonResponse = new JSONResponse(request); + + + Debug.println("esgfsearch/search received"); + try { + boolean enabled = ESGFSearchConfigurator.getEnabled(); + if(!enabled){ + jsonResponse.setMessage(new JSONObject().put("error", "ADAGUC esgfsearch is not enabled")); + }else{ + Debug.println("getoverview"); + esgfSearch.doGet(request,response); + } + } catch (Exception e) { + jsonResponse.setException("error: "+e.getMessage(), e); + } + } + + @ResponseBody + @RequestMapping("/catalog") + public void catalog(HttpServletResponse response, HttpServletRequest request) throws IOException{ + JSONResponse jsonResponse = new JSONResponse(request); + + + Debug.println("esgfsearch/catalog received"); + try { + boolean enabled = ESGFSearchConfigurator.getEnabled(); + if(!enabled){ + jsonResponse.setMessage(new JSONObject().put("error", "ADAGUC esgfsearch is not enabled")); + }else{ + Debug.println("catalog"); + + (new THREDDSCatalogToHTML()).handleCatalogBrowserRequest(request,response); + } + } catch (Exception e) { + jsonResponse.setException("error: "+e.getMessage(), e); + } + } + @ResponseBody + @RequestMapping("/getvariables") + public void getvariables(HttpServletResponse response, HttpServletRequest request) throws IOException{ + JSONResponse jsonResponse = new JSONResponse(request); + + + Debug.println("esgfsearch/getvariables received"); + try { + boolean enabled = ESGFSearchConfigurator.getEnabled(); + if(!enabled){ + jsonResponse.setMessage(new JSONObject().put("error", "ADAGUC esgfsearch is not enabled")); + }else{ + Debug.println("catalog"); + OpendapViewer viewer = new OpendapViewer( ESGFSearchConfigurator.getCacheLocation()); + viewer.doGet(request, response); + } + } catch (Exception e) { + jsonResponse.setException("error: "+e.getMessage(), e); + } + } + + + + + public static synchronized Search getESGFSearchInstance() throws ElementNotFoundException { + if(esgfSearch!=null)return esgfSearch; + + Debug.println("Creating new ESGF search instance with endpoint "+ESGFSearchConfigurator.getEsgfSearchURL()); + threadPool = Executors.newFixedThreadPool(4); + esgfSearch = new Search(ESGFSearchConfigurator.getEsgfSearchURL(), MainServicesConfigurator.getBaseDir()+"/diskCache/",threadPool); + return esgfSearch; + } + +} diff --git a/src/main/java/nl/knmi/adaguc/services/esgfsearch/LockOnQuery.java b/src/main/java/nl/knmi/adaguc/services/esgfsearch/LockOnQuery.java new file mode 100644 index 0000000..4f26425 --- /dev/null +++ b/src/main/java/nl/knmi/adaguc/services/esgfsearch/LockOnQuery.java @@ -0,0 +1,52 @@ +package nl.knmi.adaguc.services.esgfsearch; + +import java.util.Vector; + +import nl.knmi.adaguc.tools.Debug; + +public class LockOnQuery { + + static Vector busyURLs = new Vector(); + + /** + * Locks on any query + * @param query The query string to wait for + * @param waitMillis If zero, the function blocks until the query has been released by another thread. If nonzero, the amount of millis is waited and then returns. + * @return Zero when lock is released, One when lock is still there but timeout has expired. + */ + static public int lock(String query,int waitMillis){ + if(query == null){ + return 2; + } + //Debug.println("lock "+query); + synchronized(busyURLs){ + try { + if(waitMillis <=0 ){ + while(busyURLs.contains(query)){ + busyURLs.wait(); + } + }else{ + if(busyURLs.contains(query)){ + busyURLs.wait(waitMillis); + Debug.println( "lock timeout on "+query); + return 1; + } + } + + } catch (InterruptedException e) { + e.printStackTrace(); + } + + busyURLs.add(query); + } + //Debug.println("unlock"); + return 0; + } + + static public void release(String query){ + synchronized(busyURLs){ + busyURLs.remove(query); + busyURLs.notifyAll(); + } + } +} diff --git a/src/main/java/nl/knmi/adaguc/services/esgfsearch/NetCDFC.java b/src/main/java/nl/knmi/adaguc/services/esgfsearch/NetCDFC.java new file mode 100644 index 0000000..251b42b --- /dev/null +++ b/src/main/java/nl/knmi/adaguc/services/esgfsearch/NetCDFC.java @@ -0,0 +1,81 @@ +package nl.knmi.adaguc.services.esgfsearch; + +import nl.knmi.adaguc.security.user.User; +import nl.knmi.adaguc.tools.Debug; +import nl.knmi.adaguc.tools.ProcessRunner; + +public class NetCDFC { + static class NCDump{ + + + public String doNCDump(User user,String url) throws Exception{ + + class StderrPrinter implements ProcessRunner.StatusPrinterInterface{ + String ncdumpError=""; + public void print(byte[] message,int bytesRead) {Debug.errprint(new String(message,0,bytesRead));ncdumpError+=new String(message,0,bytesRead);} + public void setError(String message) {} + public String getError() {return ncdumpError;} + @Override + public boolean hasData() { + // TODO Auto-generated method stub + return false; + } + } + class StdoutPrinter implements ProcessRunner.StatusPrinterInterface{ + String ncdumpResult=""; + public void print(byte[] message,int bytesRead) {ncdumpResult+=new String(message,0,bytesRead);} + public void setError(String message) {} + public String getError() {return ncdumpResult;} + @Override + public boolean hasData() { + // TODO Auto-generated method stub + return false; + } + } + + + ProcessRunner.StatusPrinterInterface stdoutPrinter = new StdoutPrinter(); + ProcessRunner.StatusPrinterInterface stderrPrinter = new StderrPrinter(); + String userHome=""; + try{ + userHome=user.getHomeDir(); + }catch(Exception e){ + } + String[] environmentVariables = {"HOME="+userHome}; + ProcessRunner processRunner = new ProcessRunner (stdoutPrinter,stderrPrinter,environmentVariables,userHome); + try{ + String commands[]={"ncdump","-h","-x",url}; + Debug.println("starting ncdump for "+url); + processRunner.runProcess(commands,null); + } + catch (Exception e){ + Debug.errprint(e.getMessage()); + throw new Exception ("Unable to do ncdump: "+e.getMessage()); + } + + String ncdumpResult=stdoutPrinter.getError(); + String ncdumpError=stderrPrinter.getError(); + + if(ncdumpResult.length()==0){ + if(ncdumpError.length()>0){ + throw new Exception ("Unable to do ncdump, error occured:["+ncdumpError+"]"); + }else{ + throw new Exception ("Unable to do ncdump, no info available."); + } + } + + return ncdumpResult; + } + } + + + + public static String executeNCDumpCommand(User user,String url) throws Exception{ + //Debug.println("Execute ncdump on "+url); + String result; + NCDump doNCDump = new NCDump(); + result=doNCDump.doNCDump(user,url); + doNCDump = null; + return result; + } +} diff --git a/src/main/java/nl/knmi/adaguc/services/esgfsearch/OpendapViewer.java b/src/main/java/nl/knmi/adaguc/services/esgfsearch/OpendapViewer.java new file mode 100644 index 0000000..a68dda4 --- /dev/null +++ b/src/main/java/nl/knmi/adaguc/services/esgfsearch/OpendapViewer.java @@ -0,0 +1,391 @@ +package nl.knmi.adaguc.services.esgfsearch; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.util.List; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import javax.servlet.http.HttpSession; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +import nl.knmi.adaguc.security.AuthenticatorFactory; +import nl.knmi.adaguc.security.AuthenticatorInterface; +import nl.knmi.adaguc.security.user.User; +import nl.knmi.adaguc.security.user.UserManager; +import nl.knmi.adaguc.services.esgfsearch.SessionManager.DatasetViewerSession; +import nl.knmi.adaguc.tools.Debug; +import nl.knmi.adaguc.tools.HTTPTools; +import nl.knmi.adaguc.tools.JSONResponse; +import nl.knmi.adaguc.tools.MyXMLParser; +import nl.knmi.adaguc.tools.MyXMLParser.XMLElement; + + +public class OpendapViewer { + + String cacheLocation = null; + public OpendapViewer(String cacheLocation) { + + this.cacheLocation = cacheLocation; + } + + + + /** + * Returns a JSONObject containing ncdump like structure for given opendap URL. + * @param requestStr + * @param request + * @return + */ + public JSONResponse viewOpenDap(String requestStr,HttpServletRequest request){ + + JSONResponse jsonResponse = new JSONResponse(request); + /* Check if we really have an URL here and not a localfile */ + String prefixCheck = requestStr.toLowerCase(); + if(prefixCheck.startsWith("http")==false&&prefixCheck.startsWith("dods")==false){ + jsonResponse.setErrorMessage("Invalid opendap URL given, no HTTP or DODS prefix",500); + return jsonResponse; + } + HttpSession session = request.getSession(); + DatasetViewerSession datasetViewerSession=(DatasetViewerSession) session.getAttribute("datasetviewersession"); + if(datasetViewerSession==null){ + Debug.println("Creating new datasetviewersession"); + datasetViewerSession = new DatasetViewerSession();session.setAttribute("datasetviewersession",datasetViewerSession); + } + datasetViewerSession.datasetURL=requestStr; + + + try{ + //Strip the # token. + requestStr = requestStr.split("#")[0]; + Debug.println("dodsRequest="+requestStr); + + String ncdumpMessage = ""; + + + User user = null; + try{ + AuthenticatorInterface authenticator = AuthenticatorFactory.getAuthenticator(request); + if(authenticator != null){ + try { + user = UserManager.getUser(authenticator); + } catch(Exception e){ + Debug.println("No user information provided: " + e.getMessage()); + } + + } + }catch(Exception e){ + } + + try{ + ncdumpMessage=NetCDFC.executeNCDumpCommand(user,requestStr); + }catch(Exception e){ + //If not give it the ncdump error + if(jsonResponse.hasError() == false){ + jsonResponse.setException("NCDump error", e); + } + return jsonResponse; + } + + + + + //Debug.println("Trying to parse ncdump message"); + MyXMLParser.XMLElement rootElement = new MyXMLParser.XMLElement(); + rootElement.parseString(ncdumpMessage); + //Debug.println("Parsed"); + //DebugConsole.println(rootElement.toString()); + + Listdimensions = rootElement.get("netcdf").getList("dimension"); + Listvariables = rootElement.get("netcdf").getList("variable"); + + JSONArray variableInfo = new JSONArray (); + + JSONObject jsonVariable = new JSONObject(); + + String varName= "nc_global"; + jsonVariable.put("variable",varName);//.getName()); + jsonVariable.put("longname","File metadata");//.getName()); + + Listattributes = rootElement.get("netcdf").getList("attribute"); + if(attributes.size()>0){ + JSONArray jsonattributeArray = new JSONArray(); + for(int a=0;a0){ + variableInfo.put(jsonVariable); + } + + + + for(int iterateOption=0;iterateOption<2;iterateOption++){ + for(int j=0;jattributes1 = variables.get(j).getList("attribute"); + JSONArray jsonattributeArray = new JSONArray(); + + for(int a=0;a0){ + if(isDimension&&iterateOption==0){ + variableInfo.put(jsonVariable1); + }else if(!isDimension&&iterateOption==1){ + variableInfo.put(jsonVariable1); + } + //} + + } + } + + + + + jsonResponse.setMessage(variableInfo.toString()); + + variableInfo = null; + rootElement = null; + variables.clear();variables=null; + + /*}catch(WebRequestBadStatusException e){ + String msg="Unable to get file "+requestStr+".

\n"+e.getMessage()+"
\n";//+e.getResult(); + MessagePrinters.emailFatalErrorMessage("File access",msg); + DebugConsole.errprintln(msg); + JSONArray errorVar = new JSONArray (); + try { + JSONObject error = new JSONObject(); + error.put("error",msg); + errorVar.put(error); + } catch (JSONException e1) {} + out1.print(errorVar.toString()+"\n\n\n"); + + */ + }catch(Exception e){ + Debug.errprintln("ncdump exception "+e.getMessage()); + String requestStrWithOpenId = requestStr; + User user = null; + try{ + AuthenticatorInterface authenticator = AuthenticatorFactory.getAuthenticator(request); + if(authenticator != null){ + try { + user = UserManager.getUser(authenticator); + } catch(Exception e2){ + Debug.println("No user information provided: " + e2.getMessage()); + } + + } + }catch(Exception e3){ + } + if(user!=null){ + if(user.getUserId()!=null){ + if(requestStrWithOpenId.indexOf("?")==-1){ + requestStrWithOpenId +="?"; + }else{ + requestStrWithOpenId +="&"; + } + try { + requestStrWithOpenId+="openid="+URLEncoder.encode(user.getUserId(),"utf-8"); + } catch (UnsupportedEncodingException e1) { + e1.printStackTrace(); + } + } + } + String msg="Unable to get file "+requestStr+".\n\n"+e.getMessage(); + + //DebugConsole.errprintln(msg); + JSONArray errorVar = new JSONArray (); + try { + JSONObject error = new JSONObject(); + msg = msg.replaceAll("esg-orp", "impactportal/esg-orp"); + msg = msg.replaceAll("\n", "
"); + + error.put("error",msg); + errorVar.put(error); + } catch (JSONException e1) {} + jsonResponse.setErrorMessage(errorVar.toString()+"\n\n\n",500); + + } + + + + return jsonResponse; + } + + + + private String checkAttrValues(String attrValue) { + byte b[]= attrValue.getBytes(); + + //Valid tokens should be in this range, otherwise replace with exclamation mark. + for(int j=0;j126)b[j]='!'; + } + attrValue= new String(b); + return attrValue; + } + + + + + protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { + JSONResponse jsonresponse = new JSONResponse(request); + try { + String service=HTTPTools.getHTTPParam(request,"service"); + if("getvariables".equals(service)){ + String query=HTTPTools.getHTTPParam(request,"request"); + jsonresponse = viewOpenDap(query,request); + } + } catch (Exception e) { + jsonresponse.setException("Unable to get variables", e); + } + jsonresponse.print(response); + } +} diff --git a/src/main/java/nl/knmi/adaguc/services/esgfsearch/Search.java b/src/main/java/nl/knmi/adaguc/services/esgfsearch/Search.java new file mode 100644 index 0000000..6b1c19f --- /dev/null +++ b/src/main/java/nl/knmi/adaguc/services/esgfsearch/Search.java @@ -0,0 +1,874 @@ +package nl.knmi.adaguc.services.esgfsearch; + + +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.MalformedURLException; +import java.net.URLEncoder; +import java.util.Map; +import java.util.SortedMap; +import java.util.SortedSet; +import java.util.TreeMap; +import java.util.Vector; +import java.util.concurrent.Callable; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import javax.servlet.http.HttpSession; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONTokener; + +import nl.knmi.adaguc.tools.DateFunctions; +import nl.knmi.adaguc.tools.Debug; +import nl.knmi.adaguc.tools.HTTPTools; +import nl.knmi.adaguc.tools.JSONResponse; +import nl.knmi.adaguc.tools.JSONResponse.JSONResponseException; +import nl.knmi.adaguc.tools.KVPKey; +import nl.knmi.adaguc.tools.MyXMLParser; +import nl.knmi.adaguc.tools.MyXMLParser.XMLElement; +import nl.knmi.adaguc.tools.WebRequestBadStatusException; + + +public class Search { + private int maxAmountOfDataSetsInJSON = 250; + private int searchCacheTimeValiditySec = 10*60; + private int catalogContentsTimeValiditySec = 10*60; + private int catalogCheckerTimeValiditySec = 60*5; + private int catalogCheckerTimeOutMS = 2000; + private int searchGetTimeOutMS = 15000; + + private String searchEndPoint = null; + private String cacheLocation = null; + private ExecutorService getCatalogExecutor = null; + + public Search(String searchEndPoint, String cacheLocation, ExecutorService getCatalogExecutor) { + this.searchEndPoint = searchEndPoint; + this.cacheLocation = cacheLocation; + this.getCatalogExecutor =getCatalogExecutor; + } + + public JSONResponse getFacets(String facets, String query,int pageNumber,int pageLimit) { + + try{ + LockOnQuery.lock(facets+query,0); + JSONResponse r = _getFacetsImp(facets,query,pageNumber,pageLimit); + LockOnQuery.release(facets+query); + return r; + }catch(Exception e){ + JSONResponse r = new JSONResponse(); + r.setException(e.getClass().getName(), e); + + e.printStackTrace(); + return r; + } + + } + + private JSONResponse _getFacetsImp(String facets,String query, int pageNumer,int searchLimit) throws JSONException { + + JSONResponse r = new JSONResponse(); + + String esgfQuery = "facets=*&offset="+(pageNumer*searchLimit)+"&limit="+searchLimit+"&sort=true&"; + + if(facets!=null){ + esgfQuery = "facets="+facets+"&limit="+searchLimit+"&sort=true&"; + } + + if(query!=null){ + Debug.println("QUERY is "+query); + KVPKey kvp = HTTPTools.parseQueryString(query); + SortedSet kvpKeys = kvp.getKeys(); + for(String k : kvpKeys){ + if(!k.equalsIgnoreCase("time_start_stop")&&!k.equalsIgnoreCase("bbox")&&!k.equalsIgnoreCase("query")){ + //Debug.println("KEY "+k+" = "+kvp.getValue(k)); + for(String value : kvp.getValue(k)){ + try { + esgfQuery = esgfQuery+k+"="+URLEncoder.encode(value,"UTF-8")+"&"; + } catch (UnsupportedEncodingException e) { + e.printStackTrace(); + } + } + }else{ + if(k.equalsIgnoreCase("time_start_stop")){ + String timeStartStopValue = kvp.getValue(k).firstElement(); + if(timeStartStopValue.length()==9){ + String [] timeStartStopValues = timeStartStopValue.split("/"); + if(timeStartStopValues.length==2){ + int yearStart = Integer.parseInt(timeStartStopValues[0]); + int yearStop = Integer.parseInt(timeStartStopValues[1]); + if(yearStart>=0&&yearStart<=9999&&yearStop>=0&&yearStop<=9999){ + esgfQuery += "start="+String.format("%04d", yearStart)+"-01-01T00:00:00Z&"; + esgfQuery += "end="+String.format("%04d", yearStop)+"-01-01T00:00:00Z&"; + + } + } + } + } + } + if(k.equalsIgnoreCase("bbox")){ + String bboxValue = kvp.getValue(k).firstElement(); + if(bboxValue.length()>3){ + String[] bboxValueItems =bboxValue.split(","); + if(bboxValueItems.length == 4){ + esgfQuery += "bbox=%5B"+bboxValue+"%5D&"; + } + } + + } + if(k.equalsIgnoreCase("query")){ + String freeTextValue = kvp.getValue(k).firstElement(); + if(freeTextValue.length()>0){ + try { + esgfQuery += "query="+URLEncoder.encode(freeTextValue,"UTF-8")+"&"; + } catch (UnsupportedEncodingException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + + } + } + } + + Debug.println("Query is "+searchEndPoint+esgfQuery); + + String identifier = "ESGFSearch.getFacets"+esgfQuery; + + String XML = DiskCache.get(cacheLocation, identifier+".xml",searchCacheTimeValiditySec); + if(XML == null){ + String url = searchEndPoint+esgfQuery; + try { + + XML = HTTPTools.makeHTTPGetRequestWithTimeOut(url,searchGetTimeOutMS); + DiskCache.set(cacheLocation,identifier+".xml",XML); + } catch (MalformedURLException e2) { + r.setException("MalformedURLException",e2,url); + return r; + } catch (IOException e2) { + r.setException("IOException",e2,url); + return r; + } + } + + + + MyXMLParser.XMLElement el = new MyXMLParser.XMLElement(); + + try { + el.parseString(XML); + } catch (Exception e1) { + r.setErrorMessage("Unable to parse XML",500); + return r; + } + + JSONObject facetsObj = new JSONObject(); + + + try { + Vector lst=el.get("response").getList("lst"); + + for(XMLElement a : lst){ + + try{ + if(a.getAttrValue("name").equals("facet_counts")){ + Vector facet_counts=a.getList("lst"); + for(XMLElement facet_count : facet_counts){ + if(facet_count.getAttrValue("name").equals("facet_fields")){ + Vector facet_fields=facet_count.getList("lst"); + for(XMLElement facet_field : facet_fields){ + + Vector facet_names=facet_field.getList("int"); + SortedMap sortedFacetElements = new TreeMap(); + for(XMLElement facet_name : facet_names){ + sortedFacetElements.put(facet_name.getAttrValue("name"),Integer.parseInt(facet_name.getValue())); + } + + JSONObject facet = new JSONObject(); + + //int first = 0; + for (SortedMap.Entry entry : sortedFacetElements.entrySet()){ + //if(first <5){ + //Debug.println(entry.getKey()); + //first++; + facet.put(entry.getKey(), entry.getValue()); + //} + } + facetsObj.put(facet_field.getAttrValue("name"),facet); + } + } + } + } + }catch(Exception e){ + r.setErrorMessage("No name attribute",500); + return r; + } + } + + + facetsObj.put("time_start_stop",new JSONArray("[1850%2F1950]")); + facetsObj.put("bbox",new JSONArray("[0]")); + facetsObj.put("query",new JSONArray("[0]")); + } catch (Exception e) { + e.printStackTrace(); + } + + + + JSONObject result = new JSONObject(); + JSONObject responseObj = new JSONObject(); + result.put("response",responseObj); + + responseObj.put("limit",searchLimit); + if (query.contains("clear=")){ + responseObj.put("query", ""); + } else { + responseObj.put("query", query); + } + + + + JSONArray searchResults = new JSONArray(); + + try { + Vector result1=el.get("response").getList("result"); + + for(XMLElement a : result1){ + + try{ + if(a.getAttrValue("name").equals("response")){ + responseObj.put("numfound",Integer.parseInt(a.getAttrValue("numFound"))); + Vector doclist=a.getList("doc"); + + for(XMLElement doc : doclist){ + //Debug.println(doc.toString()); + String esgfurl = ""; + String esgfid = ""; + String esgfdatanode = ""; + JSONObject searchResult = new JSONObject(); + searchResults.put(searchResult); + Vector arrlist = doc.getList("arr"); + Vector strlist = doc.getList("str"); + for(XMLElement arr : arrlist){ + String attrName = arr.getAttrValue("name"); + if(attrName.equals("url")){ + String urlToCheck = arr.get("str").getValue().split("#")[0]; + urlToCheck = urlToCheck.split("\\|")[0]; + searchResult.put("url",urlToCheck); + esgfurl = urlToCheck; + } + } + for(XMLElement str : strlist){ + String attrName = str.getAttrValue("name"); + if(attrName.equals("id")){ + esgfid=str.getValue().split("\\|")[0]; + searchResult.put("esgfid",esgfid); + } + if(attrName.equals("data_node")){ + esgfdatanode=str.getValue().split("\\|")[0]; + searchResult.put("data_node",esgfdatanode); + } + + } + + //Compose the unique id; + if(esgfdatanode.length()==0){ + esgfdatanode = esgfurl; + } + searchResult.put("id",esgfdatanode+"::"+esgfid); + + } + } + }catch(Exception e){ + r.setErrorMessage("No name attribute",500); + return r; + } + } + + } catch (Exception e) { + e.printStackTrace(); + } + + try { + responseObj.put("results",searchResults ); + } catch (JSONException e) { + r.setException("JSONException unable to put response", e); + return r; + } + + try { + result.put("facets", facetsObj); + } catch (JSONException e) { + r.setException("JSONException unable to put facets", e); + return r; + } + + r.setMessage(result.toString()); + + + return r; + } + + + + public class ASyncGetCatalogRequest implements Callable { + private String url; + private HttpServletRequest request; + public ASyncGetCatalogRequest(String url, HttpServletRequest request) { + this.url = url; + this.request = request; + } + + @Override + public ASyncGetCatalogResponse call() throws Exception { + Exception e = null; + String a = null; + try{ + a=getCatalog(url,request); + //Debug.println("Catalog OK "); + }catch(Exception e2){ + //Debug.errprintln("ASyncGetCatalogResponse: exception: "+e2.getMessage()); + e=e2; + } + return new ASyncGetCatalogResponse(a,e); + } + } + + public class ASyncGetCatalogResponse { + private String body; + private Exception exception; +// boolean _isFinished=false; + + public ASyncGetCatalogResponse(String string, Exception exception) { + this.body = string; + this.exception = exception; +// _isFinished=true; + } + + public String getBody() { + return body; + } + + public Exception getException(){ + return exception; + } + +// public boolean isFin2ished(){ +// return _isFinished; +// } + } + + + static Map urlsBeingChecked = new ConcurrentHashMap (); + + class URLBeingChecked{ + public Future response; + private long creationDate; + public long getCreationDate(){ + return creationDate; + } + public URLBeingChecked(String query, HttpServletRequest request) { + creationDate = DateFunctions.getCurrentDateInMillis(); + try { + response = getCatalogExecutor.submit(new ASyncGetCatalogRequest(query,request)); + creationDate = DateFunctions.getCurrentDateInMillis(); + } catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + + } + + public JSONResponse checkURL(String query,HttpServletRequest request) { + synchronized(urlsBeingChecked){ + //Debug.println("Checking: "+query); + JSONResponse r = new JSONResponse(request); + if(urlsBeingChecked.containsKey((String)query)==false){ + //Debug.println("INSERT"); + URLBeingChecked urlBeingChecked = new URLBeingChecked(query,request); + urlsBeingChecked.put((String)query, urlBeingChecked); + r.setMessage("{\"message\":\"start checking\",\"ok\":\"busy\"}"); + }else{ + URLBeingChecked urlBeingChecked = urlsBeingChecked.get((String)query); + + if(urlBeingChecked.getCreationDate()+(catalogCheckerTimeValiditySec*1000)maxAmountOfDataSetsInJSON){ + throw new JSONResponse.JSONResponseException("Too many results, maximum of "+maxAmountOfDataSetsInJSON+" datasets allowed.",200); + }else{ + jsonresult.put("numDatasets", searchResults.getJSONObject("response").getLong("numfound")); + jsonresult.put("ok", "ok");//For client to check whether its all OK. + + + KVPKey kvp = HTTPTools.parseQueryString(query); + Vector variableList = kvp.getValue("variable"); + String variableFilter = ""; + for(int k=0;k0){ + variableFilter+="|"; + } + variableFilter+=variableList.get(k); + } + + + JSONArray results = searchResults.getJSONObject("response").getJSONArray("results"); + int numFiles = 0; + int numDap = 0; + long totalFileSize = 0; + JSONArray catalogAggregation = new JSONArray(); + for(int j=0;j0){ + numdatasets++; + result.append("catalogurl;");result.append(text);result.append(";");result.append(catalogURL);result.append("\n"); + }else{ + throw new Exception("No records for this set."); + } + } + for(int i=0;iaccesTypes = new Vector(); + static class AccesType{ + String serviceType; + String base; + } + void addAccessType(String serviceType,String base){ + serviceType = serviceType.toLowerCase(); + if(serviceType.equals("opendap"))serviceType="dapurl"; + if(serviceType.equals("httpserver"))serviceType="httpurl"; + if(serviceType.equals("catalog"))serviceType="catalogurl"; + AccesType accesType = new AccesType(); + accesType.serviceType=serviceType; + accesType.base=base; + //Debug.println("serviceType:"+serviceType); + //Debug.println("base:"+base); + accesTypes.add(accesType); + } + } + + public static JSONArray browseThreddsCatalog(HttpServletRequest request, String variableFilter,String textFilter) throws MalformedURLException, Exception { + Debug.println("variableFilter="+variableFilter); + String nodeStr=request.getParameter("node"); + + if(nodeStr!=null){nodeStr=URLDecoder.decode(nodeStr,"UTF-8");}else{ + throw new Exception("Invalid node argument given");//errorResponder.printexception("nodeStr="+nodeStr);return null; + } + if(nodeStr.indexOf("http")!=0){ + throw new Exception("Invalid URL given"); + } + return browseThreddsCatalog(request,nodeStr,variableFilter,textFilter); + + } + + public static JSONArray browseThreddsCatalog(HttpServletRequest request,String catalogURL, String variableFilter,String textFilter) throws MalformedURLException, Exception { + long startTimeInMillis = Calendar.getInstance().getTimeInMillis(); + long timeStampBefore = startTimeInMillis; + long timeStampAfter; + String _catalogURL= HTTPTools.makeCleanURL(catalogURL); + String rootCatalog = new URL(_catalogURL).toString(); + String path = new URL(rootCatalog).getFile(); + String hostPath = rootCatalog.substring(0,rootCatalog.length()-path.length()); + + if(measureTime){ + timeStampAfter = Calendar.getInstance().getTimeInMillis(); + Debug.println("TIME: URL and HOST check: ("+(timeStampAfter-timeStampBefore)+" ms)"); + timeStampBefore = timeStampAfter; + } + + MyXMLParser.XMLElement catalogElement = new MyXMLParser.XMLElement(); + try { + Debug.println("Getting "+rootCatalog); + Debug.println("Getting "+catalogURL); + + //Try to get from local storage: + Search esgfSearch = ESGFSearchRequestMapper.getESGFSearchInstance();//new Search(Configuration.VercSearchConfig.getEsgfSearchURL(),Configuration.getImpactWorkspace()+"/diskCache/"); + if(esgfSearch==null){ + Debug.println("esgfSearch==null"); + return null; + } + String result = esgfSearch.getCatalog(catalogURL,request); + + + //Debug.println("catalogURL"+catalogURL); + if(catalogURL.endsWith(".catalog")){ + //Debug.println("OK"); + //Debug.println(result); + JSONObject o = ((JSONObject) new JSONTokener(result).nextValue()); + JSONArray a = new JSONArray(); + a.put(o); + return a; + } + if(measureTime){ + timeStampAfter = Calendar.getInstance().getTimeInMillis(); + Debug.println("TIME: Catalog Get: ("+(timeStampAfter-timeStampBefore)+" ms)"); + timeStampBefore = timeStampAfter; + } + + catalogElement.parseString(result); + }catch (WebRequestBadStatusException web){ + Debug.errprintln("Unable to load catalog: unable to GET:"+web.getMessage()); + throw web; + } catch (Exception e1) { + + //MessagePrinters.emailFatalErrorMessage("Unable to load catalog: "+e1.getMessage(),rootCatalog); + Debug.errprintln("Unable to load catalog: invalid XML");//+e1.getMessage()); + Debug.printStackTrace(e1); + throw e1; + } + if(measureTime){ + timeStampAfter = Calendar.getInstance().getTimeInMillis(); + Debug.println("TIME: XML String parsing: ("+(timeStampAfter-timeStampBefore)+" ms)"); + timeStampBefore = timeStampAfter; + } + Vector supportedServices = getSupportedServices(catalogElement.get("catalog")); + + /*DebugConsole.println("SupportedServices:"); + for(int j=0;j0){ + String [] filters = textFilter.split("\\||\\+| "); + for(int f=0;f supportedServices, JSONArray catalogJSON, XMLElement xmlElement,String variableFilter,String textFilter,XMLElement parent) throws Exception { + + Vector datasets = xmlElement.getList("dataset"); + for(int j=0;j0){ + //Make a folder + JSONObject folder = new JSONObject(); + String name = dataset.getAttrValue("name"); + catalogJSON.put(folder); + folder.put("text", name); + folder.put("catalogurl", rootCatalog); + folder.put("children",c); + folder.put("expanded", true); + folder.put("cls", "folder"); + folder.put("variables",putVariableInfo(dataset)); + //folder.put("variables2",putVariableInfo(xmlElement)); + + if(succeeded==false){ + Debug.errprint("Did not succeed!"); + return false; + } + }else{ + //This node has no childs + //if(dataset.getAttrValue("name").indexOf("tas")!=-1) + { + //Create a leaf by default + JSONObject leaf = new JSONObject(); + if(checkMaxChilds(catalogJSON))return false; + + + + leaf.put("leaf", true); + + //DebugConsole.println("Leaf "+dataset.getAttrValue("name")); + + //Try to find defined serviceName, if not defined pick the first occuring one. + String serviceName=null; + Service service = null; + String supportedServicesString = ""; + + try{ + serviceName = dataset.get("serviceName").getValue(); + //Debug.println("--serviceName:"+serviceName); + service = getServiceByName(serviceName,supportedServices); + }catch(Exception e){ + service = supportedServices.get(0); + }; + //DebugConsole.println("Service="+service.name); + if(service!=null){ + for(int i=0;i0)supportedServicesString+=","; + supportedServicesString+=service.accesTypes.get(i).serviceType; + }catch(Exception e){ + + } + } + } + + //Try to find additional servicenames based on access elements + try{ + Vector access = dataset.getList("access"); + for(int k=0;k0)supportedServicesString+=","; + supportedServicesString+=service.accesTypes.get(i).serviceType; + //Debug.println("--accessServiceName:"+service.accesTypes.get(i).serviceType); + } + } + } + + }catch(Exception e){} + + String nodeName = dataset.getAttrValue("name"); + leaf.put("text", nodeName);// - ("+supportedServicesString+")"); + + String dataSize = "-"; + long fileSize = 0; + try{ + String units = dataset.get("dataSize").getAttrValue("units");; + String dataSizeValue = dataset.get("dataSize").getValue(); + if(units.equals("Tbytes")){units="T";try{fileSize = (long) Double.parseDouble(dataSizeValue)*1024*1024*1024*1024;}catch(Exception e){}} + else if(units.equals("Gbytes")){units="G";try{fileSize = (long) Double.parseDouble(dataSizeValue)*1024*1024*1024;}catch(Exception e){}} + else if(units.equals("Mbytes")){units="M";try{fileSize = (long) Double.parseDouble(dataSizeValue)*1024*1024;}catch(Exception e){}} + else if(units.equals("Kbytes")){units="K";try{fileSize = (long) Double.parseDouble(dataSizeValue)*1024;}catch(Exception e){}} + else try{fileSize = (long) Double.parseDouble(dataSizeValue);}catch(Exception e){} + dataSize = dataSizeValue+""+units; + }catch(Exception e){} + leaf.put("dataSize", dataSize); + leaf.put("fileSize", ""+fileSize); + // if(parent!=null){ + // putVariableInfo(b,parent); + //}else{ + JSONArray variables = putVariableInfo(dataset); + if(variables.length() == 0){ + variables = putVariableInfo(xmlElement); + + } + leaf.put("variables",variables); + //} + + boolean put = true; + + if(variableFilter!=null){ + try{ + if(variableFilter.length()>0){ + put=false; + JSONArray variableList=variables; + if(variableList.length() == 0){ + put = true; + }else{ +// boolean foundSomething=false; + for(int v=0;v catalogRefs = xmlElement.getList("catalogRef"); + + + + for(int j=0;j variables = null; + try{ + variables = dataset.get("variables").getList("variable"); + }catch(Exception e){ + } + for(int j1=0;j1maxNumberOfItems){ + JSONObject b = new JSONObject(); + a.put(b); + b.put("text", "..too many items for catalog browser! Only "+maxNumberOfItems+" items shown ..."); + b.put("cls", "leaf"); + b.put("leaf", true); + return true; + } + return false; + } + + private static Service getServiceByName(String serviceType, Vector supportedServices) { + for(int j=0;j supportedServices ,Vector v){ + for(int j=0;j w=v.get(j).getList("service"); + for(int i=0;i getSupportedServices(XMLElement catalogElement) { + Vector supportedServices = new Vector(); + recursivelyWalkServiceElement(supportedServices,catalogElement.getList("service")); + return supportedServices; + } + +} diff --git a/src/main/java/nl/knmi/adaguc/services/esgfsearch/THREDDSCatalogToHTML.java b/src/main/java/nl/knmi/adaguc/services/esgfsearch/THREDDSCatalogToHTML.java new file mode 100644 index 0000000..3574549 --- /dev/null +++ b/src/main/java/nl/knmi/adaguc/services/esgfsearch/THREDDSCatalogToHTML.java @@ -0,0 +1,373 @@ +package nl.knmi.adaguc.services.esgfsearch; + +import java.io.IOException; +import java.util.Calendar; +import java.util.Vector; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +import nl.knmi.adaguc.security.AuthenticatorFactory; +import nl.knmi.adaguc.security.AuthenticatorInterface; +import nl.knmi.adaguc.security.user.UserManager; +import nl.knmi.adaguc.tools.Debug; +import nl.knmi.adaguc.tools.HTTPTools; +import nl.knmi.adaguc.tools.JSONResponse; +import nl.knmi.adaguc.tools.WebRequestBadStatusException; + + +public class THREDDSCatalogToHTML { + private String getHTTPParamNoExceptionButNull(HttpServletRequest request, String name){ + try { + return HTTPTools.getHTTPParam(request, name); + } catch (Exception e) { + } + return null; + } + class BuildHTMlResult{ + String result; + int rn; + } + + BuildHTMlResult buildHTML(JSONArray array,String root,int oddEven,String openid,int rn){ + if(array == null)return null; + StringBuffer html = new StringBuffer(); + try { + //Try to get the catalogURL + + for(int j=0;j"; + html.append(""+rn+""); + }else{ + //html+=""; + html.append(""+rn+""); + } + if(hrefURL == null){ + // html+=root+nodeText+"
"; + html.append(root+nodeText+"
"); + }else{ + //html+=root+""+nodeText+""; + html.append(root+""+nodeText+""); + } + + // html+=""+dataSize+""; + html.append(""+fileSize+""); + //html.append("");html.append(dataSize);html.append(""); + + String dapLink = ""; + String httpLink = ""; + + /* + * Only show opendap when it is really advertised by the server. + */ if(opendapURL == null && httpserverURL !=null){ + opendapURL = httpserverURL.replace("fileServer", "dodsC")+"#"; + } + + if(opendapURL!=null){ + dapLink="view"; + //dapLink="view"; + } + if(httpserverURL!=null){ + if(openid!=null){ + httpLink="download"; + }else{ + httpLink="download"; + } + } + //html+=""+dapLink+""+httpLink; + html.append("");html.append(dapLink);html.append("");html.append(httpLink); + html.append(""); + if(httpserverURL == null && opendapURL == null && catalogURL == null){ + //html+="-"; + html.append(""); + }else{ + html.append("\n"); + } + + + //html+=""; + html.append(""); + + try{ + JSONArray children = a.getJSONArray("children"); + //html+=buildHTML(children,root+"    ",oddEven); + BuildHTMlResult b=buildHTML(children,root+"    ",1-oddEven,openid,rn); + html.append(b.result); + rn=b.rn; + } catch (JSONException e) { + } + } + } catch (JSONException e) { + } + + BuildHTMlResult b=new BuildHTMlResult(); + b.result=html.toString(); + b.rn=rn; + return b; + } + public void handleCatalogBrowserRequest(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { + Debug.println("SERVICE CATALOGBROWSER: "+request.getQueryString()); + String format= getHTTPParamNoExceptionButNull(request,"format"); + JSONResponse jsonResponse = new JSONResponse(request); + if(format==null)format=""; + boolean flat = false; + try{ + String variableFilter="",textFilter = ""; + try{ + variableFilter = HTTPTools.getHTTPParam(request, "variables"); + }catch(Exception e){ + } + + try{ + textFilter = HTTPTools.getHTTPParam(request, "filter"); + }catch(Exception e){ + } + try{ + String mode= HTTPTools.getHTTPParam(request, "mode"); + if(mode.equals("flat")){ + flat=true; + } + }catch(Exception e){ + } + + + if(variableFilter.length()==0){ + /* Try to obtain variable filter from catalog url */ + try{ + String catalogURL= HTTPTools.getHTTPParam(request, "node"); + String[] s = catalogURL.split("#"); + Debug.println(""+s.length); + if(s.length == 2){ + if(s[1].length()>0) + variableFilter = s[1]; + } + }catch(Exception e){ + + } + + } + + HTTPTools.validateInputTokens(variableFilter); + HTTPTools.validateInputTokens(textFilter); + + Debug.println("Starting CATALOG: "+request.getQueryString()); + long startTimeInMillis = Calendar.getInstance().getTimeInMillis(); + JSONArray treeElements = null; + try{ + treeElements = THREDDSCatalogBrowser.browseThreddsCatalog(request, variableFilter,textFilter); + }catch(Exception e){ + jsonResponse.setException("Unable to read catalog", e); + try { + jsonResponse.print(response); + } catch (Exception e1) { + e1.printStackTrace(); + } + return; + + } + + long stopTimeInMillis = Calendar.getInstance().getTimeInMillis(); + Debug.println("Finished CATALOG: "+" ("+(stopTimeInMillis-startTimeInMillis)+" ms) "+request.getQueryString()); + + if(treeElements == null){ + jsonResponse.setErrorMessage("Unable to read catalog", 200); + try { + jsonResponse.print(response); + } catch (Exception e1) { + e1.printStackTrace(); + } + return; + + } + + + + + if(jsonResponse.hasError()==false){ + if(flat == false){ + + + String html=""; + String catalogURL= HTTPTools.getHTTPParam(request, "node"); + html+="Catalog url: "+catalogURL+""; + /*for(int j=0;j"; + } */ + + try{ + Vector availableVars = new Vector(); + try{ + JSONArray variablesToChoose = treeElements.getJSONObject(0).getJSONArray("variables"); + for(int j=0;j
"; + if(availableVars.size()>0){ + html+="Variables:"; + for(int j=0;j0){ + checked=""; + if(availableVars.get(j).matches(variableFilter))checked="checked=\"yes\""; + } + html+=""+availableVars.get(j); + } + html+="

"; + } + html+="Text filter: "; + + //html+="
"; + //html+="  "; + + html+=""; + + }catch(Exception e){ + e.printStackTrace(); + } + + html+="
"; + html+=""; + html+=""; + html+=""; + html+=""; + html+=""; + html+=""; + html+=""; + html+=""; + + long startTimeInMillis1 = Calendar.getInstance().getTimeInMillis(); + + String openid = null; + try{ + AuthenticatorInterface authenticator = AuthenticatorFactory.getAuthenticator(request); + if(authenticator != null){ + try { + openid = UserManager.getUser(authenticator).getUserId(); + } catch(Exception e){ + Debug.println("No user information provided: " + e.getMessage()); + } + + } + }catch(Exception e){ + } + html+=buildHTML(treeElements,"",0,openid,0).result+"
#Resource titleSizeOpendapDownloadBasket
"; + long stopTimeInMillis1 = Calendar.getInstance().getTimeInMillis(); + Debug.println("Finished building HTML with length "+html.length() +" in ("+(stopTimeInMillis1-startTimeInMillis1)+" ms)"); + if(format.equals("text/html")){ + response.setContentType("text/html"); + response.getWriter().print(html); + }else if(format.equals("application/json")){ + JSONObject a = new JSONObject(); + a.put("html", html); + try{ + jsonResponse.setMessage(a); + } catch(Exception e){ + jsonResponse.setException("Catalogbrowser error:",e); + } + try { + jsonResponse.print(response); + } catch (Exception e1) { + + } + }else{ + try{ + jsonResponse.setMessage(treeElements.toString()); + } catch(Exception e){ + jsonResponse.setException("Catalogbrowser error:",e); + } + try { + jsonResponse.print(response); + } catch (Exception e1) { + + } + } + } + + if(flat == true){ + //Debug.println(treeElements.toString()); + //THREDDSCatalogBrowser.MakeFlat b = new THREDDSCatalogBrowser.MakeFlat(); + JSONArray allFilesFlat = THREDDSCatalogBrowser.makeFlat(treeElements); + JSONObject data = new JSONObject(); + data.put("files",allFilesFlat); + jsonResponse.setMessage(data); + //Debug.println("Found "+allFilesFlat.length()); + + try { + jsonResponse.print(response); + } catch (Exception e1) { + + } + } + Debug.println("Catalog request finished."); + } + }catch(WebRequestBadStatusException e){ + if(format.equals("text/html")){ + response.setContentType("text/html"); + String html=""; + if(e.getStatusCode() == 404){ + html="Catalog not found! (404)"; + }else{ + html=e.getMessage(); + } + response.getWriter().print(html); + }else{ + jsonResponse.setException("error", e); + try {jsonResponse.print(response);} catch (Exception e1) {} + } + + + } catch (Exception e2) { + e2.printStackTrace(); + + if(format.equals("text/html")){ + response.setContentType("text/html"); + String html=""; + html=e2.getMessage(); + html+="
"; + response.getWriter().print(html); + }else{ + jsonResponse.setException("error", e2); + try {jsonResponse.print(response);} catch (Exception e1) {} + } + } + } + +} From 1334ebe0445d5689e29c37e8ea225e16b66b3939 Mon Sep 17 00:00:00 2001 From: Maarten Plieger Date: Fri, 7 Sep 2018 17:02:50 +0200 Subject: [PATCH 5/7] V 1.0.8 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ab38c96..c6679b8 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ nl.knmi.adagucservices adaguc-services - 1.0.3 + 1.0.8 war adaguc-services From 4e5065260830ce60fbc1e0ad27af90efe2eca195 Mon Sep 17 00:00:00 2001 From: Maarten Plieger Date: Fri, 7 Sep 2018 18:55:52 +0200 Subject: [PATCH 6/7] Fixed unit tests --- .../adaguc/services/esgfsearch/ESGFSearchRequestMapper.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/nl/knmi/adaguc/services/esgfsearch/ESGFSearchRequestMapper.java b/src/main/java/nl/knmi/adaguc/services/esgfsearch/ESGFSearchRequestMapper.java index 90b59c9..4058bcc 100644 --- a/src/main/java/nl/knmi/adaguc/services/esgfsearch/ESGFSearchRequestMapper.java +++ b/src/main/java/nl/knmi/adaguc/services/esgfsearch/ESGFSearchRequestMapper.java @@ -34,7 +34,7 @@ public class ESGFSearchRequestMapper implements DisposableBean { public ESGFSearchRequestMapper() throws ElementNotFoundException { super(); - esgfSearch=getESGFSearchInstance(); + } public void destroy() { @@ -68,7 +68,7 @@ public void search(HttpServletResponse response, HttpServletRequest request) thr jsonResponse.setMessage(new JSONObject().put("error", "ADAGUC esgfsearch is not enabled")); }else{ Debug.println("getoverview"); - esgfSearch.doGet(request,response); + getESGFSearchInstance().doGet(request,response); } } catch (Exception e) { jsonResponse.setException("error: "+e.getMessage(), e); From 37b765a64d83572ee7ea66a97db97bc902b9fb70 Mon Sep 17 00:00:00 2001 From: Maarten Plieger Date: Thu, 27 Sep 2018 17:57:20 +0200 Subject: [PATCH 7/7] [AUTOWMS] Extra extensions are now supported: png, h5, hdf5, nc, nc4, geojson --- .settings/org.eclipse.wst.common.component | 2 +- .../adaguc/services/autowms/AutoWMSRequestMapper.java | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.settings/org.eclipse.wst.common.component b/.settings/org.eclipse.wst.common.component index 3e8bd1b..e027834 100644 --- a/.settings/org.eclipse.wst.common.component +++ b/.settings/org.eclipse.wst.common.component @@ -1,5 +1,5 @@ - + diff --git a/src/main/java/nl/knmi/adaguc/services/autowms/AutoWMSRequestMapper.java b/src/main/java/nl/knmi/adaguc/services/autowms/AutoWMSRequestMapper.java index c8ce85a..364939e 100644 --- a/src/main/java/nl/knmi/adaguc/services/autowms/AutoWMSRequestMapper.java +++ b/src/main/java/nl/knmi/adaguc/services/autowms/AutoWMSRequestMapper.java @@ -86,7 +86,13 @@ public void autowms(HttpServletResponse response, HttpServletRequest request) th if( directory.exists() ) { File[] files = directory.listFiles(); for( File file : files){ - if(file.isDirectory() || file.getName().endsWith(".nc") || file.getName().endsWith(".geojson") + if(file.isDirectory() || + file.getName().endsWith(".png") || + file.getName().endsWith(".h5") || + file.getName().endsWith(".hdf5") || + file.getName().endsWith(".nc") || + file.getName().endsWith(".nc4") || + file.getName().endsWith(".geojson") ){ String filePath = file.getAbsolutePath().substring(basePath.length()+1); fileList.put(