From d1c505bccd8a8960a333a878ec512874bf09e634 Mon Sep 17 00:00:00 2001 From: Kevin McGoldrick Date: Sat, 11 Apr 2020 09:18:51 -0700 Subject: [PATCH] Release 2.3.4 * Fix agent debugger * Security to SessionScoped * Minimum single thread for debugger * Validate position of list before every use to avoid IndexOutOfBoundsException after a item removal * AWSXRay: setUser to currentSegment at filter * update README.MD with badges * Fix kill instances across regions * filter instances by present in region before terminate/stop. * update libraries versions * java streams * requestInstances re-write * AgentStartup retry loop * backoff Strategy for supportFiles download * NonNull TPS results return * JSF to 2.3 * CloudWatchMetrics * Remove SimpleDB DataSource * Add CloudWatch Metrics DataSource * FIBONACCI retry backoff strategy --- agent/agent_common/pom.xml | 4 +- .../com/intuit/tank/http/TankHttpUtil.java | 94 ++-- .../tank/http/xml/GenericXMLHandler.java | 24 +- .../intuit/tank/http/TankHttpUtilTest.java | 7 + .../tank/http/xml/GenericXMLHandlerTest.java | 2 +- agent/agent_standalone/pom.xml | 2 +- agent/agent_standalone_pkg/pom.xml | 2 +- agent/agent_startup/pom.xml | 2 +- .../com/intuit/tank/agent/AgentStartup.java | 41 +- agent/agent_startup_pkg/pom.xml | 2 +- agent/apiharness/pom.xml | 2 +- .../intuit/tank/harness/APITestHarness.java | 95 ++-- .../tank/harness/TestPlanSingleton.java | 47 +- .../intuit/tank/harness/TestPlanStarter.java | 2 +- .../tank/runner/method/RequestRunner.java | 4 +- .../java/com/intuit/tank/common/TPSTest.java | 2 +- agent/apiharness_pkg/pom.xml | 2 +- agent/http_client_3/pom.xml | 2 +- agent/http_client_4/pom.xml | 2 +- agent/http_client_5/pom.xml | 2 +- .../tank/httpclient5/TankHttpClient5.java | 4 +- agent/pom.xml | 2 +- api/pom.xml | 2 +- data_access/pom.xml | 2 +- data_model/pom.xml | 2 +- doc/doc_xslt/pom.xml | 2 +- .../main/resources/xslt/org/tank/eclipse.xsl | 2 +- .../src/main/resources/xslt/org/tank/pdf.xsl | 2 +- .../resources/xslt/org/tank/xhtml-single.xsl | 2 +- .../main/resources/xslt/org/tank/xhtml.xsl | 2 +- doc/jdocbook_style/pom.xml | 2 +- doc/pom.xml | 2 +- doc/tank_installation_guide/pom.xml | 2 +- doc/tank_user_guide/pom.xml | 2 +- harness_data/pom.xml | 2 +- mail/pom.xml | 2 +- pom.xml | 99 ++--- proxy-parent/WebConversation/pom.xml | 2 +- proxy-parent/owasp-proxy/pom.xml | 2 +- .../java/org/owasp/proxy/util/Base64.java | 4 +- proxy-parent/pom.xml | 2 +- proxy-parent/proxy-extension/pom.xml | 2 +- proxy-parent/proxy_pkg/pom.xml | 2 +- readme.md | 13 +- reporting/api/pom.xml | 2 +- .../tank/reporting/databases/IDatabase.java | 5 +- reporting/db/pom.xml | 28 +- .../databases/AmazonDynamoDatabaseDocApi.java | 22 +- .../databases/AmazonSimpleDatabase.java | 411 ------------------ .../databases/CloudWatchDataSource.java | 150 +++++++ .../databases/DataBaseFactory.java | 21 +- .../databases/GraphiteDatasource.java | 10 +- .../persistence/databases/S3Datasource.java | 198 ++++----- .../databases/WavefrontDatasource.java | 10 +- .../reporting/db/DatabaseResultsReader.java | 1 - .../reporting/db/DatabaseResultsReporter.java | 15 +- .../databases/AmazonDynamoDatabaseTest.java | 19 +- .../databases/AmazonSimpleDBTest.java | 35 -- reporting/local/pom.xml | 2 +- reporting/pom.xml | 2 +- reporting/rest/pom.xml | 2 +- rest/api/agent/pom.xml | 2 +- rest/api/automation/pom.xml | 2 +- rest/api/cloud/pom.xml | 2 +- rest/api/common/pom.xml | 2 +- rest/api/datafile/pom.xml | 2 +- rest/api/filter/pom.xml | 2 +- rest/api/job/pom.xml | 2 +- rest/api/pom.xml | 2 +- rest/api/project/pom.xml | 2 +- rest/api/reporting/pom.xml | 2 +- rest/api/script/pom.xml | 2 +- rest/api/user/pom.xml | 2 +- rest/client/agent/pom.xml | 2 +- rest/client/automation/pom.xml | 2 +- rest/client/cloud/pom.xml | 2 +- rest/client/common/pom.xml | 2 +- rest/client/datafile/pom.xml | 2 +- rest/client/filter/pom.xml | 2 +- rest/client/job/pom.xml | 2 +- rest/client/pom.xml | 2 +- rest/client/project/pom.xml | 2 +- rest/client/reporting/pom.xml | 2 +- rest/client/script/pom.xml | 2 +- rest/client/user/pom.xml | 2 +- rest/pom.xml | 2 +- rest/service/agent/pom.xml | 2 +- rest/service/automation/pom.xml | 2 +- rest/service/cloud/pom.xml | 2 +- rest/service/common/pom.xml | 2 +- rest/service/datafile/pom.xml | 2 +- rest/service/filter/pom.xml | 2 +- rest/service/job/pom.xml | 2 +- rest/service/pom.xml | 2 +- rest/service/project/pom.xml | 2 +- rest/service/reporting/pom.xml | 2 +- rest/service/script/pom.xml | 2 +- rest/service/user/pom.xml | 2 +- script_processor/pom.xml | 4 +- search/document_util/pom.xml | 2 +- .../tank/search/util/SearchConstants.java | 2 +- .../tank/search/util/TankAnalyzerTest.java | 4 +- search/lucene_indexer/pom.xml | 2 +- search/pom.xml | 2 +- search/script_search/pom.xml | 2 +- tank_common/pom.xml | 2 +- tank_vmManager/pom.xml | 2 +- .../intuit/tank/vmManager/VMTrackerImpl.java | 5 +- .../vmManager/environment/JobRequest.java | 2 +- .../vmManager/environment/VMChannelImpl.java | 9 +- .../environment/amazon/AmazonInstance.java | 155 +++---- test_support/pom.xml | 2 +- tools/agent_debugger/pom.xml | 2 +- .../tank/tools/debugger/ActionProducer.java | 5 +- .../tools/debugger/AgentDebuggerFrame.java | 5 +- tools/agent_debugger_pkg/pom.xml | 2 +- tools/jenkins_plugin/pom.xml | 2 +- tools/pom.xml | 2 +- tools/script_engine/pom.xml | 2 +- tools/script_filter/pom.xml | 2 +- tools/script_filter_pkg/pom.xml | 2 +- web/pom.xml | 2 +- web/web_support/pom.xml | 10 +- .../java/com/intuit/tank/ProjectBean.java | 1 - .../intuit/tank/auth/InternalSecurity.java | 10 +- .../com/intuit/tank/config/LoginFilter.java | 22 +- .../com/intuit/tank/prefs/TableViewState.java | 12 +- .../intuit/tank/project/FileUploadBean.java | 4 +- .../intuit/tank/project/JobTreeTableBean.java | 10 - .../intuit/tank/script/LogicStepEditor.java | 1 - .../com/intuit/tank/script/ScriptBean.java | 1 - .../tank/script/ScriptCreationBean.java | 4 +- .../com/intuit/tank/script/ScriptEditor.java | 1 - .../intuit/tank/script/TankXmlUploadBean.java | 20 +- .../intuit/tank/script/TestUploadBean.java | 4 +- .../tank/util/UploadedFileIterator.java | 4 +- .../com/intuit/tank/util/UserNameFilter.java | 12 +- .../intuit/tank/prefs/TableViewStateTest.java | 3 +- .../tank/script/ScriptCreationBeanTest.java | 34 +- .../tank/script/TankXmlUploadBeanTest.java | 12 - .../tank/script/TestUploadBeanTest.java | 10 +- web/web_ui/pom.xml | 4 +- .../webapp/WEB-INF/.faces-config.xml.jsfdia | 286 ------------ web/web_ui/src/main/webapp/WEB-INF/web.xml | 67 ++- web/web_ui/src/main/webapp/admin/logs.xhtml | 12 +- .../src/main/webapp/datafiles/index.xhtml | 2 +- .../main/webapp/projects/associateFiles.xhtml | 2 +- .../src/main/webapp/projects/index.xhtml | 5 +- .../src/main/webapp/scripts/index.xhtml | 17 +- .../webapp/scripts/script-edit-view.xhtml | 10 +- 150 files changed, 813 insertions(+), 1495 deletions(-) delete mode 100644 reporting/db/src/main/java/com/intuit/tank/persistence/databases/AmazonSimpleDatabase.java create mode 100644 reporting/db/src/main/java/com/intuit/tank/persistence/databases/CloudWatchDataSource.java delete mode 100644 reporting/db/src/test/java/com/intuit/tank/persistence/databases/AmazonSimpleDBTest.java delete mode 100644 web/web_ui/src/main/webapp/WEB-INF/.faces-config.xml.jsfdia diff --git a/agent/agent_common/pom.xml b/agent/agent_common/pom.xml index f79aba4e2..2cdf1e141 100644 --- a/agent/agent_common/pom.xml +++ b/agent/agent_common/pom.xml @@ -5,7 +5,7 @@ com.intuit.tank agent-parent - 2.3.3 + 2.3.4 agent-common @@ -33,7 +33,7 @@ - jdom + org.jdom jdom diff --git a/agent/agent_common/src/main/java/com/intuit/tank/http/TankHttpUtil.java b/agent/agent_common/src/main/java/com/intuit/tank/http/TankHttpUtil.java index cae1cf3e7..6221b64f5 100644 --- a/agent/agent_common/src/main/java/com/intuit/tank/http/TankHttpUtil.java +++ b/agent/agent_common/src/main/java/com/intuit/tank/http/TankHttpUtil.java @@ -2,6 +2,7 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; +import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; import java.net.URL; import java.net.URLEncoder; @@ -10,7 +11,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import org.apache.commons.codec.binary.Base64; import org.apache.commons.fileupload.MultipartStream; @@ -22,76 +22,48 @@ import com.intuit.tank.http.json.JsonResponse; import com.intuit.tank.http.xml.XMLResponse; +import static java.util.stream.Collectors.joining; + /** * utitly methods for tank http clients * @author denisa * */ public class TankHttpUtil { - private static Logger LOG = LogManager.getLogger(TankHttpUtil.class); public static URL buildUrl(String protocol, String host, int port, String path, Map urlVariables) { - try { - // no default port specified for http - if (protocol.equalsIgnoreCase("http") && port == -1) { - return new URL(protocol, host, path + getQueryString(urlVariables)); - } else if (protocol.equalsIgnoreCase("https") && port == -1) { - return new URL(protocol, host, path + getQueryString(urlVariables)); - } // ensure that port 80 and 8080 requests use http and not https if (port == 80 || port == 8080) { protocol = "http"; } - return new URL(protocol, host, port, path + getQueryString(urlVariables)); + // no default port specified for http + return (port == -1) ? + new URL(protocol, host, path + getQueryString(urlVariables)) : + new URL(protocol, host, port, path + getQueryString(urlVariables)); } catch (MalformedURLException e) { throw new RuntimeException(e); } } public static String getQueryString(Map urlVariables) { - - StringBuilder queryString = new StringBuilder(); - - // Set the query string - if (urlVariables != null) { - if (!urlVariables.isEmpty()) { - - queryString.append("?"); - - // Set> set = urlVariables.entrySet(); - // Iterator> iterator = - // set.iterator(); - for (Entry entry : urlVariables.entrySet()) { - try { - StringBuilder nvp = new StringBuilder(); - nvp.append(URLEncoder.encode(entry.getKey(), StandardCharsets.UTF_8.toString())); - if (entry.getValue() != null) { - nvp.append("="); - nvp.append(URLEncoder.encode(entry.getValue(), StandardCharsets.UTF_8.toString())); + if (urlVariables != null && !urlVariables.isEmpty()) { + return "?" + urlVariables.entrySet().stream() + .map(entry -> { + try { + return URLEncoder.encode(entry.getKey(), StandardCharsets.UTF_8.name()) + + "=" + + URLEncoder.encode(entry.getValue(), StandardCharsets.UTF_8.name()); + } catch (UnsupportedEncodingException ex) { + LOG.warn("Unable to set query string value: " + ex.getMessage()); } - nvp.append("&"); - queryString.append(nvp.toString()); - - } catch (Exception ex) { - LOG.warn("Unable to set query string value: " + ex.getMessage()); - } - } - } - } - - // Remove the last & - String reqQueryString = ""; - if (queryString.length() > 0) { - if (queryString.charAt(queryString.length() - 1) == '&') - reqQueryString = queryString.deleteCharAt(queryString.length() - 1).toString(); - else - reqQueryString = queryString.toString(); + return ""; + }) + .collect(joining("&")); } - - return reqQueryString; + return ""; } /** @@ -101,13 +73,11 @@ public static String getQueryString(Map urlVariables) { */ public static BaseResponse newResponseObject(String contentTypeHeader) { String contentType = StringUtils.isNotBlank(contentTypeHeader) ? contentTypeHeader : ""; - if (contentType.contains("xml")) { - return new XMLResponse(); - } else if (contentType.contains("json")) { - return new JsonResponse(); - } else { - return new BinaryResponse(); - } + return contentType.contains("xml") ? + new XMLResponse() : + contentType.contains("json") ? + new JsonResponse() : + new BinaryResponse(); } public static List getPartsFromBody(BaseRequest request) { @@ -196,20 +166,18 @@ public String getFileName() { // filename="diamond-sword.png" public String getContentType() { String ct = headerMap.get("Content-Type"); - if (ct == null) { - ct = "text/plain"; - } - return ct; + return ct == null ? + "text/plain" : + ct; } // Content-Disposition: form-data; name="uploadname1"; // filename="diamond-sword.png" public String getContentDisposition() { String ct = headerMap.get("Content-Disposition"); - if (ct == null) { - ct = "form-data"; - } - return ct; + return ct == null ? + "form-data" : + ct; } /** diff --git a/agent/agent_common/src/main/java/com/intuit/tank/http/xml/GenericXMLHandler.java b/agent/agent_common/src/main/java/com/intuit/tank/http/xml/GenericXMLHandler.java index 1c78ebb9a..554e95867 100644 --- a/agent/agent_common/src/main/java/com/intuit/tank/http/xml/GenericXMLHandler.java +++ b/agent/agent_common/src/main/java/com/intuit/tank/http/xml/GenericXMLHandler.java @@ -27,14 +27,14 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.jdom.Attribute; -import org.jdom.Document; -import org.jdom.Element; -import org.jdom.JDOMException; -import org.jdom.Namespace; -import org.jdom.input.SAXBuilder; -import org.jdom.output.XMLOutputter; -import org.jdom.xpath.XPath; +import org.jdom2.Attribute; +import org.jdom2.Document; +import org.jdom2.Element; +import org.jdom2.JDOMException; +import org.jdom2.Namespace; +import org.jdom2.input.SAXBuilder; +import org.jdom2.output.XMLOutputter; +import org.jdom2.xpath.XPath; import org.xml.sax.InputSource; /** @@ -70,7 +70,7 @@ public GenericXMLHandler() { public GenericXMLHandler(File xmlFile) { try { this.xmlFile = xmlFile; - this.xmlDocument = new org.jdom.Document(); + this.xmlDocument = new org.jdom2.Document(); SAXBuilder builder = new SAXBuilder(); builder.setValidation(false); this.xmlDocument = builder.build(this.xmlFile); @@ -92,7 +92,7 @@ public GenericXMLHandler(String xmlFile) { this.xml = xmlFile; try { this.xmlFile = null; - this.xmlDocument = new org.jdom.Document(); + this.xmlDocument = new org.jdom2.Document(); SAXBuilder builder = new SAXBuilder(); builder.setValidation(false); // LOG.debug("XML string to load: "+xmlFile); @@ -155,7 +155,7 @@ private Element SetElementText(String xPathExpression, int currentNode) throws J if (xPathExists(currentPath)) { if (currentPath.equals(xPathExpression)) { - return (org.jdom.Element) XPath.selectSingleNode(this.xmlDocument, xPathExpression); + return (org.jdom2.Element) XPath.selectSingleNode(this.xmlDocument, xPathExpression); } else { return SetElementText(xPathExpression, currentNode + 1); @@ -225,7 +225,7 @@ public String GetElementText(String xPathExpression) { */ public String GetElementAttr(String xPathExpression) { try { - org.jdom.Attribute node = (Attribute) XPath.selectSingleNode(this.xmlDocument, xPathExpression); + org.jdom2.Attribute node = (Attribute) XPath.selectSingleNode(this.xmlDocument, xPathExpression); return node.getValue(); } catch (Exception ex) { LOG.error("Error in handler: " + ex.getMessage(), ex); diff --git a/agent/agent_common/src/test/java/com/intuit/tank/http/TankHttpUtilTest.java b/agent/agent_common/src/test/java/com/intuit/tank/http/TankHttpUtilTest.java index f221f9088..085442821 100644 --- a/agent/agent_common/src/test/java/com/intuit/tank/http/TankHttpUtilTest.java +++ b/agent/agent_common/src/test/java/com/intuit/tank/http/TankHttpUtilTest.java @@ -1,5 +1,6 @@ package com.intuit.tank.http; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; /* @@ -146,9 +147,13 @@ public void testGetQueryString_1() public void testGetQueryString_2() throws Exception { Map urlVariables = new HashMap(); + urlVariables.put("val1", "val1"); + urlVariables.put("val2", "val2"); + urlVariables.put("val3", "val3"); String result = TankHttpUtil.getQueryString(urlVariables); assertNotNull(result); + assertEquals( "?val3=val3&val2=val2&val1=val1", result); } /** @@ -162,9 +167,11 @@ public void testGetQueryString_2() public void testGetQueryString_3() throws Exception { Map urlVariables = new HashMap(); + urlVariables.put("data", "{\"val1\":\"val1\", \"val2\":\"val2\", \"val3\":\"val3\"}"); String result = TankHttpUtil.getQueryString(urlVariables); assertNotNull(result); + assertEquals( "?data=%7B%22val1%22%3A%22val1%22%2C+%22val2%22%3A%22val2%22%2C+%22val3%22%3A%22val3%22%7D", result); } /** diff --git a/agent/agent_common/src/test/java/com/intuit/tank/http/xml/GenericXMLHandlerTest.java b/agent/agent_common/src/test/java/com/intuit/tank/http/xml/GenericXMLHandlerTest.java index 9ccb8dc2b..4b7350acb 100644 --- a/agent/agent_common/src/test/java/com/intuit/tank/http/xml/GenericXMLHandlerTest.java +++ b/agent/agent_common/src/test/java/com/intuit/tank/http/xml/GenericXMLHandlerTest.java @@ -16,7 +16,7 @@ import java.io.File; import java.util.HashMap; -import org.jdom.Document; +import org.jdom2.Document; import org.junit.jupiter.api.*; import static org.junit.jupiter.api.Assertions.*; diff --git a/agent/agent_standalone/pom.xml b/agent/agent_standalone/pom.xml index b09ec6773..93a5cd7ab 100755 --- a/agent/agent_standalone/pom.xml +++ b/agent/agent_standalone/pom.xml @@ -5,7 +5,7 @@ com.intuit.tank agent-parent - 2.3.3 + 2.3.4 agent-standalone diff --git a/agent/agent_standalone_pkg/pom.xml b/agent/agent_standalone_pkg/pom.xml index 352ab0ad3..5fe5ffc32 100755 --- a/agent/agent_standalone_pkg/pom.xml +++ b/agent/agent_standalone_pkg/pom.xml @@ -5,7 +5,7 @@ com.intuit.tank agent-parent - 2.3.3 + 2.3.4 agent-standalone-pkg diff --git a/agent/agent_startup/pom.xml b/agent/agent_startup/pom.xml index 19dd42c97..aaf661531 100755 --- a/agent/agent_startup/pom.xml +++ b/agent/agent_startup/pom.xml @@ -5,7 +5,7 @@ com.intuit.tank agent-parent - 2.3.3 + 2.3.4 agent-startup diff --git a/agent/agent_startup/src/main/java/com/intuit/tank/agent/AgentStartup.java b/agent/agent_startup/src/main/java/com/intuit/tank/agent/AgentStartup.java index 4601e84a4..07737c472 100644 --- a/agent/agent_startup/src/main/java/com/intuit/tank/agent/AgentStartup.java +++ b/agent/agent_startup/src/main/java/com/intuit/tank/agent/AgentStartup.java @@ -13,12 +13,15 @@ * #L% */ +import java.io.EOFException; import java.io.File; import java.io.FileOutputStream; import java.io.InputStream; +import java.net.ConnectException; import java.net.URL; import java.nio.charset.StandardCharsets; import java.util.zip.ZipEntry; +import java.util.zip.ZipException; import java.util.zip.ZipInputStream; import com.amazonaws.util.StringUtils; @@ -31,12 +34,12 @@ import com.intuit.tank.vm.common.TankConstants; public class AgentStartup implements Runnable { - private static Logger logger = LogManager.getLogger(AgentStartup.class); private static final String SERVICE_RELATIVE_PATH = "/rest/v1/agent-service"; private static final String METHOD_SETTINGS = "/settings"; private static final String API_HARNESS_COMMAND = "./startAgent.sh"; private static final String METHOD_SUPPORT = "/supportFiles"; + private static final int[] FIBONACCI = new int[] { 1, 1, 2, 3, 5, 8, 13 }; private final String controllerBaseUrl; @@ -55,25 +58,39 @@ public void run() { String settings = IOUtils.toString(settingsStream, StandardCharsets.UTF_8); FileUtils.writeStringToFile(new File("settings.xml"), settings, StandardCharsets.UTF_8); logger.info("got settings file..."); + } catch (ConnectException ce) { + logger.error("Error creating connection to " + + controllerBaseUrl + " : this is normal during the bake : " + ce.getMessage()); } + // Download Support Files url = new URL(controllerBaseUrl + SERVICE_RELATIVE_PATH + METHOD_SUPPORT); logger.info("Making call to tank service url to get support files " + url.toExternalForm()); - try ( ZipInputStream zip = new ZipInputStream(url.openStream()) ){ - ZipEntry entry = zip.getNextEntry(); - while (entry != null) { - String name = entry.getName(); - logger.info("Got file from controller: " + name); - File f = new File(name); - try ( FileOutputStream fout = FileUtils.openOutputStream(f) ) { - IOUtils.copy(zip, fout); + int retryCount = 0; + while (true) { + try (ZipInputStream zip = new ZipInputStream(url.openStream())) { + ZipEntry entry = zip.getNextEntry(); + while (entry != null) { + String name = entry.getName(); + logger.info("Got file from controller: " + name); + File f = new File(name); + try (FileOutputStream fout = FileUtils.openOutputStream(f)) { + IOUtils.copy(zip, fout); + } + entry = zip.getNextEntry(); } - entry = zip.getNextEntry(); + break; + } catch (EOFException | ZipException e) { + logger.error("Error unzipping support files : retryCount=" + + retryCount + " : " + e.getMessage()); + if (retryCount < FIBONACCI.length) { + Thread.sleep( FIBONACCI[++retryCount] * 1000 ); + } else throw e; } } // now start the harness String jvmArgs = AmazonUtil.getUserDataAsMap().get(TankConstants.KEY_JVM_ARGS); - logger.info("Starting apiharness with command: " + API_HARNESS_COMMAND + " -http=" + controllerBaseUrl + " " - + jvmArgs); + logger.info("Starting apiharness with command: " + + API_HARNESS_COMMAND + " -http=" + controllerBaseUrl + " " + jvmArgs); Runtime.getRuntime().exec(API_HARNESS_COMMAND + " -http=" + controllerBaseUrl + " " + jvmArgs); } catch (Exception e) { logger.error("Error in AgentStartup " + e, e); diff --git a/agent/agent_startup_pkg/pom.xml b/agent/agent_startup_pkg/pom.xml index 11f6bd28b..9d5995e6c 100755 --- a/agent/agent_startup_pkg/pom.xml +++ b/agent/agent_startup_pkg/pom.xml @@ -5,7 +5,7 @@ com.intuit.tank agent-parent - 2.3.3 + 2.3.4 agent-startup-pkg diff --git a/agent/apiharness/pom.xml b/agent/apiharness/pom.xml index 57aebfb58..93889de75 100644 --- a/agent/apiharness/pom.xml +++ b/agent/apiharness/pom.xml @@ -5,7 +5,7 @@ com.intuit.tank agent-parent - 2.3.3 + 2.3.4 agent diff --git a/agent/apiharness/src/main/java/com/intuit/tank/harness/APITestHarness.java b/agent/apiharness/src/main/java/com/intuit/tank/harness/APITestHarness.java index 61ce0f79f..01b3b4520 100644 --- a/agent/apiharness/src/main/java/com/intuit/tank/harness/APITestHarness.java +++ b/agent/apiharness/src/main/java/com/intuit/tank/harness/APITestHarness.java @@ -21,7 +21,7 @@ import java.text.DateFormat; import java.util.ArrayList; import java.util.Arrays; -import java.util.Calendar; +import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.List; @@ -35,6 +35,7 @@ import com.intuit.tank.http.TankHttpClient; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.time.DateUtils; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -69,20 +70,17 @@ import org.apache.logging.log4j.message.ObjectMessage; public class APITestHarness { - private static Logger LOG = LogManager.getLogger(APITestHarness.class); + + private static final int[] FIBONACCI = new int[] { 1, 1, 2, 3, 5, 8, 13 }; public static final int POLL_INTERVAL = 15000; - private static final int RETRY_SLEEP = 2000; - private static final int MAX_RETRIES = 10; private static APITestHarness instance; private AgentRunData agentRunData; - private static final int BATCH_SIZE = 100; private String testPlans = ""; private String instanceId; - private List testPlanXmls = null; private ArrayList threadGroupArray = new ArrayList<>(); private int currentNumThreads = 0; private long startTime = 0; @@ -107,9 +105,8 @@ public class APITestHarness { private ResultsReporter resultsReporter; private String tankHttpClientClass; - private Calendar c = Calendar.getInstance(); private Date send = new Date(); - private int interval = 15; // SECONDS + private static final int interval = 15; // SECONDS /** * @@ -235,11 +232,7 @@ private void initializeFromArgs(String[] args) { } else { resultsReporter = new DummyResultsReporter(); TestPlanSingleton plans = TestPlanSingleton.getInstance(); - if (null == testPlanXmls) { - plans.setTestPlans(testPlans); - } else { - plans.setTestPlans(testPlanXmls); - } + plans.setTestPlans(testPlans); runConcurrentTestPlans(); } } @@ -281,19 +274,16 @@ private void startHttp(String baseUrl) { } AgentServiceClient client = new AgentServiceClient(baseUrl); String instanceUrl = null; - int tries = 0; + int retryCount = 0; while (instanceUrl == null) { try { instanceUrl = "http://" + AmazonUtil.getPublicHostName() + ":" + tankConfig.getAgentConfig().getAgentPort(); - } catch (IOException e1) { - tries++; - if (tries < 10) { + } catch (IOException e) { + if (retryCount < FIBONACCI.length) { try { - Thread.sleep(200); - } catch (InterruptedException e) { - // ignore - } + Thread.sleep(FIBONACCI[++retryCount] * 100); + } catch ( InterruptedException ie) { /*Ignore*/ } } else { LOG.error("Error getting amazon host. maybe local."); String publicIp = hostInfo.getPublicIp(); @@ -375,9 +365,8 @@ private void startHttp(String baseUrl) { public void writeXmlToFile(String scriptUrl) throws IOException { File file = new File("script.xml"); LOG.info(new ObjectMessage(ImmutableMap.of("Message", "Writing xml to " + file.getAbsolutePath()))); - int count = 0; - - while (count++ < MAX_RETRIES) { + int retryCount = 0; + while (true) { try { if (file.exists()) { file.delete(); @@ -387,21 +376,16 @@ public void writeXmlToFile(String scriptUrl) throws IOException { LOG.info(new ObjectMessage(ImmutableMap.of("Message", "Downloading file from url " + scriptUrl + " to file " + file.getAbsolutePath()))); FileUtils.copyURLToFile(url, file); String scriptXML = FileUtils.readFileToString(file, "UTF-8"); - List tps = new ArrayList(); - tps.add(scriptXML); - setTestPlans(tps); - TestPlanSingleton.getInstance().setTestPlans(tps); + TestPlanSingleton.getInstance().setTestPlans(Collections.singletonList(scriptXML)); break; } catch (Exception e) { - if (count < MAX_RETRIES) { - LOG.warn(LogUtil.getLogMessage("Failed to download script file because of: " + e.toString() - + ". Will try " - + (MAX_RETRIES - count) + " more times.", LogEventType.System)); + LOG.warn(LogUtil.getLogMessage("Failed to download script file because of: " + e.toString() + + ". Will try " + + (FIBONACCI.length - retryCount) + " more times.", LogEventType.System)); + if (retryCount < FIBONACCI.length) { try { - Thread.sleep(RETRY_SLEEP); - } catch (InterruptedException e1) { - // ignore - } + Thread.sleep(FIBONACCI[++retryCount] * 1000); + } catch ( InterruptedException ie) { /*Ignore*/ } } else { LOG.error(LogUtil.getLogMessage("Error writing script file: " + e, LogEventType.IO), e); throw new RuntimeException(e); @@ -419,8 +403,8 @@ private void saveDataFile(DataFileRequest dataFileRequest) { } } File dataFile = new File(dataFileDir, dataFileRequest.getFileName()); - int count = 0; - while (count++ < MAX_RETRIES) { + int retryCount = 0; + while (true) { try { URL url = new URL(dataFileRequest.getFileUrl()); LOG.info(new ObjectMessage(ImmutableMap.of("Message", @@ -437,15 +421,13 @@ private void saveDataFile(DataFileRequest dataFileRequest) { } break; } catch (Exception e) { - if (count < MAX_RETRIES) { - LOG.warn(new ObjectMessage(ImmutableMap.of("Message", "Failed to download CSV file because of: " + e.toString() - + ". Will try " - + (MAX_RETRIES - count) + " more times."))); + LOG.warn(new ObjectMessage(ImmutableMap.of("Message", "Failed to download CSV file because of: " + e.toString() + + ". Will try " + + (FIBONACCI.length - retryCount) + " more times."))); + if (retryCount < FIBONACCI.length) { try { - Thread.sleep(RETRY_SLEEP); - } catch (InterruptedException e1) { - // ignore - } + Thread.sleep(FIBONACCI[++retryCount] * 1000); + } catch ( InterruptedException ie) { /*Ignore*/ } } else { LOG.error(LogUtil.getLogMessage("Error downloading csv file: " + e, LogEventType.IO), e); throw new RuntimeException(e); @@ -507,7 +489,7 @@ public UserTracker getUserTracker() { */ public void runConcurrentTestPlans() { if (started) { - LOG.warn("Agent already started. Ignoring start command"); + LOG.warn(new ObjectMessage(ImmutableMap.of("Message", "Agent already started. Ignoring start command"))); return; } tpsMonitor = new TPSMonitor(tankConfig.getAgentConfig().getTPSPeriod()); @@ -558,7 +540,7 @@ public void runConcurrentTestPlans() { monitorThread.start(); } - LOG.info(LogUtil.getLogMessage("Starting threads...")); + LOG.info(new ObjectMessage(ImmutableMap.of("Message", "Starting threads..."))); // start initial users startTime = System.currentTimeMillis(); DateFormat df = DateFormat.getDateTimeInstance(DateFormat.SHORT, DateFormat.MEDIUM); @@ -603,7 +585,7 @@ public void runConcurrentTestPlans() { doneSignal.await(); } } catch (Throwable t) { - LOG.error("error executing..." + t, t); + LOG.error(new ObjectMessage(ImmutableMap.of("Message", "error executing..." + t)),t); } finally { LOG.info(new ObjectMessage(ImmutableMap.of("Message", "Test Complete..."))); if (!isDebug()) { @@ -685,13 +667,6 @@ public synchronized void threadStarted(Thread thread) { currentUsers++; } - public void setTestPlans(List scripts) { - testPlanXmls = scripts; - for (String script : scripts) { - LOG.debug(script); - } - } - public boolean isDebug() { return DEBUG; } @@ -709,7 +684,7 @@ public long getSimulationEndTimeMillis() { public boolean hasMetSimulationTime() { if (agentRunData.getSimulationTime() > 0) { - if (System.currentTimeMillis() > getSimulationEndTimeMillis()) { + if (System.currentTimeMillis() > getSimulationEndTimeMillis()) { if (!loggedSimTime) { LOG.info(new ObjectMessage(ImmutableMap.of("Message", "Simulation time met"))); loggedSimTime = true; @@ -788,13 +763,9 @@ public WatsAgentCommand getCmd() { public void queueTimingResult(TankResult result) { if (logTiming) { results.add(result); - //if (results.size() >= BATCH_SIZE) { if (send.before(new Date())) { sendBatchToDB(true); - - c.setTime(new Date()); - c.add(Calendar.SECOND, interval); - send = new Date(c.getTime().getTime()); + send = DateUtils.addSeconds(new Date(), interval); } } } diff --git a/agent/apiharness/src/main/java/com/intuit/tank/harness/TestPlanSingleton.java b/agent/apiharness/src/main/java/com/intuit/tank/harness/TestPlanSingleton.java index e3fab4c5c..dc802d325 100644 --- a/agent/apiharness/src/main/java/com/intuit/tank/harness/TestPlanSingleton.java +++ b/agent/apiharness/src/main/java/com/intuit/tank/harness/TestPlanSingleton.java @@ -15,7 +15,10 @@ import java.io.File; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -44,36 +47,28 @@ private TestPlanSingleton() { public void setTestPlans(String plans) { - String[] testPlanLists = plans.split(","); + List testPlanLists = Arrays.asList(plans.split(",")); - for (String testPlanList : testPlanLists) { - try { - File xmlFile = new File(testPlanList); - if (!xmlFile.exists()) { - throw new Exception("File not found"); - } - - WorkloadParser parser = new WorkloadParser(xmlFile); - HDWorkload workload = parser.getWorkload(); - if (workload != null) { - workloads.add(workload); - } - } catch (Exception e) { - LOG.error(LogUtil.getLogMessage(e.getMessage(), LogEventType.System), e); - throw new RuntimeException(e); - } - } + workloads = testPlanLists.stream() + .filter(xmlFile -> { + if (new File(xmlFile).exists()) { + return true; + } else { + Exception e = new Exception("File not found"); + LOG.error(LogUtil.getLogMessage(e.getMessage(), LogEventType.System), e); + return false; + } + }) + .map(xml -> new WorkloadParser(xml).getWorkload()) + .filter(Objects::nonNull) + .collect(Collectors.toList()); } public void setTestPlans(List testPlanXmls) { - workloads.clear(); - for (String xml : testPlanXmls) { - WorkloadParser parser = new WorkloadParser(xml); - HDWorkload workload = parser.getWorkload(); - if (workload != null) { - workloads.add(workload); - } - } + workloads = testPlanXmls.stream() + .map(xml -> new WorkloadParser(xml).getWorkload()) + .filter(Objects::nonNull) + .collect(Collectors.toList()); } public List getTestPlans() { diff --git a/agent/apiharness/src/main/java/com/intuit/tank/harness/TestPlanStarter.java b/agent/apiharness/src/main/java/com/intuit/tank/harness/TestPlanStarter.java index ab5058301..e2741686d 100644 --- a/agent/apiharness/src/main/java/com/intuit/tank/harness/TestPlanStarter.java +++ b/agent/apiharness/src/main/java/com/intuit/tank/harness/TestPlanStarter.java @@ -42,7 +42,7 @@ public TestPlanStarter(Object httpClient, HDTestPlan plan, int numThreads, Strin this.plan = plan; this.threadGroup = threadGroup; this.tankHttpClientClass = tankHttpClientClass; - this.numThreads = (int) Math.floor(numThreads * (plan.getUserPercentage() / 100D)); + this.numThreads = (int) Math.max(1, Math.floor(numThreads * (plan.getUserPercentage() / 100D))); this.rampDelay = calcRampTime(); } diff --git a/agent/apiharness/src/main/java/com/intuit/tank/runner/method/RequestRunner.java b/agent/apiharness/src/main/java/com/intuit/tank/runner/method/RequestRunner.java index 87f6735a7..9df652032 100644 --- a/agent/apiharness/src/main/java/com/intuit/tank/runner/method/RequestRunner.java +++ b/agent/apiharness/src/main/java/com/intuit/tank/runner/method/RequestRunner.java @@ -226,7 +226,7 @@ private void processPerfResponse(String result, String uniqueName, int threadNum try { TankResultBuilder builder = new TankResultBuilder(); builder.withJobId(APITestHarness.getInstance().getAgentRunData().getJobId()) - .withInstanceId(APITestHarness.getInstance().getAgentRunData().getInstanceId()); + .withInstanceId(APITestHarness.getInstance().getAgentRunData().getInstanceId()); if (resp != null) { builder.withResponseTime((int) resp.getResponseTime()); builder.withStatusCode(resp.getHttpCode()); @@ -279,9 +279,7 @@ private boolean checkPreValidations() { ret = evaluateResult(item.getKey(), item.getValue(), item.getCondition(), variables); } - } - return ret; } diff --git a/agent/apiharness/src/test/java/com/intuit/tank/common/TPSTest.java b/agent/apiharness/src/test/java/com/intuit/tank/common/TPSTest.java index 9a03833a6..e0f7395d0 100644 --- a/agent/apiharness/src/test/java/com/intuit/tank/common/TPSTest.java +++ b/agent/apiharness/src/test/java/com/intuit/tank/common/TPSTest.java @@ -64,7 +64,7 @@ private void sendTps() { items.add(item); } String tpsTableName = "test_qa"; - db.createTable(tpsTableName); + db.initNamespace(tpsTableName); LOG.info("Sending " + items.size() + " to TPS Table " + tpsTableName); db.addItems(tpsTableName, items, false); } catch (Exception e) { diff --git a/agent/apiharness_pkg/pom.xml b/agent/apiharness_pkg/pom.xml index 5c743b1b8..0a09f24aa 100644 --- a/agent/apiharness_pkg/pom.xml +++ b/agent/apiharness_pkg/pom.xml @@ -6,7 +6,7 @@ com.intuit.tank agent-parent - 2.3.3 + 2.3.4 diff --git a/agent/http_client_3/pom.xml b/agent/http_client_3/pom.xml index 6c77fa468..298650e43 100644 --- a/agent/http_client_3/pom.xml +++ b/agent/http_client_3/pom.xml @@ -5,7 +5,7 @@ com.intuit.tank agent-parent - 2.3.3 + 2.3.4 http_client_commons_3 diff --git a/agent/http_client_4/pom.xml b/agent/http_client_4/pom.xml index 55e2a39c5..0d43e5630 100644 --- a/agent/http_client_4/pom.xml +++ b/agent/http_client_4/pom.xml @@ -5,7 +5,7 @@ com.intuit.tank agent-parent - 2.3.3 + 2.3.4 http_client_commons_4 diff --git a/agent/http_client_5/pom.xml b/agent/http_client_5/pom.xml index 4439a2d4f..fffa8b84e 100644 --- a/agent/http_client_5/pom.xml +++ b/agent/http_client_5/pom.xml @@ -5,7 +5,7 @@ com.intuit.tank agent-parent - 2.3.3 + 2.3.4 http_client_commons_5 diff --git a/agent/http_client_5/src/main/java/com/intuit/tank/httpclient5/TankHttpClient5.java b/agent/http_client_5/src/main/java/com/intuit/tank/httpclient5/TankHttpClient5.java index 474885880..b1f52468f 100644 --- a/agent/http_client_5/src/main/java/com/intuit/tank/httpclient5/TankHttpClient5.java +++ b/agent/http_client_5/src/main/java/com/intuit/tank/httpclient5/TankHttpClient5.java @@ -44,7 +44,7 @@ import org.apache.hc.client5.http.config.RequestConfig; import org.apache.hc.client5.http.cookie.BasicCookieStore; import org.apache.hc.client5.http.cookie.Cookie; -import org.apache.hc.client5.http.cookie.CookieSpecs; +import org.apache.hc.client5.http.cookie.StandardCookieSpec; import org.apache.hc.client5.http.entity.mime.MultipartEntityBuilder; import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; @@ -96,7 +96,7 @@ public TankHttpClient5() { .setCircularRedirectsAllowed(true) .setAuthenticationEnabled(true) .setRedirectsEnabled(true) - .setCookieSpec(CookieSpecs.STANDARD.ident) + .setCookieSpec(StandardCookieSpec.RELAXED) .setMaxRedirects(100) .build(); diff --git a/agent/pom.xml b/agent/pom.xml index 42cd0843e..e39d6069e 100644 --- a/agent/pom.xml +++ b/agent/pom.xml @@ -6,7 +6,7 @@ com.intuit.tank tank-parent - 2.3.3 + 2.3.4 agent-parent diff --git a/api/pom.xml b/api/pom.xml index c6cffc7a5..2cd764b95 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -6,7 +6,7 @@ com.intuit.tank tank-parent - 2.3.3 + 2.3.4 api diff --git a/data_access/pom.xml b/data_access/pom.xml index 14585438e..e66f2e1c5 100644 --- a/data_access/pom.xml +++ b/data_access/pom.xml @@ -6,7 +6,7 @@ com.intuit.tank tank-parent - 2.3.3 + 2.3.4 diff --git a/data_model/pom.xml b/data_model/pom.xml index 95f6d3238..fc04a8204 100644 --- a/data_model/pom.xml +++ b/data_model/pom.xml @@ -6,7 +6,7 @@ com.intuit.tank tank-parent - 2.3.3 + 2.3.4 data-model diff --git a/doc/doc_xslt/pom.xml b/doc/doc_xslt/pom.xml index 1ee536597..9efa90ba2 100644 --- a/doc/doc_xslt/pom.xml +++ b/doc/doc_xslt/pom.xml @@ -6,7 +6,7 @@ com.intuit.tank tank-parent-doc - 2.3.3 + 2.3.4 pressgang-xslt-ns diff --git a/doc/doc_xslt/src/main/resources/xslt/org/tank/eclipse.xsl b/doc/doc_xslt/src/main/resources/xslt/org/tank/eclipse.xsl index 0606701cc..4ca2a7779 100644 --- a/doc/doc_xslt/src/main/resources/xslt/org/tank/eclipse.xsl +++ b/doc/doc_xslt/src/main/resources/xslt/org/tank/eclipse.xsl @@ -12,7 +12,7 @@ - + diff --git a/doc/doc_xslt/src/main/resources/xslt/org/tank/pdf.xsl b/doc/doc_xslt/src/main/resources/xslt/org/tank/pdf.xsl index 7125523f6..302d9e039 100644 --- a/doc/doc_xslt/src/main/resources/xslt/org/tank/pdf.xsl +++ b/doc/doc_xslt/src/main/resources/xslt/org/tank/pdf.xsl @@ -22,7 +22,7 @@ xmlns:jbh="java:org.jboss.highlight.renderer.FORenderer" exclude-result-prefixes="jbh"> - + diff --git a/doc/doc_xslt/src/main/resources/xslt/org/tank/xhtml-single.xsl b/doc/doc_xslt/src/main/resources/xslt/org/tank/xhtml-single.xsl index cb63bc07e..1d045f2da 100644 --- a/doc/doc_xslt/src/main/resources/xslt/org/tank/xhtml-single.xsl +++ b/doc/doc_xslt/src/main/resources/xslt/org/tank/xhtml-single.xsl @@ -11,7 +11,7 @@ - + diff --git a/doc/doc_xslt/src/main/resources/xslt/org/tank/xhtml.xsl b/doc/doc_xslt/src/main/resources/xslt/org/tank/xhtml.xsl index 6e8e4fe2c..1550ecb13 100644 --- a/doc/doc_xslt/src/main/resources/xslt/org/tank/xhtml.xsl +++ b/doc/doc_xslt/src/main/resources/xslt/org/tank/xhtml.xsl @@ -11,7 +11,7 @@ - + diff --git a/doc/jdocbook_style/pom.xml b/doc/jdocbook_style/pom.xml index 6f8e999c5..87e05f270 100644 --- a/doc/jdocbook_style/pom.xml +++ b/doc/jdocbook_style/pom.xml @@ -6,7 +6,7 @@ com.intuit.tank tank-parent-doc - 2.3.3 + 2.3.4 tank-common-jdocbook-style diff --git a/doc/pom.xml b/doc/pom.xml index 57927c3af..949d26ad3 100755 --- a/doc/pom.xml +++ b/doc/pom.xml @@ -5,7 +5,7 @@ com.intuit.tank tank-parent - 2.3.3 + 2.3.4 tank-parent-doc diff --git a/doc/tank_installation_guide/pom.xml b/doc/tank_installation_guide/pom.xml index 620b7072a..e8d6d6ab7 100644 --- a/doc/tank_installation_guide/pom.xml +++ b/doc/tank_installation_guide/pom.xml @@ -5,7 +5,7 @@ com.intuit.tank tank-parent-doc - 2.3.3 + 2.3.4 tank-installation-guide diff --git a/doc/tank_user_guide/pom.xml b/doc/tank_user_guide/pom.xml index fa724739d..77c258e59 100755 --- a/doc/tank_user_guide/pom.xml +++ b/doc/tank_user_guide/pom.xml @@ -5,7 +5,7 @@ com.intuit.tank tank-parent-doc - 2.3.3 + 2.3.4 tank-user-guide diff --git a/harness_data/pom.xml b/harness_data/pom.xml index 8dbc0fbc0..5038ac5fd 100644 --- a/harness_data/pom.xml +++ b/harness_data/pom.xml @@ -5,7 +5,7 @@ com.intuit.tank tank-parent - 2.3.3 + 2.3.4 harness-data diff --git a/mail/pom.xml b/mail/pom.xml index 7bf61c6c8..14cdee8b2 100644 --- a/mail/pom.xml +++ b/mail/pom.xml @@ -6,7 +6,7 @@ com.intuit.tank tank-parent - 2.3.3 + 2.3.4 tank-mail diff --git a/pom.xml b/pom.xml index 4e9c76ffa..0b21c27e0 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.intuit.tank tank-parent - 2.3.3 + 2.3.4 pom Tank Parent @@ -28,27 +28,27 @@ self_signed_tank.p12 5.1.17.Final - 1.11.694 + 1.11.762 - 9.0.30 + 9.0.34 - 3.1.2.Final + 3.1.3.Final 2.0 2.7.1.Final 2.12.1 - 2.2.20 - 2.2.20 - 7.0 - 7.0.2 + 2.3.9 + 2.3 + 8.0 + 8.0 1.7.0 2.22.2 2.29 - 5.5.2 - 4.5.10 - 5.0-beta6 - 8.2.0 + 5.6.0 + 4.5.12 + 5.0 + 8.4.1 2.4.17 1.3-groovy-2.5 @@ -292,7 +292,7 @@ ${project.groupId} test-support - 2.3.3 + 2.3.4 test @@ -357,13 +357,13 @@ mysql mysql-connector-java - 8.0.18 + 8.0.19 com.google.guava guava - 28.0-jre + 28.2-jre @@ -437,27 +437,27 @@ - com.sun.faces - jsf-api + javax.faces + javax.faces-api ${version.jsf-api} - com.sun.faces - jsf-impl + org.glassfish + javax.faces ${version.mojarra} org.apache.poi poi - 3.17 + 4.1.2 com.itextpdf itextpdf - 5.5.13 + 5.5.13.1 @@ -502,7 +502,7 @@ com.h2database h2 - 1.4.197 + 1.4.200 @@ -514,7 +514,7 @@ org.jboss jandex - 2.1.1.Final + 2.1.2.Final @@ -526,7 +526,7 @@ org.apache.commons commons-compress - 1.19 + 1.20 @@ -586,7 +586,7 @@ com.opencsv opencsv - 4.1 + 5.1 @@ -657,7 +657,7 @@ commons-fileupload commons-fileupload - 1.3.3 + 1.4 commons-io @@ -667,12 +667,12 @@ org.apache.commons commons-lang3 - 3.7 + 3.9 javax.mail javax.mail-api - 1.6.1 + 1.6.2 de.congrace @@ -682,13 +682,13 @@ xerces xercesImpl - 2.11.0 + 2.12.0 provided xml-apis xml-apis - 1.4.01 + 2.0.2 test @@ -719,17 +719,17 @@ commons-codec commons-codec - 1.13 + 1.14 xalan xalan - 2.7.1 + 2.7.2 xalan serializer - 2.7.1 + 2.7.2 axis @@ -754,13 +754,13 @@ jaxen jaxen - 1.1.1 + 1.2.0 test - jdom + org.jdom jdom - 1.0 + 2.0.2 javax.xml @@ -952,12 +952,12 @@ org.apache.maven.plugins maven-checkstyle-plugin - 3.1.0 + 3.1.1 ${project.groupId} test-support - 2.3.3 + 2.3.4 @@ -979,20 +979,15 @@ - - org.apache.maven.plugins - maven-deploy-plugin - 2.8.2 - org.apache.maven.plugins maven-assembly-plugin - 3.1.1 + 3.2.0 org.apache.maven.plugins maven-source-plugin - 3.0.1 + 3.2.1 org.apache.maven.plugins @@ -1011,7 +1006,7 @@ org.jacoco jacoco-maven-plugin - 0.8.4 + 0.8.5 @@ -1023,10 +1018,7 @@ - - - org.codehaus.mojo versions-maven-plugin @@ -1079,12 +1071,12 @@ org.apache.maven.plugins maven-site-plugin - 3.7.1 + 3.9.0 org.apache.maven.plugins maven-jar-plugin - 3.1.1 + 3.2.0 @@ -1097,9 +1089,10 @@ - maven-javadoc-plugin + org.apache.maven.plugins - 3.0.1 + maven-javadoc-plugin + 3.2.0 org.apache.maven.plugins diff --git a/proxy-parent/WebConversation/pom.xml b/proxy-parent/WebConversation/pom.xml index f9d252a29..6da243810 100644 --- a/proxy-parent/WebConversation/pom.xml +++ b/proxy-parent/WebConversation/pom.xml @@ -5,7 +5,7 @@ com.intuit.tank proxy-parent - 2.3.3 + 2.3.4 WebConversation diff --git a/proxy-parent/owasp-proxy/pom.xml b/proxy-parent/owasp-proxy/pom.xml index d14be8654..5a4f23be6 100755 --- a/proxy-parent/owasp-proxy/pom.xml +++ b/proxy-parent/owasp-proxy/pom.xml @@ -6,7 +6,7 @@ com.intuit.tank proxy-parent - 2.3.3 + 2.3.4 owasp-proxy diff --git a/proxy-parent/owasp-proxy/src/main/java/org/owasp/proxy/util/Base64.java b/proxy-parent/owasp-proxy/src/main/java/org/owasp/proxy/util/Base64.java index 931be79f4..4178efbb8 100644 --- a/proxy-parent/owasp-proxy/src/main/java/org/owasp/proxy/util/Base64.java +++ b/proxy-parent/owasp-proxy/src/main/java/org/owasp/proxy/util/Base64.java @@ -74,7 +74,7 @@ * encoding (by padding with equals signs) too soon. Also added an option to suppress the automatic decoding of gzipped * streams. Also added experimental support for specifying a class loader when using the * {@link #decodeToObject(java.lang.String, int, java.lang.ClassLoader)} method. - *
  • v2.3.3 - Changed default char encoding to US-ASCII which reduces the internal Java footprint with its + *
  • v2.3.4-SNAPSHOT - Changed default char encoding to US-ASCII which reduces the internal Java footprint with its * CharEncoders and so forth. Fixed some javadocs that were inconsistent. Removed imports and specified things like * java.io.IOException explicitly inline.
    • *
  • v2.3.2 - Reduced memory footprint! Finally refined the "guessing" of how big the final encoded data will be so @@ -147,7 +147,7 @@ * * @author Robert Harder * @author rob@iharder.net - * @version 2.3.3 + * @version 2.3.4-SNAPSHOT */ public class Base64 { diff --git a/proxy-parent/pom.xml b/proxy-parent/pom.xml index 10b27fdce..733c503f0 100644 --- a/proxy-parent/pom.xml +++ b/proxy-parent/pom.xml @@ -6,7 +6,7 @@ com.intuit.tank tank-parent - 2.3.3 + 2.3.4 proxy-parent diff --git a/proxy-parent/proxy-extension/pom.xml b/proxy-parent/proxy-extension/pom.xml index 1e4f9cdd1..f984c4a63 100644 --- a/proxy-parent/proxy-extension/pom.xml +++ b/proxy-parent/proxy-extension/pom.xml @@ -5,7 +5,7 @@ com.intuit.tank proxy-parent - 2.3.3 + 2.3.4 proxy-extension diff --git a/proxy-parent/proxy_pkg/pom.xml b/proxy-parent/proxy_pkg/pom.xml index 8d426eb10..5ef842b74 100644 --- a/proxy-parent/proxy_pkg/pom.xml +++ b/proxy-parent/proxy_pkg/pom.xml @@ -6,7 +6,7 @@ com.intuit.tank proxy-parent - 2.3.3 + 2.3.4 diff --git a/readme.md b/readme.md index 5aa9a2eb6..d4ad58f7b 100644 --- a/readme.md +++ b/readme.md @@ -1,19 +1,24 @@ ![](https://raw.githubusercontent.com/intuit/tank/master/assets/TankLogo.gif) +[![Build Status](https://travis-ci.org/intuit/Tank.svg?branch=master)](https://travis-ci.org/intuit/Tank) +![Build Status](https://codebuild.us-east-2.amazonaws.com/badges?uuid=eyJlbmNyeXB0ZWREYXRhIjoiT1Nuc1prblk5Y0E3a05mRjF0QUtBblI0Rk1zTjZyZi9NMWNCaFg4cGlzL1dFN0xVMHgzcGt1ZCtBdFZjNWpMRkhXbGN2T1ZKSmpZbGQ3YjhyaWkzdkJBPSIsIml2UGFyYW1ldGVyU3BlYyI6IlpGbU5vTHM2cHQrbUowOVkiLCJtYXRlcmlhbFNldFNlcmlhbCI6MX0%3D&branch=master) +[![release](https://img.shields.io/badge/release-2.3.4-yellogreen.svg)](https://github.com/intuit/Tank/releases/tag/2.3.3) +[![java](https://img.shields.io/badge/java-1.8-blue.svg)](https://aws.amazon.com/corretto/) +[![tomcat](https://img.shields.io/badge/tomcat-8%20%7C%208.5%20%7C%209-blue.svg)](http://tomcat.apache.org/) # Intuit Tank -Intuit Tank is a load test platform that runs in a cloud environment. It currently supports Amazon web interfaces and utilizes services from EC2, S3, and DynamoDb. +Intuit Tank is a load test platform that runs in a cloud environment. It currently supports Amazon web interfaces and utilizes services from EC2, S3, CloudWatch (Logs/Metrics). Intuit Tank has two main components: A controller and agents. -The Controller is the central hub in the deployment. It supports the GUI for managing tests and for orchestrating. It utilizes Tomcat 6x for a web container and communicates with the agents via http. -It exposes a RESTful interface for invoking services. Data is stored in a MySql database, on the file system, and in DynamoDb. +The Controller is the central hub in the deployment. It supports the GUI for managing tests and for orchestrating. It utilizes Apache Tomcat for a web container and communicates with the agents via http. +It exposes a RESTful interface for invoking services. Data is stored in a MySql database, on the file system, and reporting metrics can be stored in CloudWatch Metrics, S3, DynamoDB or Wavefront. Agents are instantiated on demand and exist for the duration of a test. They communicate with the controller via RESTful interfaces. ## Building Intuit Tank Intuit Tank uses Maven and should be able to be built using public repositories. -You may need to increase the default memory settings for maven to build. e.g. export MAVEN_OPTS="-Xmx1024m -XX:MaxPermSize=128m" +You may need to increase the default memory settings for maven to build. e.g. export MAVEN_OPTS="-Xmx1g" There are several profiles (for the initial build you should build the release profile so that the installation guide is built. e.g. mvn clean install -P release) * default -- builds source but does not build the docs or package tools. diff --git a/reporting/api/pom.xml b/reporting/api/pom.xml index fee6a2777..89d09a177 100644 --- a/reporting/api/pom.xml +++ b/reporting/api/pom.xml @@ -6,7 +6,7 @@ com.intuit.tank reporting-parent - 2.3.3 + 2.3.4 reporting-api diff --git a/reporting/api/src/main/java/com/intuit/tank/reporting/databases/IDatabase.java b/reporting/api/src/main/java/com/intuit/tank/reporting/databases/IDatabase.java index 14c9ec038..d08433677 100644 --- a/reporting/api/src/main/java/com/intuit/tank/reporting/databases/IDatabase.java +++ b/reporting/api/src/main/java/com/intuit/tank/reporting/databases/IDatabase.java @@ -28,7 +28,7 @@ public interface IDatabase { * @param tableName * the name of the table this db instance is dealing with */ - public void createTable(@Nonnull String tableName); + public void initNamespace(@Nonnull String tableName); /** * Deletes the table from the datastore. @@ -36,7 +36,7 @@ public interface IDatabase { * @param tableName * the name of the table this db instance is dealing with */ - public void deleteTable(@Nonnull String tableName); + public void removeNamespace(@Nonnull String tableName); /** * Deletes the table from the datastore. @@ -105,6 +105,7 @@ public interface IDatabase { * the maxRange as a String in format DATE_FORMAT * @return */ + @Nonnull public List getItems(String tableName, String minRange, String maxRange, String instanceId, String... jobId); /** diff --git a/reporting/db/pom.xml b/reporting/db/pom.xml index a14420181..307cd83b8 100644 --- a/reporting/db/pom.xml +++ b/reporting/db/pom.xml @@ -6,7 +6,7 @@ com.intuit.tank reporting-parent - 2.3.3 + 2.3.4 reporting-db @@ -15,7 +15,6 @@ Reporting database support - ${project.groupId} api @@ -28,26 +27,25 @@ ${project.version} - - com.amazonaws - aws-java-sdk-dynamodb - - - - com.amazonaws - aws-java-sdk-simpledb - - - + + com.amazonaws + aws-java-sdk-dynamodb + + + + com.amazonaws + aws-java-sdk-cloudwatchmetrics + + + com.opencsv opencsv - + org.apache.commons commons-math3 - diff --git a/reporting/db/src/main/java/com/intuit/tank/persistence/databases/AmazonDynamoDatabaseDocApi.java b/reporting/db/src/main/java/com/intuit/tank/persistence/databases/AmazonDynamoDatabaseDocApi.java index 00217f7c6..e07713bd9 100644 --- a/reporting/db/src/main/java/com/intuit/tank/persistence/databases/AmazonDynamoDatabaseDocApi.java +++ b/reporting/db/src/main/java/com/intuit/tank/persistence/databases/AmazonDynamoDatabaseDocApi.java @@ -110,8 +110,6 @@ public AmazonDynamoDatabaseDocApi() { } else { this.dynamoDb = AmazonDynamoDBClientBuilder.defaultClient(); } - - } /** @@ -127,7 +125,7 @@ public AmazonDynamoDatabaseDocApi(AmazonDynamoDB dynamoDb) { * @inheritDoc */ @Override - public void createTable(String tableName) { + public void initNamespace(String tableName) { try { if (!hasTable(tableName)) { logger.info("Creating table: " + tableName); @@ -181,7 +179,7 @@ private long getCapacity(HierarchicalConfiguration resultsProviderConfig, String * @inheritDoc */ @Override - public void deleteTable(String tableName) { + public void removeNamespace(String tableName) { try { if (hasTable(tableName)) { logger.info("Deleting table: " + tableName); @@ -275,7 +273,6 @@ public Set getTables(String regex) { @Override public PagedDatabaseResult getPagedItems(String tableName, Object nextToken, String minRange, String maxRange, String instanceId, String jobId) { - List ret; Map lastKeyEvaluated = (Map) nextToken; ScanRequest scanRequest = new ScanRequest().withTableName(tableName); Map conditions = new HashMap(); @@ -315,7 +312,7 @@ public PagedDatabaseResult getPagedItems(String tableName, Object nextToken, Str scanRequest.withExclusiveStartKey(lastKeyEvaluated); ScanResult result = dynamoDb.scan(scanRequest); - ret = result.getItems().stream().map(this::getItemFromResult).collect(Collectors.toList()); + List ret = result.getItems().stream().map(this::getItemFromResult).collect(Collectors.toList()); return new PagedDatabaseResult(ret, result.getLastEvaluatedKey()); } @@ -323,6 +320,7 @@ public PagedDatabaseResult getPagedItems(String tableName, Object nextToken, Str * * @inheritDoc */ + @Nonnull @Override public List getItems(String tableName, String minRange, String maxRange, String instanceId, String... jobIds) { @@ -351,7 +349,11 @@ public void run() { MethodTimer mt = new MethodTimer(logger, this.getClass(), "addItems (" + items + ")"); List requests; try { - requests = items.stream().map(item -> itemToMap(item)).map(toInsert -> new PutRequest().withItem(toInsert)).map(putRequest -> new WriteRequest().withPutRequest(putRequest)).collect(Collectors.toList()); + requests = items.stream() + .map(item -> itemToMap(item)) + .map(toInsert -> new PutRequest().withItem(toInsert)) + .map(putRequest -> new WriteRequest().withPutRequest(putRequest)) + .collect(Collectors.toList()); sendBatch(tableName, requests); } catch (Exception t) { logger.error("Error adding results: " + t.getMessage(), t); @@ -433,7 +435,7 @@ public boolean hasJobData(String tableName, String jobId) { } /** - * @param item + * @param attributeMap * @return */ private Item getItemFromResult(Map attributeMap) { @@ -560,7 +562,7 @@ private void waitForStatus(String tableName, TableStatus status) { if (tableStatus.equals(status.toString())) return; } catch (AmazonServiceException ase) { - if (ase.getErrorCode().equalsIgnoreCase("ResourceNotFoundException") == false) + if (!ase.getErrorCode().equalsIgnoreCase("ResourceNotFoundException")) throw ase; } } @@ -583,7 +585,7 @@ private void waitForDelete(String tableName) { return; } } catch (AmazonServiceException ase) { - if (ase.getErrorCode().equalsIgnoreCase("ResourceNotFoundException") == false) + if (!ase.getErrorCode().equalsIgnoreCase("ResourceNotFoundException")) throw ase; } } diff --git a/reporting/db/src/main/java/com/intuit/tank/persistence/databases/AmazonSimpleDatabase.java b/reporting/db/src/main/java/com/intuit/tank/persistence/databases/AmazonSimpleDatabase.java deleted file mode 100644 index 76605e306..000000000 --- a/reporting/db/src/main/java/com/intuit/tank/persistence/databases/AmazonSimpleDatabase.java +++ /dev/null @@ -1,411 +0,0 @@ -package com.intuit.tank.persistence.databases; - -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.Executors; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; - -import javax.annotation.Nonnull; - -import com.amazonaws.auth.AWSStaticCredentialsProvider; -import com.amazonaws.services.simpledb.AmazonSimpleDBClientBuilder; -import org.apache.commons.lang3.StringUtils; -import org.apache.http.HttpStatus; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import com.amazonaws.AmazonServiceException; -import com.amazonaws.ClientConfiguration; -import com.amazonaws.auth.AWSCredentials; -import com.amazonaws.auth.BasicAWSCredentials; -import com.amazonaws.services.simpledb.AmazonSimpleDB; -import com.amazonaws.services.simpledb.model.Attribute; -import com.amazonaws.services.simpledb.model.BatchPutAttributesRequest; -import com.amazonaws.services.simpledb.model.CreateDomainRequest; -import com.amazonaws.services.simpledb.model.DeleteDomainRequest; -import com.amazonaws.services.simpledb.model.ListDomainsRequest; -import com.amazonaws.services.simpledb.model.ListDomainsResult; -import com.amazonaws.services.simpledb.model.ReplaceableAttribute; -import com.amazonaws.services.simpledb.model.ReplaceableItem; -import com.amazonaws.services.simpledb.model.SelectRequest; -import com.amazonaws.services.simpledb.model.SelectResult; -import com.intuit.tank.reporting.databases.IDatabase; -import com.intuit.tank.reporting.databases.Item; -import com.intuit.tank.reporting.databases.PagedDatabaseResult; -import com.intuit.tank.reporting.databases.TankDatabaseType; -import com.intuit.tank.results.TankResult; -import com.intuit.tank.vm.common.util.ReportUtil; -import com.intuit.tank.vm.settings.CloudCredentials; -import com.intuit.tank.vm.settings.CloudProvider; -import com.intuit.tank.vm.settings.TankConfig; - -public class AmazonSimpleDatabase implements IDatabase { - private static final Logger logger = LogManager.getLogger(AmazonSimpleDatabase.class); - private static final int MAX_NUMBER_OF_RETRIES = 5; - private AmazonSimpleDB db; - - private static final ThreadPoolExecutor EXECUTOR = new ThreadPoolExecutor(10, 50, 60, TimeUnit.SECONDS, new ArrayBlockingQueue(50), Executors.defaultThreadFactory(), - new ThreadPoolExecutor.DiscardOldestPolicy()); - - private static TankConfig config = new TankConfig(); - - /** - * - * @param db - */ - public AmazonSimpleDatabase(AmazonSimpleDB db) { - this.db = db; - } - - /** - * - */ - public AmazonSimpleDatabase() { - createDatabase(); - } - - /** - * - * @inheritDoc - */ - public void createTable(String tableName) { - try { - if (!hasTable(tableName)) { - logger.info("Creating table: " + tableName); - db.createDomain(new CreateDomainRequest(tableName)); - } - } catch (Exception t) { - logger.error(t, t); - } - } - - public void deleteTable(String tableName) { - if (hasTable(tableName)) { - logger.info("Deleting table: " + tableName); - db.deleteDomain(new DeleteDomainRequest(tableName)); - } - } - - /** - * @inheritDoc - */ - @Override - public void deleteForJob(final String tableName, String jobId, boolean asynch) { - Runnable task = () -> deleteTable(tableName); - if (asynch) { - EXECUTOR.execute(task); - } else { - task.run(); - } - } - - /** - * @inheritDoc - */ - public Set getTables(String regex) { - Set result = new HashSet(); - ListDomainsResult listDomains = null; - String nextToken = null; - do { - listDomains = db.listDomains(new ListDomainsRequest().withNextToken(nextToken)); - for (String s : listDomains.getDomainNames()) { - if (s.matches(regex)) { - result.add(s); - } - } - nextToken = listDomains.getNextToken(); - } while (nextToken != null); - return result; - } - - /** - * - * @inheritDoc - */ - public void addTimingResults(final @Nonnull String tableName, final @Nonnull List messages, boolean asynch) { - if (!messages.isEmpty()) { - Runnable task = () -> { - List items = new ArrayList(); - try { - for (TankResult result : messages) { - ReplaceableItem item = new ReplaceableItem(); - item.setAttributes(getTimingAttributes(result)); - item.setName(UUID.randomUUID().toString()); - items.add(item); - if (items.size() == 25) { - addItemsToTable(new BatchPutAttributesRequest(tableName, new ArrayList(items))); - // logger.info("Sending " + items.size() + " - // results to table " + tableName); - items.clear(); - } - } - if (items.size() > 0) { - addItemsToTable(new BatchPutAttributesRequest(tableName, items)); - logger.info("Sending " + items.size() + " results to table " + tableName); - } - } catch (Exception t) { - logger.error("Error adding results: " + t.getMessage(), t); - throw new RuntimeException(t); - } - }; - if (asynch) { - EXECUTOR.execute(task); - } else { - task.run(); - } - } - } - - /** - * @inheritDoc - */ - @Override - public void addItems(final String tableName, final List items, boolean asynch) { - Runnable task = () -> { - List tmpItems = new ArrayList(); - for (Item item : items) { - tmpItems.add(itemToAWSItem(item)); - if (tmpItems.size() == 25) { - addItemsToTable(new BatchPutAttributesRequest(tableName, tmpItems)); - tmpItems.clear(); - } - } - addItemsToTable(new BatchPutAttributesRequest(tableName, tmpItems)); - }; - if (asynch) { - EXECUTOR.execute(task); - } else { - task.run(); - } - - } - - /** - * @inheritDoc - */ - @Override - public boolean hasJobData(String tableName, String jobId) { - boolean ret = false; - if (hasTable(tableName)) { - SelectRequest request = new SelectRequest("SELECT * from `" + tableName + "`"); - SelectResult result = db.select(request); - ret = !result.getItems().isEmpty(); - } - return ret; - } - - /** - * - * @inheritDoc - */ - public String getDatabaseName(TankDatabaseType type, String jobId) { - return type.name() + "_" + new TankConfig().getInstanceName() + "_" + jobId; - } - - /** - * - * @param tableName - * @param items - */ - public void addItems(String tableName, List items) { - try { - List tmpItems = new ArrayList(); - for (ReplaceableItem item : items) { - tmpItems.add(item); - if (tmpItems.size() == 25) { - addItemsToTable(new BatchPutAttributesRequest(tableName, tmpItems)); - tmpItems.clear(); - } - } - addItemsToTable(new BatchPutAttributesRequest(tableName, tmpItems)); - } catch (Exception t) { - logger.error("Error adding result: " + t, t); - } - } - - /** - * - * @inheritDoc - */ - public boolean hasTable(@Nonnull String tableName) { - boolean hasMore = true; - String nextToken = null; - while (hasMore) { - ListDomainsResult listDomains = db.listDomains(new ListDomainsRequest().withNextToken(nextToken)); - for (String name : listDomains.getDomainNames()) { - if (tableName.equalsIgnoreCase(name)) { - return true; - } - } - nextToken = listDomains.getNextToken(); - hasMore = !StringUtils.isEmpty(nextToken); - } - return false; - } - - /** - * - * @param tableName - * @return - */ - public List filterExisting(List tableName) { - boolean hasMore = true; - String nextToken = null; - - Set tables = new HashSet(); - while (hasMore) { - ListDomainsResult listDomains = db.listDomains(new ListDomainsRequest().withNextToken(nextToken)); - tables.addAll(listDomains.getDomainNames()); - nextToken = listDomains.getNextToken(); - hasMore = !StringUtils.isEmpty(nextToken); - } - return tableName.stream().filter(tables::contains).collect(Collectors.toCollection(() -> new ArrayList<>(tableName.size()))); - } - - /** - * @inheritDoc - */ - @Override - public PagedDatabaseResult getPagedItems(String tableName, Object token, String minRange, String maxRange, String instanceId, String jobId) { - List ret; - String whereClause = null; - if (minRange != null && maxRange != null) { - whereClause = " Timestamp between '" + minRange + "' and '" + maxRange + "' "; - } else if (minRange != null) { - whereClause = " Timestamp >= '" + minRange + "' "; - } else if (maxRange != null) { - whereClause = " Timestamp < '" + maxRange + "' "; - } else { - whereClause = ""; - } - SelectRequest request = new SelectRequest("SELECT * from `" + tableName + "`" + whereClause).withConsistentRead(true); - String nextToken = (String) token; - request.withNextToken(nextToken); - SelectResult result = db.select(request); - ret = result.getItems().stream().map(this::resultToItem).collect(Collectors.toList()); - nextToken = result.getNextToken(); - return new PagedDatabaseResult(ret, result.getNextToken()); - } - - /** - * @inheritDoc - */ - @Override - public List getItems(String tableName, String minRange, String maxRange, String instanceId, String... jobIds) { - List ret = new ArrayList(); - for (String jobId : jobIds) { - String nextToken = null; - do { - PagedDatabaseResult pagedItems = getPagedItems(tableName, nextToken, minRange, maxRange, instanceId, jobId); - ret.addAll(pagedItems.getItems()); - nextToken = (String) pagedItems.getNextToken(); - } while (nextToken != null); - } - return ret; - } - - /** - * @param item - * @return - */ - private ReplaceableItem itemToAWSItem(Item item) { - List attributes = new ArrayList(); - for (com.intuit.tank.reporting.databases.Attribute attr : item.getAttributes()) { - addAttribute(attributes, attr.getName(), attr.getValue()); - } - return new ReplaceableItem(item.getName(), attributes); - } - - /** - * @param item - * @return - */ - private com.intuit.tank.reporting.databases.Item resultToItem(com.amazonaws.services.simpledb.model.Item item) { - List attrs = new ArrayList(); - com.intuit.tank.reporting.databases.Item ret = new com.intuit.tank.reporting.databases.Item(item.getName(), attrs); - for (Attribute attr : item.getAttributes()) { - attrs.add(new com.intuit.tank.reporting.databases.Attribute(attr.getName(), attr.getValue())); - } - return ret; - } - - private void addItemsToTable(final BatchPutAttributesRequest request) { - - boolean shouldRetry; - int retries = 0; - - do { - shouldRetry = false; - try { - db.batchPutAttributes(request); - } catch (AmazonServiceException e) { - int status = e.getStatusCode(); - if (status == HttpStatus.SC_INTERNAL_SERVER_ERROR || status == HttpStatus.SC_SERVICE_UNAVAILABLE) { - shouldRetry = true; - long delay = (long) (Math.random() * (Math.pow(4, retries++) * 100L)); - try { - Thread.sleep(delay); - } catch (InterruptedException iex) { - logger.error("Caught InterruptedException exception", iex); - } - } else if ("DuplicateItemName".equals(e.getErrorCode())) { - // ignore. - } else { - logger.error("Error writing to DB: " + e.getMessage()); - } - } - } while (shouldRetry && retries < MAX_NUMBER_OF_RETRIES); - - } - - private List getTimingAttributes(TankResult result) { - List attributes = new ArrayList(); - String timestamp = ReportUtil.getTimestamp(result.getTimeStamp()); - addAttribute(attributes, DatabaseKeys.TIMESTAMP_KEY.getShortKey(), timestamp); - addAttribute(attributes, DatabaseKeys.REQUEST_NAME_KEY.getShortKey(), timestamp + "-" + UUID.randomUUID().toString()); - addAttribute(attributes, DatabaseKeys.JOB_ID_KEY.getShortKey(), result.getJobId()); - addAttribute(attributes, DatabaseKeys.LOGGING_KEY_KEY.getShortKey(), result.getRequestName()); - addAttribute(attributes, DatabaseKeys.STATUS_CODE_KEY.getShortKey(), String.valueOf(result.getStatusCode())); - addAttribute(attributes, DatabaseKeys.RESPONSE_TIME_KEY.getShortKey(), String.valueOf(result.getResponseTime())); - addAttribute(attributes, DatabaseKeys.RESPONSE_SIZE_KEY.getShortKey(), String.valueOf(result.getResponseSize())); - addAttribute(attributes, DatabaseKeys.INSTANCE_ID_KEY.getShortKey(), String.valueOf(result.getInstanceId())); - addAttribute(attributes, DatabaseKeys.IS_ERROR_KEY.getShortKey(), String.valueOf(result.isError())); - return attributes; - } - - private void addAttribute(List attributes, String key, String value) { - if (value == null) { - value = ""; - } - attributes.add(new ReplaceableAttribute().withName(key).withValue(value)); - } - - private void createDatabase() { - CloudCredentials creds = config.getVmManagerConfig().getCloudCredentials(CloudProvider.amazon); - ClientConfiguration config = new ClientConfiguration(); - if (StringUtils.isNotBlank(System.getProperty("http.proxyHost"))) { - try { - config.setProxyHost(System.getProperty("http.proxyHost")); - if (StringUtils.isNotBlank(System.getProperty("http.proxyPort"))) { - config.setProxyPort(Integer.valueOf(System.getProperty("http.proxyPort"))); - } - } catch (NumberFormatException e) { - logger.error("invalid proxy setup."); - } - - } - if (StringUtils.isNotBlank(creds.getKeyId()) && StringUtils.isNotBlank(creds.getKey())) { - AWSCredentials credentials = new BasicAWSCredentials(creds.getKeyId(), creds.getKey()); - this.db = AmazonSimpleDBClientBuilder.standard().withCredentials(new AWSStaticCredentialsProvider(credentials)).build(); - } else { - this.db = AmazonSimpleDBClientBuilder.standard().withClientConfiguration(config).build(); - } - } - -} diff --git a/reporting/db/src/main/java/com/intuit/tank/persistence/databases/CloudWatchDataSource.java b/reporting/db/src/main/java/com/intuit/tank/persistence/databases/CloudWatchDataSource.java new file mode 100644 index 000000000..1fe684ed5 --- /dev/null +++ b/reporting/db/src/main/java/com/intuit/tank/persistence/databases/CloudWatchDataSource.java @@ -0,0 +1,150 @@ +package com.intuit.tank.persistence.databases; + +import com.amazonaws.services.cloudwatch.AmazonCloudWatch; +import com.amazonaws.services.cloudwatch.AmazonCloudWatchClientBuilder; +import com.amazonaws.services.cloudwatch.model.Dimension; +import com.amazonaws.services.cloudwatch.model.MetricDatum; +import com.amazonaws.services.cloudwatch.model.PutMetricDataRequest;; +import com.amazonaws.services.cloudwatch.model.StandardUnit; +import com.amazonaws.services.cloudwatch.model.StatisticSet; +import com.intuit.tank.reporting.databases.IDatabase; +import com.intuit.tank.reporting.databases.Item; +import com.intuit.tank.reporting.databases.PagedDatabaseResult; +import com.intuit.tank.reporting.databases.TankDatabaseType; +import com.intuit.tank.results.TankResult; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import javax.annotation.Nonnull; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Supplier; +import java.util.stream.Collectors; +import java.util.stream.DoubleStream; + +import static java.util.stream.Collectors.groupingBy; +import static java.util.stream.Collectors.toList; + +/** + * CloudWatchDataSource + * + * @author Kevin McGoldrick + * + */ +public class CloudWatchDataSource implements IDatabase { + private static final Logger LOG = LogManager.getLogger(CloudWatchDataSource.class); + + private final AmazonCloudWatch cloudWatchClient = AmazonCloudWatchClientBuilder.defaultClient(); + private static final String namespace = "Intuit/Tank"; + + @Override + public void initNamespace(@Nonnull String tableName) {} + + @Override + public void removeNamespace(@Nonnull String tableName) {} + + @Override + public void deleteForJob(@Nonnull String tableName, @Nonnull String jobId, boolean asynch) {} + + @Override + public boolean hasTable(@Nonnull String tableName) { + return true; + } + + @Override + public boolean hasJobData(@Nonnull String tableName, String jobId) { + return true; + } + + @Override + public void addTimingResults(@Nonnull String tableName, @Nonnull List results, boolean asynch) { + LOG.trace("Starting addTimingResults with " + results.size() + " items"); + Date timestamp = results.get(results.size()-1).getTimeStamp(); + List datumList = new ArrayList<>(); + + Dimension instanceId = new Dimension() + .withName("InstanceId") + .withValue(results.get(0).getInstanceId()); + + Dimension jobId = new Dimension() + .withName("JobId") + .withValue(results.get(0).getJobId()); + try { + Map> grouped = results.stream() + .collect(groupingBy(TankResult::getRequestName, + Collectors.mapping(TankResult::getResponseTime, toList()))); + LOG.trace("Sorted into " + grouped.size() + " request name groups"); + + for (Map.Entry> entry : grouped.entrySet()) { + List groupResults = entry.getValue(); + int size = groupResults.size(); + Supplier doubleStream = + () -> groupResults.stream() + .sorted() + .mapToDouble(Double::valueOf); + Collection sortedList = doubleStream.get().boxed().collect(Collectors.toList()); + double[] sortedArray = doubleStream.get().toArray(); + double sum = doubleStream.get().sum(); + + Dimension request = new Dimension() + .withName("RequestName") + .withValue(entry.getKey()); + + datumList.add(new MetricDatum() + .withMetricName("ResponseTime") + .withUnit(StandardUnit.Milliseconds) + .withStatisticValues(new StatisticSet() + .withMaximum(sortedArray[size - 1]) + .withMinimum(sortedArray[0]) + .withSampleCount((double) size) + .withSum(sum)) + .withValues(sortedList) + .withTimestamp(timestamp) + .withDimensions(request, instanceId, jobId)); + + datumList.add(new MetricDatum() + .withMetricName("RequestCount") + .withUnit(StandardUnit.Count) + .withValue((double) size) + .withTimestamp(timestamp) + .withDimensions(request, instanceId, jobId)); + } + LOG.trace("Sending to CloudWatchMetrics: " + datumList.size() + " to " + namespace); + PutMetricDataRequest request = new PutMetricDataRequest() + .withNamespace(namespace) + .withMetricData(datumList); + + cloudWatchClient.putMetricData(request); + } catch (Exception e) { + LOG.error("Failed to push metric data to cloudwatch: " + e.getMessage(), e); + } + } + + @Override + public Set getTables(String regex) { + return null; + } + + @Override + public void addItems(String tableName, List items, boolean asynch) {} + + @Nonnull + @Override + public List getItems(String tableName, String minRange, String maxRange, String instanceId, String... jobId) { + return new ArrayList<>(); + } + + @Override + public PagedDatabaseResult getPagedItems(String tableName, Object nextToken, String minRange, String maxRange, String instanceId, String jobId) { + return null; + } + + @Override + public String getDatabaseName(TankDatabaseType type, String jobId) { + return namespace; + } +} diff --git a/reporting/db/src/main/java/com/intuit/tank/persistence/databases/DataBaseFactory.java b/reporting/db/src/main/java/com/intuit/tank/persistence/databases/DataBaseFactory.java index 41a3773e6..f510a957e 100644 --- a/reporting/db/src/main/java/com/intuit/tank/persistence/databases/DataBaseFactory.java +++ b/reporting/db/src/main/java/com/intuit/tank/persistence/databases/DataBaseFactory.java @@ -18,6 +18,8 @@ import com.intuit.tank.reporting.databases.IDatabase; import com.intuit.tank.vm.settings.TankConfig; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; /** * DataBaseFactory @@ -26,26 +28,19 @@ * */ public class DataBaseFactory { - - public static final String resultsProvider = new TankConfig().getVmManagerConfig().getResultsProvider(); + private static final Logger LOG = LogManager.getLogger(DataBaseFactory.class); + private static final String resultsProvider = new TankConfig().getVmManagerConfig().getResultsProvider(); -// private static IDatabase provider; -// private static String providerClass; - - public static final IDatabase getDatabase() { -// if (provider == null || !provider.getClass().getName().equals(providerClass)) { + public static IDatabase getDatabase() { return initProvider(); -// } -// return provider; } private static IDatabase initProvider() { - IDatabase provider = null; try { - provider = (IDatabase) Class.forName(resultsProvider).newInstance(); + return (IDatabase) Class.forName(resultsProvider).newInstance(); } catch (Exception e) { - provider = new AmazonDynamoDatabaseDocApi(); + LOG.error("Unable to create DataSource Provider : " + resultsProvider + ", update settings.xml", e); } - return provider; + return new CloudWatchDataSource(); } } diff --git a/reporting/db/src/main/java/com/intuit/tank/persistence/databases/GraphiteDatasource.java b/reporting/db/src/main/java/com/intuit/tank/persistence/databases/GraphiteDatasource.java index b72b546bc..927f91cce 100644 --- a/reporting/db/src/main/java/com/intuit/tank/persistence/databases/GraphiteDatasource.java +++ b/reporting/db/src/main/java/com/intuit/tank/persistence/databases/GraphiteDatasource.java @@ -23,6 +23,8 @@ import com.intuit.tank.results.TankResult; import com.intuit.tank.vm.settings.TankConfig; +import javax.annotation.Nonnull; + /** * GraphiteDatasource * @@ -41,12 +43,12 @@ public class GraphiteDatasource implements IDatabase { private HierarchicalConfiguration resultsProviderConfig = config.getVmManagerConfig().getResultsProviderConfig(); @Override - public void createTable(String tableName) { + public void initNamespace(String tableName) { // TODO Auto-generated method stub } @Override - public void deleteTable(String tableName) { + public void removeNamespace(String tableName) { // TODO Auto-generated method stub } @@ -175,10 +177,10 @@ public void addItems(String tableName, List items, boolean asynch) { } + @Nonnull @Override public List getItems(String tableName, String minRange, String maxRange, String instanceId, String... jobId) { - // TODO Auto-generated method stub - return null; + return new ArrayList<>(); } @Override diff --git a/reporting/db/src/main/java/com/intuit/tank/persistence/databases/S3Datasource.java b/reporting/db/src/main/java/com/intuit/tank/persistence/databases/S3Datasource.java index 4b6e16815..fd930febe 100644 --- a/reporting/db/src/main/java/com/intuit/tank/persistence/databases/S3Datasource.java +++ b/reporting/db/src/main/java/com/intuit/tank/persistence/databases/S3Datasource.java @@ -3,11 +3,14 @@ import java.io.ByteArrayInputStream; import java.io.InputStream; import java.net.InetAddress; +import java.net.UnknownHostException; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.UUID; +import java.util.stream.Collectors; import org.apache.commons.configuration.HierarchicalConfiguration; import org.apache.commons.lang3.StringUtils; @@ -28,6 +31,11 @@ import com.intuit.tank.results.TankResult; import com.intuit.tank.vm.settings.TankConfig; +import javax.annotation.Nonnull; + +import static java.util.stream.Collectors.groupingBy; +import static java.util.stream.Collectors.toList; + /** * S3Datasource * @@ -35,165 +43,123 @@ * */ public class S3Datasource implements IDatabase { - private static final Logger LOG = LogManager.getLogger(S3Datasource.class); - - private String hostname = "fail"; - private String metricString = ""; + private static final Logger LOG = LogManager.getLogger(S3Datasource.class); + + private final AmazonS3 s3Client = AmazonS3ClientBuilder.defaultClient(); + private String hostname = "fail"; + private String metricString = ""; private String tags = ""; private String bucketName = ""; - - private TankConfig config = new TankConfig(); - private HierarchicalConfiguration resultsProviderConfig = config.getVmManagerConfig().getResultsProviderConfig(); + + private HierarchicalConfiguration resultsProviderConfig = + new TankConfig().getVmManagerConfig().getResultsProviderConfig(); @Override - public void createTable(String tableName) { - // TODO Auto-generated method stub - } + public void initNamespace(String tableName) {} @Override - public void deleteTable(String tableName) { - // TODO Auto-generated method stub - - } + public void removeNamespace(String tableName) {} @Override - public void deleteForJob(String tableName, String jobId, boolean asynch) { - // TODO Auto-generated method stub - - } + public void deleteForJob(String tableName, String jobId, boolean asynch) {} @Override public boolean hasTable(String tableName) { - // TODO Auto-generated method stub return true; } @Override public boolean hasJobData(String tableName, String jobId) { - // TODO Auto-generated method stub return true; } - @SuppressWarnings("unchecked") @Override public void addTimingResults(String tableName, List results, boolean asynch) { - if (resultsProviderConfig != null) { - try { - metricString = resultsProviderConfig.getString("metricString","test.tank.transaction"); - tags = resultsProviderConfig.getString("tags"); //Example "bu=ctg app=tto pool=agent service=tank env=prf" - bucketName = resultsProviderConfig.getString("bucket", "tank-test"); - hostname = InetAddress.getLocalHost().getHostName(); - } catch (Exception e) { - LOG.error("Failed to get S3 Datasource parameters " + e.toString(), e); - } - } + LOG.trace("Starting addTimingResults with " + results.size() + " items"); + if (resultsProviderConfig != null) { + try { + metricString = resultsProviderConfig.getString("metricString","test.tank.transaction"); + tags = resultsProviderConfig.getString("tags", ""); //Example "bu=ctg app=tto pool=agent service=tank env=prf" + bucketName = resultsProviderConfig.getString("bucket", "tank-test"); + hostname = InetAddress.getLocalHost().getHostName(); + } catch (UnknownHostException e) { + LOG.error("Failed to get hostname " + e.toString(), e); + } + } else { + LOG.error("Results Provider Config is Empty. Please update settings.xml to include S3Datasource Configuration"); + } + + if (StringUtils.endsWith(bucketName, "-")) { + bucketName = bucketName.concat(s3Client.getRegionName()); + } + + StringBuilder sb = new StringBuilder(); String jobId = results.get(0).getJobId(); String instance = results.get(0).getInstanceId(); long timestamp = results.get(results.size()-1).getTimeStamp().getTime() / 1000; try { - AmazonS3 s3Client = AmazonS3ClientBuilder.defaultClient(); - - String Region = s3Client.getRegionName(); - if (StringUtils.endsWith(bucketName, "-")) { - bucketName = bucketName.concat(Region); - } String tagsComplete = " source=" + hostname + " instanceid=" + instance + - " location=" + Region + + " location=" + s3Client.getRegionName() + " " + tags + " jobid=" + jobId + System.getProperty("line.separator"); - StringBuilder sb = new StringBuilder(); - List groupResults = new ArrayList(); - String requestName = ""; - long sum = 0; - Collections.sort(results); - for (TankResult metric: results) { - if (StringUtils.equalsIgnoreCase(metric.getRequestName(), requestName)) { //Middle of the Group - groupResults.add(Long.valueOf(metric.getResponseTime())); - sum += metric.getResponseTime(); - } else if (StringUtils.isEmpty(requestName)) { // Handles the first time through // - requestName = metric.getRequestName(); - sum = metric.getResponseTime(); - groupResults.add(Long.valueOf(sum)); - } else { // Handles the last time through of the group// - int size = groupResults.size(); - Collections.sort(groupResults); - Long[] sortedList = groupResults.toArray(new Long[size]); - long average = sum / size; - int fiftieth = (size/2); - if (fiftieth >= 1) fiftieth--; - float ninety = 0.9f; - int ninetieth = Math.round(size * ninety); - if (ninetieth >= 1) ninetieth--; - float ninetynine = 0.99f; - int ninetynineth = Math.round(size * ninetynine); - if (ninetynineth >= 1) ninetynineth--; - sb.append(metricString + ".resp_time.min " + sortedList[0] + " " + timestamp + " transaction=" + requestName) - .append(tagsComplete) - .append(metricString + ".resp_time.avg " + average + " " + timestamp + " transaction=" + requestName) - .append(tagsComplete) - .append(metricString + ".resp_time.max " + sortedList[size - 1] + " " + timestamp + " transaction=" + requestName) - .append(tagsComplete) - .append(metricString + ".resp_time.tp_50 " + sortedList[fiftieth] + " " + timestamp + " transaction=" + requestName) - .append(tagsComplete) - .append(metricString + ".resp_time.tp_90 " + sortedList[ninetieth] + " " + timestamp + " transaction=" + requestName) - .append(tagsComplete) - .append(metricString + ".resp_time.tp_99 " + sortedList[ninetynineth] + " " + timestamp + " transaction=" + requestName) - .append(tagsComplete) - .append(metricString + ".rpi " + size + " " + timestamp + " transaction=" + requestName) - .append(tagsComplete); - requestName = metric.getRequestName(); - groupResults.clear(); - sum = metric.getResponseTime(); - groupResults.add(Long.valueOf(sum)); - } - } // Get that last one // - int size = groupResults.size(); - Collections.sort(groupResults); - Long[] sortedList = groupResults.toArray(new Long[size]); - long average = sum / size; - int fiftieth = (size/2); - if (fiftieth >= 1) fiftieth--; - float ninety = 0.9f; - int ninetieth = Math.round(size * ninety); - if (ninetieth >= 1) ninetieth--; - float ninetynine = 0.99f; - int ninetynineth = Math.round(size * ninetynine); - if (ninetynineth >= 1) ninetynineth--; - sb.append(metricString + ".resp_time.min " + sortedList[0] + " " + timestamp + " transaction=" + requestName) - .append(tagsComplete) - .append(metricString + ".resp_time.avg " + average + " " + timestamp + " transaction=" + requestName) - .append(tagsComplete) - .append(metricString + ".resp_time.max " + sortedList[size - 1] + " " + timestamp + " transaction=" + requestName) - .append(tagsComplete) - .append(metricString + ".resp_time.tp_50 " + sortedList[fiftieth] + " " + timestamp + " transaction=" + requestName) - .append(tagsComplete) - .append(metricString + ".resp_time.tp_90 " + sortedList[ninetieth] + " " + timestamp + " transaction=" + requestName) - .append(tagsComplete) - .append(metricString + ".resp_time.tp_99 " + sortedList[ninetynineth] + " " + timestamp + " transaction=" + requestName) - .append(tagsComplete) - .append(metricString + ".rpi " + size + " " + timestamp + " transaction=" + requestName) - .append(tagsComplete); + + Map> grouped = results.stream() + .collect(groupingBy(TankResult::getRequestName, + Collectors.mapping(TankResult::getResponseTime, toList()))); + LOG.trace("Sorted into " + grouped.size() + " request name groups"); + + for (Map.Entry> entry : grouped.entrySet()) { + String requestName = entry.getKey(); + List groupResults = entry.getValue(); + int size = groupResults.size(); + long sum = groupResults.stream().mapToInt(Integer::intValue).sum(); + Collections.sort(groupResults); + Integer[] sortedList = groupResults.toArray(new Integer[0]); + long average = sum / size; + int fiftieth = (size/2); + if (fiftieth >= 1) fiftieth--; + int ninetieth = Math.round(size * 0.9f); + if (ninetieth >= 1) ninetieth--; + int ninetynineth = Math.round(size * 0.99f); + if (ninetynineth >= 1) ninetynineth--; + sb.append(metricString + ".resp_time.min " + sortedList[0] + " " + timestamp + " transaction=" + requestName) + .append(tagsComplete) + .append(metricString + ".resp_time.avg " + average + " " + timestamp + " transaction=" + requestName) + .append(tagsComplete) + .append(metricString + ".resp_time.max " + sortedList[size - 1] + " " + timestamp + " transaction=" + requestName) + .append(tagsComplete) + .append(metricString + ".resp_time.tp_50 " + sortedList[fiftieth] + " " + timestamp + " transaction=" + requestName) + .append(tagsComplete) + .append(metricString + ".resp_time.tp_90 " + sortedList[ninetieth] + " " + timestamp + " transaction=" + requestName) + .append(tagsComplete) + .append(metricString + ".resp_time.tp_99 " + sortedList[ninetynineth] + " " + timestamp + " transaction=" + requestName) + .append(tagsComplete) + .append(metricString + ".rpi " + size + " " + timestamp + " transaction=" + requestName) + .append(tagsComplete); + } + LOG.trace("Sending to S3: " + sb.toString()); InputStream is = new ByteArrayInputStream(sb.toString().getBytes()); ObjectMetadata metaData = new ObjectMetadata(); metaData.setContentLength(sb.length()); - s3Client.putObject( + PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, "TANK-AgentData-" + UUID.randomUUID() + ".log", is, metaData) - .withCannedAcl(CannedAccessControlList.BucketOwnerFullControl)); + .withCannedAcl(CannedAccessControlList.BucketOwnerFullControl); + s3Client.putObject(putObjectRequest); } catch (AmazonServiceException ase) { LOG.error("AmazonServiceException: which " + "means your request made it " + "to Amazon S3, but was rejected with an error response" + - " for some reason. bucket=" + bucketName + + " for some reason. bucket=" + bucketName + "," + ase.getMessage(), ase); } catch (AmazonClientException ace) { LOG.error("AmazonClientException: which " + "means the client encountered " + "an internal error while trying to " + "communicate with S3, " + - "such as not being able to access the network. bucket=" + + "such as not being able to access the network. bucket=" + bucketName + "," + ace.getMessage(), ace); } catch (Exception e) { LOG.error("Error: " + e.getMessage(), e); @@ -212,10 +178,10 @@ public void addItems(String tableName, List items, boolean asynch) { } + @Nonnull @Override public List getItems(String tableName, String minRange, String maxRange, String instanceId, String... jobId) { - // TODO Auto-generated method stub - return null; + return new ArrayList<>(); } @Override diff --git a/reporting/db/src/main/java/com/intuit/tank/persistence/databases/WavefrontDatasource.java b/reporting/db/src/main/java/com/intuit/tank/persistence/databases/WavefrontDatasource.java index 37d77092b..214095596 100644 --- a/reporting/db/src/main/java/com/intuit/tank/persistence/databases/WavefrontDatasource.java +++ b/reporting/db/src/main/java/com/intuit/tank/persistence/databases/WavefrontDatasource.java @@ -24,6 +24,8 @@ import com.intuit.tank.results.TankResult; import com.intuit.tank.vm.settings.TankConfig; +import javax.annotation.Nonnull; + /** * WavefrontDatasource * @@ -44,12 +46,12 @@ public class WavefrontDatasource implements IDatabase { private HierarchicalConfiguration resultsProviderConfig = config.getVmManagerConfig().getResultsProviderConfig(); @Override - public void createTable(String tableName) { + public void initNamespace(String tableName) { // TODO Auto-generated method stub } @Override - public void deleteTable(String tableName) { + public void removeNamespace(String tableName) { // TODO Auto-generated method stub } @@ -218,10 +220,10 @@ public void addItems(String tableName, List items, boolean asynch) { } + @Nonnull @Override public List getItems(String tableName, String minRange, String maxRange, String instanceId, String... jobId) { - // TODO Auto-generated method stub - return null; + return new ArrayList<>(); } @Override diff --git a/reporting/db/src/main/java/com/intuit/tank/reporting/db/DatabaseResultsReader.java b/reporting/db/src/main/java/com/intuit/tank/reporting/db/DatabaseResultsReader.java index 19aa13e70..9dcf9ac3a 100644 --- a/reporting/db/src/main/java/com/intuit/tank/reporting/db/DatabaseResultsReader.java +++ b/reporting/db/src/main/java/com/intuit/tank/reporting/db/DatabaseResultsReader.java @@ -172,7 +172,6 @@ private Map> getTpsMap(Date minDate, String instanceI timestamp = ReportUtil.parseTimestamp(att.getValue()); } catch (ParseException e) { LOG.error("Error processing timestamp " + att.getValue() + ":" + e); - continue; } } else if (DatabaseKeys.TRANSACTIONS_KEY.getShortKey().equals(att.getName())) { transactions = Integer.parseInt(att.getValue()); diff --git a/reporting/db/src/main/java/com/intuit/tank/reporting/db/DatabaseResultsReporter.java b/reporting/db/src/main/java/com/intuit/tank/reporting/db/DatabaseResultsReporter.java index f2245a5e9..2ea444b9d 100644 --- a/reporting/db/src/main/java/com/intuit/tank/reporting/db/DatabaseResultsReporter.java +++ b/reporting/db/src/main/java/com/intuit/tank/reporting/db/DatabaseResultsReporter.java @@ -41,9 +41,10 @@ public class DatabaseResultsReporter implements ResultsReporter { private static final Logger LOG = LogManager.getLogger(DatabaseResultsReporter.class); - private static final ThreadPoolExecutor EXECUTOR = new ThreadPoolExecutor(10, 50, 60, TimeUnit.SECONDS, - new ArrayBlockingQueue(50), Executors.defaultThreadFactory(), - new ThreadPoolExecutor.DiscardOldestPolicy()); + private static final ThreadPoolExecutor EXECUTOR = + new ThreadPoolExecutor(10, 50, 60, TimeUnit.SECONDS, + new ArrayBlockingQueue(50), Executors.defaultThreadFactory(), + new ThreadPoolExecutor.DiscardOldestPolicy()); private String tpsTableName; private String timingTableName; @@ -62,7 +63,9 @@ public void sendTpsResults(final String jobId, final String instanceId, final TP Runnable task = () -> { try { - List items = container.getTpsInfos().stream().map(info -> createItem(jobId, instanceId, info)).collect(Collectors.toList()); + List items = container.getTpsInfos().stream() + .map(info -> createItem(jobId, instanceId, info)) + .collect(Collectors.toList()); if (!items.isEmpty()) { String tableName = getTpsTableName(db); LOG.info(new ObjectMessage(ImmutableMap.of("Message", "Sending " + items.size() + " to TPS Table " + tableName))); @@ -132,7 +135,7 @@ public void config(HierarchicalConfiguration config) { private String getTimingTableName(IDatabase db, String jobId) { if (StringUtils.isBlank(timingTableName)) { timingTableName = db.getDatabaseName(TankDatabaseType.timing, jobId); - db.createTable(timingTableName); + db.initNamespace(timingTableName); } return timingTableName; } @@ -140,7 +143,7 @@ private String getTimingTableName(IDatabase db, String jobId) { private String getTpsTableName(IDatabase db) { if (StringUtils.isBlank(tpsTableName)) { tpsTableName = new TankConfig().getInstanceName() + "_tps"; - db.createTable(tpsTableName); + db.initNamespace(tpsTableName); } return tpsTableName; } diff --git a/reporting/db/src/test/java/com/intuit/tank/persistence/databases/AmazonDynamoDatabaseTest.java b/reporting/db/src/test/java/com/intuit/tank/persistence/databases/AmazonDynamoDatabaseTest.java index 6307685ea..5c948562a 100644 --- a/reporting/db/src/test/java/com/intuit/tank/persistence/databases/AmazonDynamoDatabaseTest.java +++ b/reporting/db/src/test/java/com/intuit/tank/persistence/databases/AmazonDynamoDatabaseTest.java @@ -5,16 +5,17 @@ import java.util.Date; import java.util.List; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.services.dynamodbv2.AmazonDynamoDB; +import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClientBuilder; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.core.LoggerContext; import org.apache.logging.log4j.core.config.Configuration; import org.junit.jupiter.api.BeforeEach; -import com.amazonaws.ClientConfiguration; import com.amazonaws.auth.AWSCredentials; import com.amazonaws.auth.BasicAWSCredentials; -import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient; import com.amazonaws.services.dynamodbv2.model.ListTablesResult; import com.intuit.tank.reporting.databases.Attribute; import com.intuit.tank.reporting.databases.Item; @@ -33,7 +34,7 @@ public class AmazonDynamoDatabaseTest { private static final String TEST_TABLE = "TestTable"; private static final int NUM_ENTRIES = 100; private AmazonDynamoDatabaseDocApi db; - private AmazonDynamoDBClient dynamoDb; + private AmazonDynamoDB dynamoDb; @BeforeEach public void init() { @@ -47,7 +48,7 @@ public void init() { public void before() { AWSCredentials credentials = new BasicAWSCredentials(System.getProperty("AWS_SECRET_KEY_ID"), System.getProperty("AWS_SECRET_KEY")); - dynamoDb = new AmazonDynamoDBClient(credentials, new ClientConfiguration()); + dynamoDb = AmazonDynamoDBClientBuilder.standard().withCredentials(new AWSStaticCredentialsProvider(credentials)).build(); db = new AmazonDynamoDatabaseDocApi(dynamoDb); } @@ -55,7 +56,7 @@ public void before() { public void cleanTables() { if (db != null) { try { - db.deleteTable(TEST_TABLE); + db.removeNamespace(TEST_TABLE); } catch (Exception e) { e.printStackTrace(); } @@ -75,16 +76,16 @@ public void testListTables() { @Test @Tag(TestGroups.EXPERIMENTAL) public void testCreateDelete() { - db.createTable(TEST_TABLE); + db.initNamespace(TEST_TABLE); boolean hasTable = db.hasTable(TEST_TABLE); assertEquals(true, hasTable); - db.deleteTable(TEST_TABLE); + db.removeNamespace(TEST_TABLE); } @Test @Tag(TestGroups.EXPERIMENTAL) public void testInsertTiming() { - db.createTable(TEST_TABLE); + db.initNamespace(TEST_TABLE); boolean hasTable = db.hasTable(TEST_TABLE); assertEquals(true, hasTable); @@ -110,7 +111,7 @@ public void testInsertTiming() { @Test @Tag(TestGroups.EXPERIMENTAL) public void testInsertTps() { - db.createTable(TEST_TABLE); + db.initNamespace(TEST_TABLE); int numJobs = 3; boolean hasTable = db.hasTable(TEST_TABLE); assertEquals(true, hasTable); diff --git a/reporting/db/src/test/java/com/intuit/tank/persistence/databases/AmazonSimpleDBTest.java b/reporting/db/src/test/java/com/intuit/tank/persistence/databases/AmazonSimpleDBTest.java deleted file mode 100644 index be67458bc..000000000 --- a/reporting/db/src/test/java/com/intuit/tank/persistence/databases/AmazonSimpleDBTest.java +++ /dev/null @@ -1,35 +0,0 @@ -package com.intuit.tank.persistence.databases; - -import java.util.Set; - -import com.amazonaws.ClientConfiguration; -import com.amazonaws.auth.AWSCredentials; -import com.amazonaws.auth.BasicAWSCredentials; -import com.amazonaws.services.simpledb.AmazonSimpleDB; -import com.amazonaws.services.simpledb.AmazonSimpleDBClient; -import com.intuit.tank.test.TestGroups; -import com.intuit.tank.vm.api.enumerated.VMRegion; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; - -public class AmazonSimpleDBTest { - - private AmazonSimpleDB simpeDB; - private AmazonSimpleDatabase db; - - @BeforeEach - public void before() { - AWSCredentials credentials = new BasicAWSCredentials(System.getProperty("AWS_SECRET_KEY_ID"), - System.getProperty("AWS_SECRET_KEY")); - simpeDB = new AmazonSimpleDBClient(credentials, new ClientConfiguration()); - simpeDB.setEndpoint(VMRegion.US_WEST_2.getEndpoint()); - } - - @Test - @Tag(TestGroups.EXPERIMENTAL) - public void testSimpleDB() { - Set tables = db.getTables("job"); - System.out.println(tables); - } -} diff --git a/reporting/local/pom.xml b/reporting/local/pom.xml index fa4b8916e..60bc97272 100644 --- a/reporting/local/pom.xml +++ b/reporting/local/pom.xml @@ -6,7 +6,7 @@ com.intuit.tank reporting-parent - 2.3.3 + 2.3.4 reporting-local-storage diff --git a/reporting/pom.xml b/reporting/pom.xml index 3a8726dc8..b452b51dc 100644 --- a/reporting/pom.xml +++ b/reporting/pom.xml @@ -6,7 +6,7 @@ com.intuit.tank tank-parent - 2.3.3 + 2.3.4 reporting-parent diff --git a/reporting/rest/pom.xml b/reporting/rest/pom.xml index 8db6ff5cd..352ee7815 100644 --- a/reporting/rest/pom.xml +++ b/reporting/rest/pom.xml @@ -6,7 +6,7 @@ com.intuit.tank reporting-parent - 2.3.3 + 2.3.4 reporting-rest diff --git a/rest/api/agent/pom.xml b/rest/api/agent/pom.xml index dc7457c79..e990dd471 100644 --- a/rest/api/agent/pom.xml +++ b/rest/api/agent/pom.xml @@ -6,7 +6,7 @@ com.intuit.tank api-parent - 2.3.3 + 2.3.4 agent-api diff --git a/rest/api/automation/pom.xml b/rest/api/automation/pom.xml index decaf3d47..f4c337a9a 100644 --- a/rest/api/automation/pom.xml +++ b/rest/api/automation/pom.xml @@ -6,7 +6,7 @@ com.intuit.tank api-parent - 2.3.3 + 2.3.4 automation-api diff --git a/rest/api/cloud/pom.xml b/rest/api/cloud/pom.xml index 57912c926..8f2da35cc 100644 --- a/rest/api/cloud/pom.xml +++ b/rest/api/cloud/pom.xml @@ -6,7 +6,7 @@ com.intuit.tank api-parent - 2.3.3 + 2.3.4 cloud-api diff --git a/rest/api/common/pom.xml b/rest/api/common/pom.xml index ca24e59bd..2296c1bac 100644 --- a/rest/api/common/pom.xml +++ b/rest/api/common/pom.xml @@ -6,7 +6,7 @@ com.intuit.tank api-parent - 2.3.3 + 2.3.4 api-common diff --git a/rest/api/datafile/pom.xml b/rest/api/datafile/pom.xml index d19f2e9e9..39ab55927 100644 --- a/rest/api/datafile/pom.xml +++ b/rest/api/datafile/pom.xml @@ -6,7 +6,7 @@ com.intuit.tank api-parent - 2.3.3 + 2.3.4 diff --git a/rest/api/filter/pom.xml b/rest/api/filter/pom.xml index 9b887fe36..55c4be6a2 100644 --- a/rest/api/filter/pom.xml +++ b/rest/api/filter/pom.xml @@ -6,7 +6,7 @@ com.intuit.tank api-parent - 2.3.3 + 2.3.4 filter-api diff --git a/rest/api/job/pom.xml b/rest/api/job/pom.xml index 68a2f5f7d..528f8f62e 100644 --- a/rest/api/job/pom.xml +++ b/rest/api/job/pom.xml @@ -6,7 +6,7 @@ com.intuit.tank api-parent - 2.3.3 + 2.3.4 job-api diff --git a/rest/api/pom.xml b/rest/api/pom.xml index f8927bc4b..74b8f4eb8 100644 --- a/rest/api/pom.xml +++ b/rest/api/pom.xml @@ -6,7 +6,7 @@ com.intuit.tank rest-parent - 2.3.3 + 2.3.4 api-parent diff --git a/rest/api/project/pom.xml b/rest/api/project/pom.xml index 6d7d623f2..20c3e7a28 100644 --- a/rest/api/project/pom.xml +++ b/rest/api/project/pom.xml @@ -6,7 +6,7 @@ com.intuit.tank api-parent - 2.3.3 + 2.3.4 project-api diff --git a/rest/api/reporting/pom.xml b/rest/api/reporting/pom.xml index c00436e92..488eaa6e1 100644 --- a/rest/api/reporting/pom.xml +++ b/rest/api/reporting/pom.xml @@ -6,7 +6,7 @@ com.intuit.tank api-parent - 2.3.3 + 2.3.4 report-api diff --git a/rest/api/script/pom.xml b/rest/api/script/pom.xml index d70325a14..6ff09e630 100644 --- a/rest/api/script/pom.xml +++ b/rest/api/script/pom.xml @@ -6,7 +6,7 @@ com.intuit.tank api-parent - 2.3.3 + 2.3.4 script-api diff --git a/rest/api/user/pom.xml b/rest/api/user/pom.xml index 38cd461e5..987c8a017 100644 --- a/rest/api/user/pom.xml +++ b/rest/api/user/pom.xml @@ -6,7 +6,7 @@ com.intuit.tank api-parent - 2.3.3 + 2.3.4 user-api diff --git a/rest/client/agent/pom.xml b/rest/client/agent/pom.xml index eead230e4..17838ff4b 100644 --- a/rest/client/agent/pom.xml +++ b/rest/client/agent/pom.xml @@ -6,7 +6,7 @@ com.intuit.tank client-parent - 2.3.3 + 2.3.4 agent-client diff --git a/rest/client/automation/pom.xml b/rest/client/automation/pom.xml index c499a4c14..ebbd97b47 100644 --- a/rest/client/automation/pom.xml +++ b/rest/client/automation/pom.xml @@ -6,7 +6,7 @@ com.intuit.tank client-parent - 2.3.3 + 2.3.4 automation-client diff --git a/rest/client/cloud/pom.xml b/rest/client/cloud/pom.xml index 8da597f4b..3569396d1 100644 --- a/rest/client/cloud/pom.xml +++ b/rest/client/cloud/pom.xml @@ -6,7 +6,7 @@ com.intuit.tank client-parent - 2.3.3 + 2.3.4 cloud-client diff --git a/rest/client/common/pom.xml b/rest/client/common/pom.xml index 5b1816f04..481e827d1 100644 --- a/rest/client/common/pom.xml +++ b/rest/client/common/pom.xml @@ -6,7 +6,7 @@ com.intuit.tank client-parent - 2.3.3 + 2.3.4 client-common diff --git a/rest/client/datafile/pom.xml b/rest/client/datafile/pom.xml index 00039abca..ef0408831 100644 --- a/rest/client/datafile/pom.xml +++ b/rest/client/datafile/pom.xml @@ -6,7 +6,7 @@ com.intuit.tank client-parent - 2.3.3 + 2.3.4 datafile-client diff --git a/rest/client/filter/pom.xml b/rest/client/filter/pom.xml index 5c4e7ed10..ee99d0038 100644 --- a/rest/client/filter/pom.xml +++ b/rest/client/filter/pom.xml @@ -6,7 +6,7 @@ com.intuit.tank client-parent - 2.3.3 + 2.3.4 filter-client diff --git a/rest/client/job/pom.xml b/rest/client/job/pom.xml index 420661d35..2c5de8e24 100644 --- a/rest/client/job/pom.xml +++ b/rest/client/job/pom.xml @@ -6,7 +6,7 @@ com.intuit.tank client-parent - 2.3.3 + 2.3.4 job-client diff --git a/rest/client/pom.xml b/rest/client/pom.xml index 6b572aca1..24d55f332 100644 --- a/rest/client/pom.xml +++ b/rest/client/pom.xml @@ -6,7 +6,7 @@ com.intuit.tank rest-parent - 2.3.3 + 2.3.4 client-parent diff --git a/rest/client/project/pom.xml b/rest/client/project/pom.xml index bcaf4af9f..004ab393d 100644 --- a/rest/client/project/pom.xml +++ b/rest/client/project/pom.xml @@ -6,7 +6,7 @@ com.intuit.tank client-parent - 2.3.3 + 2.3.4 project-client diff --git a/rest/client/reporting/pom.xml b/rest/client/reporting/pom.xml index 116e8f951..4e7bc9e4f 100644 --- a/rest/client/reporting/pom.xml +++ b/rest/client/reporting/pom.xml @@ -6,7 +6,7 @@ com.intuit.tank client-parent - 2.3.3 + 2.3.4 report-client diff --git a/rest/client/script/pom.xml b/rest/client/script/pom.xml index 66952f093..ebf972245 100644 --- a/rest/client/script/pom.xml +++ b/rest/client/script/pom.xml @@ -6,7 +6,7 @@ com.intuit.tank client-parent - 2.3.3 + 2.3.4 script-client diff --git a/rest/client/user/pom.xml b/rest/client/user/pom.xml index 4a2c74d13..8360703de 100644 --- a/rest/client/user/pom.xml +++ b/rest/client/user/pom.xml @@ -6,7 +6,7 @@ com.intuit.tank client-parent - 2.3.3 + 2.3.4 user-client diff --git a/rest/pom.xml b/rest/pom.xml index bc0df157c..be474ddc2 100644 --- a/rest/pom.xml +++ b/rest/pom.xml @@ -6,7 +6,7 @@ com.intuit.tank tank-parent - 2.3.3 + 2.3.4 rest-parent diff --git a/rest/service/agent/pom.xml b/rest/service/agent/pom.xml index a9c81aa5d..dc451a1b5 100644 --- a/rest/service/agent/pom.xml +++ b/rest/service/agent/pom.xml @@ -6,7 +6,7 @@ com.intuit.tank service-parent - 2.3.3 + 2.3.4 agent-service diff --git a/rest/service/automation/pom.xml b/rest/service/automation/pom.xml index 137f55ea5..031df9a8e 100644 --- a/rest/service/automation/pom.xml +++ b/rest/service/automation/pom.xml @@ -6,7 +6,7 @@ com.intuit.tank service-parent - 2.3.3 + 2.3.4 automation-service diff --git a/rest/service/cloud/pom.xml b/rest/service/cloud/pom.xml index b0074c563..eda824ac4 100644 --- a/rest/service/cloud/pom.xml +++ b/rest/service/cloud/pom.xml @@ -6,7 +6,7 @@ com.intuit.tank service-parent - 2.3.3 + 2.3.4 cloud-service diff --git a/rest/service/common/pom.xml b/rest/service/common/pom.xml index 018c0833d..3be6be4ca 100644 --- a/rest/service/common/pom.xml +++ b/rest/service/common/pom.xml @@ -6,7 +6,7 @@ com.intuit.tank service-parent - 2.3.3 + 2.3.4 service-common diff --git a/rest/service/datafile/pom.xml b/rest/service/datafile/pom.xml index 08112b90d..0705b655b 100644 --- a/rest/service/datafile/pom.xml +++ b/rest/service/datafile/pom.xml @@ -6,7 +6,7 @@ com.intuit.tank service-parent - 2.3.3 + 2.3.4 datafile-service diff --git a/rest/service/filter/pom.xml b/rest/service/filter/pom.xml index 4219229cc..a815e5488 100644 --- a/rest/service/filter/pom.xml +++ b/rest/service/filter/pom.xml @@ -6,7 +6,7 @@ com.intuit.tank service-parent - 2.3.3 + 2.3.4 filter-service diff --git a/rest/service/job/pom.xml b/rest/service/job/pom.xml index 4c9a434b6..347c496eb 100644 --- a/rest/service/job/pom.xml +++ b/rest/service/job/pom.xml @@ -6,7 +6,7 @@ com.intuit.tank service-parent - 2.3.3 + 2.3.4 job-service diff --git a/rest/service/pom.xml b/rest/service/pom.xml index a169a4698..b206a42dd 100644 --- a/rest/service/pom.xml +++ b/rest/service/pom.xml @@ -6,7 +6,7 @@ com.intuit.tank rest-parent - 2.3.3 + 2.3.4 service-parent diff --git a/rest/service/project/pom.xml b/rest/service/project/pom.xml index de9e1da02..cee079360 100644 --- a/rest/service/project/pom.xml +++ b/rest/service/project/pom.xml @@ -6,7 +6,7 @@ com.intuit.tank service-parent - 2.3.3 + 2.3.4 project-service diff --git a/rest/service/reporting/pom.xml b/rest/service/reporting/pom.xml index 2c26e30f1..0072cb0e4 100644 --- a/rest/service/reporting/pom.xml +++ b/rest/service/reporting/pom.xml @@ -6,7 +6,7 @@ com.intuit.tank service-parent - 2.3.3 + 2.3.4 report-service diff --git a/rest/service/script/pom.xml b/rest/service/script/pom.xml index 1dd7d6d8d..efeaf76ed 100644 --- a/rest/service/script/pom.xml +++ b/rest/service/script/pom.xml @@ -6,7 +6,7 @@ com.intuit.tank service-parent - 2.3.3 + 2.3.4 script-service diff --git a/rest/service/user/pom.xml b/rest/service/user/pom.xml index 1c5e18f49..04480f294 100644 --- a/rest/service/user/pom.xml +++ b/rest/service/user/pom.xml @@ -6,7 +6,7 @@ com.intuit.tank service-parent - 2.3.3 + 2.3.4 user-service diff --git a/script_processor/pom.xml b/script_processor/pom.xml index 133e572fe..a57efd4fc 100644 --- a/script_processor/pom.xml +++ b/script_processor/pom.xml @@ -6,7 +6,7 @@ com.intuit.tank tank-parent - 2.3.3 + 2.3.4 tank-script-processor @@ -57,7 +57,7 @@ - jdom + org.jdom jdom test diff --git a/search/document_util/pom.xml b/search/document_util/pom.xml index ba69977a8..105e1c766 100644 --- a/search/document_util/pom.xml +++ b/search/document_util/pom.xml @@ -5,7 +5,7 @@ com.intuit.tank search-parent - 2.3.3 + 2.3.4 document-util diff --git a/search/document_util/src/main/java/com/intuit/tank/search/util/SearchConstants.java b/search/document_util/src/main/java/com/intuit/tank/search/util/SearchConstants.java index ff30db8aa..34fcc0501 100644 --- a/search/document_util/src/main/java/com/intuit/tank/search/util/SearchConstants.java +++ b/search/document_util/src/main/java/com/intuit/tank/search/util/SearchConstants.java @@ -17,7 +17,7 @@ import org.apache.lucene.util.Version; public class SearchConstants { - public static final Version version = Version.LUCENE_8_2_0; + public static final Version version = Version.LATEST; public static final Analyzer analyzer = new TankAnalyzer(version); } diff --git a/search/document_util/src/test/java/com/intuit/tank/search/util/TankAnalyzerTest.java b/search/document_util/src/test/java/com/intuit/tank/search/util/TankAnalyzerTest.java index 2e6a3e7d8..a94e73691 100644 --- a/search/document_util/src/test/java/com/intuit/tank/search/util/TankAnalyzerTest.java +++ b/search/document_util/src/test/java/com/intuit/tank/search/util/TankAnalyzerTest.java @@ -38,7 +38,7 @@ public class TankAnalyzerTest { @Test public void testTAnkAnalyzer_1() throws Exception { - Version matchVersion = Version.LUCENE_8_2_0; + Version matchVersion = Version.LATEST; TankAnalyzer result = new TankAnalyzer(matchVersion); @@ -56,7 +56,7 @@ public void testTAnkAnalyzer_1() @Test public void testCreateComponents_1() throws Exception { - TankAnalyzer fixture = new TankAnalyzer(Version.LUCENE_8_2_0); + TankAnalyzer fixture = new TankAnalyzer(Version.LATEST); String fieldName = ""; Analyzer.TokenStreamComponents result = fixture.createComponents(fieldName); diff --git a/search/lucene_indexer/pom.xml b/search/lucene_indexer/pom.xml index 121cd0df6..39f649e52 100644 --- a/search/lucene_indexer/pom.xml +++ b/search/lucene_indexer/pom.xml @@ -5,7 +5,7 @@ com.intuit.tank search-parent - 2.3.3 + 2.3.4 lucene-indexer diff --git a/search/pom.xml b/search/pom.xml index a07ff8b48..59d9205c3 100755 --- a/search/pom.xml +++ b/search/pom.xml @@ -6,7 +6,7 @@ com.intuit.tank tank-parent - 2.3.3 + 2.3.4 search-parent diff --git a/search/script_search/pom.xml b/search/script_search/pom.xml index 40fd6535a..85424cdf0 100644 --- a/search/script_search/pom.xml +++ b/search/script_search/pom.xml @@ -5,7 +5,7 @@ com.intuit.tank search-parent - 2.3.3 + 2.3.4 script-search diff --git a/tank_common/pom.xml b/tank_common/pom.xml index 32e06eeb6..8a38f133a 100644 --- a/tank_common/pom.xml +++ b/tank_common/pom.xml @@ -6,7 +6,7 @@ com.intuit.tank tank-parent - 2.3.3 + 2.3.4 tank-common diff --git a/tank_vmManager/pom.xml b/tank_vmManager/pom.xml index 011d06ef2..da7671e1d 100644 --- a/tank_vmManager/pom.xml +++ b/tank_vmManager/pom.xml @@ -6,7 +6,7 @@ com.intuit.tank tank-parent - 2.3.3 + 2.3.4 tank-vm-manager diff --git a/tank_vmManager/src/main/java/com/intuit/tank/vmManager/VMTrackerImpl.java b/tank_vmManager/src/main/java/com/intuit/tank/vmManager/VMTrackerImpl.java index 80d3eba04..e80a17f50 100644 --- a/tank_vmManager/src/main/java/com/intuit/tank/vmManager/VMTrackerImpl.java +++ b/tank_vmManager/src/main/java/com/intuit/tank/vmManager/VMTrackerImpl.java @@ -20,6 +20,7 @@ import java.text.SimpleDateFormat; import java.util.Arrays; +import java.util.Collections; import java.util.Date; import java.util.HashSet; import java.util.Map; @@ -158,7 +159,7 @@ private void setStatusThread(@Nonnull final CloudVmStatus status) { && (status.getJobStatus() == JobStatus.Completed) && !isDevMode()) { AmazonInstance amzInstance = new AmazonInstance(status.getVmRegion()); - amzInstance.killInstances(Arrays.asList(new String[] { status.getInstanceId() })); + amzInstance.killInstances(Collections.singletonList(status.getInstanceId())); } } String jobId = status.getJobId(); @@ -363,7 +364,7 @@ private void addStatusToJobContainer(CloudVmStatus status, } } - LOG.info("Setting Container for job=" + status.getJobId() + " newStatus to " + newStatus); + LOG.trace("Setting Container for job=" + status.getJobId() + " newStatus to " + newStatus); job.setStatus(newStatus); new JobInstanceDao().saveOrUpdate(job); diff --git a/tank_vmManager/src/main/java/com/intuit/tank/vmManager/environment/JobRequest.java b/tank_vmManager/src/main/java/com/intuit/tank/vmManager/environment/JobRequest.java index 09479adec..8379cf894 100644 --- a/tank_vmManager/src/main/java/com/intuit/tank/vmManager/environment/JobRequest.java +++ b/tank_vmManager/src/main/java/com/intuit/tank/vmManager/environment/JobRequest.java @@ -101,7 +101,7 @@ private void persistInstances(VMInstanceRequest instanceRequest, List findInstancesOfType(VMRegion region, VMImageType type @Override public void terminateInstances(@Nonnull List instanceIds) { for (VMRegion region : new TankConfig().getVmManagerConfig().getRegions()) { - AmazonInstance amazonInstance = new AmazonInstance(region); - amazonInstance.killInstances(instanceIds); + new AmazonInstance(region).killInstances(instanceIds); } } @@ -66,8 +65,7 @@ public void terminateInstances(@Nonnull List instanceIds) { @Override public void stopInstances(@Nonnull List instanceIds) { for (VMRegion region : new TankConfig().getVmManagerConfig().getRegions()) { - AmazonInstance amazonInstance = new AmazonInstance(region); - amazonInstance.stopInstances(instanceIds); + new AmazonInstance(region).stopInstances(instanceIds); } } @@ -76,8 +74,7 @@ public void stopInstances(@Nonnull List instanceIds) { */ @Override public List startInstances(@Nonnull VMInstanceRequest request) { - AmazonInstance amazonInstance = new AmazonInstance(request.getRegion()); - return amazonInstance.create(request); + return new AmazonInstance(request.getRegion()).create(request); } } diff --git a/tank_vmManager/src/main/java/com/intuit/tank/vmManager/environment/amazon/AmazonInstance.java b/tank_vmManager/src/main/java/com/intuit/tank/vmManager/environment/amazon/AmazonInstance.java index 65e995f85..f1b7506dd 100644 --- a/tank_vmManager/src/main/java/com/intuit/tank/vmManager/environment/amazon/AmazonInstance.java +++ b/tank_vmManager/src/main/java/com/intuit/tank/vmManager/environment/amazon/AmazonInstance.java @@ -142,25 +142,19 @@ public void attachVolume(String volumneId, String instanceId, String device) { */ @Override public List describeInstances(String... instanceIds) { - List result = new ArrayList(); try { - - DescribeInstancesResult results = asynchEc2Client.describeInstances(); HashSet ids = new HashSet(Arrays.asList(instanceIds)); - for (Reservation reservationDescription : results.getReservations()) { - for (com.amazonaws.services.ec2.model.Instance instance : reservationDescription.getInstances()) { - if (ids.contains(instance.getInstanceId())) { - result.add(new AmazonDataConverter().instanceToVmInformation(reservationDescription, instance, vmRegion)); - } - } - // result.addAll(TypicaDataConverter.processReservationDescription(reservationDescription)); - } + + return asynchEc2Client.describeInstances().getReservations().stream() + .flatMap(reservationDescription -> reservationDescription.getInstances() + .stream() + .filter(instance -> ids.contains(instance.getInstanceId())) + .map(instance -> new AmazonDataConverter().instanceToVmInformation(reservationDescription, instance, vmRegion))) + .collect(Collectors.toList()); } catch (Exception e) { LOG.error("Failed to retrieve instance from Amazon: " + e.getMessage()); throw new RuntimeException(e); } - - return result; } /** @@ -240,9 +234,9 @@ public List create(VMRequest request) { String image = getAMI(instanceDescription); LOG.info("Requesting " + remaining + " instances in " + vmRegion.getName() + " with AMI=" + image); - RunInstancesRequest runInstancesRequest = new RunInstancesRequest(); + RunInstancesRequest runInstancesRequestTemplate = new RunInstancesRequest(); Tenancy tenancy = StringUtils.isEmpty(instanceDescription.getTenancy()) ? Tenancy.Default : Tenancy.fromValue(instanceDescription.getTenancy()); - runInstancesRequest.withImageId(image) + runInstancesRequestTemplate.withImageId(image) .withInstanceType(size.toString()) .withKeyName(keyPair) .withPlacement(new Placement().withTenancy(tenancy)) @@ -252,45 +246,39 @@ public List create(VMRequest request) { Collection c = instanceDescription.getSecurityGroupIds(); if (!c.isEmpty()) { LOG.info("Security Group IDs " + c.toString()); - runInstancesRequest.withSecurityGroupIds(c); + runInstancesRequestTemplate.withSecurityGroupIds(c); } else { - runInstancesRequest.withSecurityGroups(instanceDescription.getSecurityGroup()); + runInstancesRequestTemplate.withSecurityGroups(instanceDescription.getSecurityGroup()); } if (StringUtils.isNotBlank(instanceDescription.getIamRole())) { IamInstanceProfileSpecification iamInstanceProfile = new IamInstanceProfileSpecification().withName(instanceDescription.getIamRole()); - runInstancesRequest.withIamInstanceProfile(iamInstanceProfile); + runInstancesRequestTemplate.withIamInstanceProfile(iamInstanceProfile); } // add zone info if specified if (!StringUtils.isEmpty(instanceDescription.getZone())) { - runInstancesRequest.withPlacement(new Placement().withAvailabilityZone(instanceDescription.getZone())); + runInstancesRequestTemplate.withPlacement(new Placement().withAvailabilityZone(instanceDescription.getZone())); } // Loop to request agent instances List subnetIds = instanceDescription.getSubnetIds(); int position = ThreadLocalRandom.current().nextInt(subnetIds.size()); - while ( !subnetIds.isEmpty() && remaining > MAX_INSTANCE_BATCH_SIZE ) { - RunInstancesRequest runInstancesRequestClone = runInstancesRequest.clone(); - runInstancesRequestClone.withSubnetId(subnetIds.get(position++)); - position = (position == subnetIds.size()) ? 0 : position; + while ( !subnetIds.isEmpty() && remaining > 0 ) { + RunInstancesRequest runInstancesRequest = runInstancesRequestTemplate.clone(); + runInstancesRequest.withSubnetId(subnetIds.get(position >= subnetIds.size() ? 0 : position++)); + int requestCount = Math.min(remaining, MAX_INSTANCE_BATCH_SIZE); try { - RunInstancesResult results = asynchEc2Client.runInstances(runInstancesRequestClone.withMinCount(1) - .withMaxCount(MAX_INSTANCE_BATCH_SIZE)); + RunInstancesResult results = + asynchEc2Client.runInstances( + runInstancesRequest.withMinCount(1).withMaxCount(requestCount)); result.addAll(new AmazonDataConverter().processReservation(results.getReservation(), vmRegion)); remaining -= results.getReservation().getInstances().size(); } catch (AmazonEC2Exception ae) { - LOG.error("Amazon issue starting instances: count=" + MAX_INSTANCE_BATCH_SIZE + " : " + ae.getMessage(), ae); - subnetIds.remove(runInstancesRequestClone.getSubnetId()); + LOG.error("Amazon issue starting instances: count=" + + remaining + " : subnets=" + subnetIds.size() + " : " + ae.getMessage()); + if(StringUtils.contains(ae.getMessage(), "InsufficientInstanceCapacity")) { + subnetIds.remove(runInstancesRequest.getSubnetId()); + } } } - if (!subnetIds.isEmpty()) { - runInstancesRequest.withSubnetId(subnetIds.get(position)); - } - try { - RunInstancesResult results = asynchEc2Client.runInstances(runInstancesRequest.withMinCount(remaining) - .withMaxCount(remaining)); - result.addAll(new AmazonDataConverter().processReservation(results.getReservation(), vmRegion)); - } catch (AmazonEC2Exception ae) { - LOG.error("Amazon issue starting instances: count=" + remaining + " : " + ae.getMessage(), ae); - } if (instanceRequest.isUseEips()) { Set
    availableEips = new HashSet
    (); @@ -341,10 +329,10 @@ public List create(VMRequest request) { } } catch (AmazonEC2Exception ae) { - LOG.error("Amazon issue starting instancs: " + ae.getMessage(), ae); + LOG.error("Amazon issue starting instances: " + ae.getMessage(), ae); throw new RuntimeException(ae); } catch (Exception ex) { - LOG.error("Error starting instancs: " + ex.getMessage(), ex); + LOG.error("Error starting instances: " + ex.getMessage(), ex); throw new RuntimeException(ex); } return result; @@ -441,22 +429,30 @@ public List kill(VMRequest request) { @Override public void killInstances(List instanceIds) { - asynchEc2Client.terminateInstancesAsync( - new TerminateInstancesRequest(instanceIds), - new AsyncHandler() { - @Override - public void onError(Exception exception) { - LOG.warn("something went wrong killing the instances {}", - exception.getLocalizedMessage()); - } - @Override - public void onSuccess(TerminateInstancesRequest request, TerminateInstancesResult result) { - LOG.trace("instances killed successfully {}", - result.getTerminatingInstances()); - } - }); - //return new AmazonDataConverter().processStateChange(terminateInstances.getTerminatingInstances()); + // Filter instanceId list to only instances in the client defined region. + List instancesInRegion = describeInstances(instanceIds.toArray(new String[instanceIds.size()])); + List instanceIdsInRegion = instancesInRegion.stream().map(VMInformation::getInstanceId).collect(Collectors.toList()); + + if (!instanceIdsInRegion.isEmpty()) { + // asynchronous AWS SDK terminate instances. + asynchEc2Client.terminateInstancesAsync( + new TerminateInstancesRequest(instanceIdsInRegion), + new AsyncHandler() { + @Override + public void onError(Exception exception) { + LOG.warn("something went wrong killing the instances : {}", + exception.getLocalizedMessage()); + } + + @Override + public void onSuccess(TerminateInstancesRequest request, TerminateInstancesResult result) { + LOG.trace("instances killed successfully : {}", + result.getTerminatingInstances()); + } + }); + //return new AmazonDataConverter().processStateChange(terminateInstances.getTerminatingInstances()); + } } /** @@ -612,22 +608,30 @@ public void reboot(List instances) { * @param instanceIds */ public void stopInstances(List instanceIds) { - asynchEc2Client.stopInstancesAsync( - new StopInstancesRequest(instanceIds), - new AsyncHandler() { - @Override - public void onError(Exception exception) { - LOG.warn("something went wrong stopping the instances {}", - exception.getLocalizedMessage()); - } - @Override - public void onSuccess(StopInstancesRequest request, StopInstancesResult result) { - LOG.trace("instances stopped successfully {}", - result.getStoppingInstances()); - } - }); - //return new AmazonDataConverter().processStateChange(stoppingInstances); + // Filter instanceId list to only instances in the client defined region. + List instancesInRegion = describeInstances(instanceIds.toArray(new String[instanceIds.size()])); + List instanceIdsInRegion = instancesInRegion.stream().map(VMInformation::getInstanceId).collect(Collectors.toList()); + + if (!instanceIdsInRegion.isEmpty()) { + // asynchronous AWS SDK stop instances. + asynchEc2Client.stopInstancesAsync( + new StopInstancesRequest(instanceIdsInRegion), + new AsyncHandler() { + @Override + public void onError(Exception exception) { + LOG.warn("something went wrong stopping the instances {}", + exception.getLocalizedMessage()); + } + + @Override + public void onSuccess(StopInstancesRequest request, StopInstancesResult result) { + LOG.trace("instances stopped successfully {}", + result.getStoppingInstances()); + } + }); + //return new AmazonDataConverter().processStateChange(stoppingInstances); + } } /** @@ -636,7 +640,8 @@ public void onSuccess(StopInstancesRequest request, StopInstancesResult result) */ private String buildUserData(@Nonnull Map userDataMap) { String sb = userDataMap.entrySet().stream() - .map(entry -> entry.getKey() + "=" + entry.getValue()).collect(Collectors.joining("\n")); + .map(Object::toString) + .collect(Collectors.joining("\n")); return Base64.encodeBase64String(sb.getBytes()); } @@ -653,18 +658,16 @@ private AssociateContainer(String instanceId, Address address) { } public String findPublicName(String instanceId) { - String result = null; try { DescribeInstancesRequest describeInstancesRequest = new DescribeInstancesRequest().withInstanceIds(instanceId); - DescribeInstancesResult output = asynchEc2Client.describeInstances(describeInstancesRequest); - if (output.getReservations() != null && output.getReservations().size() == 1) { - Reservation reservation = output.getReservations().get(0); - result = reservation.getInstances().get(0).getPublicDnsName(); + DescribeInstancesResult result = asynchEc2Client.describeInstances(describeInstancesRequest); + if (result.getReservations() != null && result.getReservations().size() == 1) { + return result.getReservations().get(0).getInstances().get(0).getPublicDnsName(); } } catch (Exception e) { LOG.error("Error getting public dns: " + e, e); } - return result; + return null; } } diff --git a/test_support/pom.xml b/test_support/pom.xml index b01bfd62d..9e43026e7 100644 --- a/test_support/pom.xml +++ b/test_support/pom.xml @@ -6,7 +6,7 @@ com.intuit.tank test-support - 2.3.3 + 2.3.4 jar Intuit Tank Test Support module diff --git a/tools/agent_debugger/pom.xml b/tools/agent_debugger/pom.xml index 143cf00f8..14dcc38ae 100755 --- a/tools/agent_debugger/pom.xml +++ b/tools/agent_debugger/pom.xml @@ -5,7 +5,7 @@ com.intuit.tank tools-parent - 2.3.3 + 2.3.4 agent-debugger diff --git a/tools/agent_debugger/src/main/java/com/intuit/tank/tools/debugger/ActionProducer.java b/tools/agent_debugger/src/main/java/com/intuit/tank/tools/debugger/ActionProducer.java index bc61866d7..8f7a6e71c 100644 --- a/tools/agent_debugger/src/main/java/com/intuit/tank/tools/debugger/ActionProducer.java +++ b/tools/agent_debugger/src/main/java/com/intuit/tank/tools/debugger/ActionProducer.java @@ -908,14 +908,13 @@ private void setFromString(String scriptXml) { } private HDWorkload unmarshalWorkload(String xml) { - HDWorkload ret = null; try { - ret = JaxbUtil.unmarshall(xml, HDWorkload.class); + return JaxbUtil.unmarshall(xml, HDWorkload.class); } catch (JAXBException | ParserConfigurationException | SAXException e) { JOptionPane.showMessageDialog(debuggerFrame, e.getMessage(), "Error unmarshalling xml", JOptionPane.ERROR_MESSAGE); } - return ret; + return null; } public enum IconSize { diff --git a/tools/agent_debugger/src/main/java/com/intuit/tank/tools/debugger/AgentDebuggerFrame.java b/tools/agent_debugger/src/main/java/com/intuit/tank/tools/debugger/AgentDebuggerFrame.java index d8f1970d7..7ab2c9a95 100644 --- a/tools/agent_debugger/src/main/java/com/intuit/tank/tools/debugger/AgentDebuggerFrame.java +++ b/tools/agent_debugger/src/main/java/com/intuit/tank/tools/debugger/AgentDebuggerFrame.java @@ -36,6 +36,7 @@ import java.io.InputStream; import java.lang.reflect.InvocationTargetException; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -794,9 +795,7 @@ private void createHarness() throws JAXBException { harness.getAgentRunData().setTotalAgents(1); harness.setDebug(true); - List testPlansXmls = new ArrayList(); - testPlansXmls.add(buildScriptString()); - TestPlanSingleton.getInstance().setTestPlans(testPlansXmls); + TestPlanSingleton.getInstance().setTestPlans(Collections.singletonList(buildScriptString())); downloadDataFiles(); JexlIOFunctions.resetStatics(); JexlStringFunctions.resetStatics(); diff --git a/tools/agent_debugger_pkg/pom.xml b/tools/agent_debugger_pkg/pom.xml index 4df92b509..99009bb5a 100644 --- a/tools/agent_debugger_pkg/pom.xml +++ b/tools/agent_debugger_pkg/pom.xml @@ -6,7 +6,7 @@ com.intuit.tank tools-parent - 2.3.3 + 2.3.4 agent-debugger-package diff --git a/tools/jenkins_plugin/pom.xml b/tools/jenkins_plugin/pom.xml index 873ca688f..316cb7803 100644 --- a/tools/jenkins_plugin/pom.xml +++ b/tools/jenkins_plugin/pom.xml @@ -34,7 +34,7 @@ - 2.3.3 + 2.3.4 2.97 diff --git a/tools/pom.xml b/tools/pom.xml index eb32a7fce..6d8bfa785 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -6,7 +6,7 @@ com.intuit.tank tank-parent - 2.3.3 + 2.3.4 tools-parent diff --git a/tools/script_engine/pom.xml b/tools/script_engine/pom.xml index 441e947ce..ca1bd7c15 100644 --- a/tools/script_engine/pom.xml +++ b/tools/script_engine/pom.xml @@ -6,7 +6,7 @@ com.intuit.tank tools-parent - 2.3.3 + 2.3.4 script-engine diff --git a/tools/script_filter/pom.xml b/tools/script_filter/pom.xml index fce3ae422..c7a775291 100755 --- a/tools/script_filter/pom.xml +++ b/tools/script_filter/pom.xml @@ -5,7 +5,7 @@ com.intuit.tank tools-parent - 2.3.3 + 2.3.4 script-filter diff --git a/tools/script_filter_pkg/pom.xml b/tools/script_filter_pkg/pom.xml index d7bc559ce..f38cad25d 100644 --- a/tools/script_filter_pkg/pom.xml +++ b/tools/script_filter_pkg/pom.xml @@ -6,7 +6,7 @@ com.intuit.tank tools-parent - 2.3.3 + 2.3.4 script-filter-package diff --git a/web/pom.xml b/web/pom.xml index 30690ecae..a124323e2 100644 --- a/web/pom.xml +++ b/web/pom.xml @@ -6,7 +6,7 @@ com.intuit.tank tank-parent - 2.3.3 + 2.3.4 web-parent diff --git a/web/web_support/pom.xml b/web/web_support/pom.xml index 417d123b1..25785080b 100644 --- a/web/web_support/pom.xml +++ b/web/web_support/pom.xml @@ -6,7 +6,7 @@ com.intuit.tank web-parent - 2.3.3 + 2.3.4 web-support @@ -104,13 +104,13 @@ - com.sun.faces - jsf-api + javax.faces + javax.faces-api - com.sun.faces - jsf-impl + org.glassfish + javax.faces diff --git a/web/web_support/src/main/java/com/intuit/tank/ProjectBean.java b/web/web_support/src/main/java/com/intuit/tank/ProjectBean.java index 8adb6ff24..159e2373d 100644 --- a/web/web_support/src/main/java/com/intuit/tank/ProjectBean.java +++ b/web/web_support/src/main/java/com/intuit/tank/ProjectBean.java @@ -152,7 +152,6 @@ public void setComments(String comment) { * @param prj */ public void openProject(Project prj) { - AWSXRay.getCurrentSegment().setUser(identityManager.lookupById(User.class, identity.getAccount().getId()).getLoginName()); conversation.begin(); AWSXRay.createSubsegment("Open.Project." + prj.getName(), (subsegment) -> { subsegment.putAnnotation("project", prj.getName()); diff --git a/web/web_support/src/main/java/com/intuit/tank/auth/InternalSecurity.java b/web/web_support/src/main/java/com/intuit/tank/auth/InternalSecurity.java index ac12bd855..797658e8f 100644 --- a/web/web_support/src/main/java/com/intuit/tank/auth/InternalSecurity.java +++ b/web/web_support/src/main/java/com/intuit/tank/auth/InternalSecurity.java @@ -16,8 +16,9 @@ * #L% */ -import javax.enterprise.context.ApplicationScoped; +import javax.enterprise.context.SessionScoped; import javax.inject.Inject; +import javax.inject.Named; import org.apache.commons.lang3.StringUtils; import org.apache.deltaspike.security.api.authorization.Secures; @@ -34,6 +35,8 @@ import com.intuit.tank.qualifier.Current; import com.intuit.tank.vm.common.TankConstants; +import java.io.Serializable; + import static org.picketlink.idm.model.basic.BasicModel.*; /** @@ -42,8 +45,9 @@ * @author dangleton * */ -@ApplicationScoped -public class InternalSecurity { +@Named +@SessionScoped +public class InternalSecurity implements Serializable { @Inject private IdentityManager identityManager; diff --git a/web/web_support/src/main/java/com/intuit/tank/config/LoginFilter.java b/web/web_support/src/main/java/com/intuit/tank/config/LoginFilter.java index 2ac9a5261..c135d5612 100644 --- a/web/web_support/src/main/java/com/intuit/tank/config/LoginFilter.java +++ b/web/web_support/src/main/java/com/intuit/tank/config/LoginFilter.java @@ -12,6 +12,11 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import com.amazonaws.xray.AWSXRay; +import com.amazonaws.xray.exceptions.SegmentNotFoundException; +import com.intuit.tank.vm.common.ThreadLocalUsernameProvider; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.picketlink.Identity; @@ -22,15 +27,13 @@ * */ public class LoginFilter implements Filter { + private static Logger LOG = LogManager.getLogger(LoginFilter.class); @Inject private Identity identity; @Override - public void destroy() { - // TODO Auto-generated method stub - - } + public void init(FilterConfig arg0) throws ServletException {} @Override public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) throws IOException, ServletException { @@ -39,13 +42,16 @@ public void doFilter(ServletRequest request, ServletResponse response, FilterCha String contextPath = ((HttpServletRequest)request).getContextPath(); ((HttpServletResponse)response).sendRedirect(contextPath + "/denied.xhtml"); } + try { + AWSXRay.getCurrentSegment().setUser(ThreadLocalUsernameProvider.getUsernameProvider().getUserName()); + } catch (SegmentNotFoundException snfe ) { //Ignore + } catch (Exception e) { + LOG.error("Failed to set User on current segment : " + e.getMessage(), e); + } chain.doFilter(request, response); } @Override - public void init(FilterConfig arg0) throws ServletException { - // TODO Auto-generated method stub - - } + public void destroy() {} } diff --git a/web/web_support/src/main/java/com/intuit/tank/prefs/TableViewState.java b/web/web_support/src/main/java/com/intuit/tank/prefs/TableViewState.java index 2772c5d71..ab6c734b7 100644 --- a/web/web_support/src/main/java/com/intuit/tank/prefs/TableViewState.java +++ b/web/web_support/src/main/java/com/intuit/tank/prefs/TableViewState.java @@ -22,6 +22,7 @@ import java.util.Map.Entry; import org.primefaces.event.data.FilterEvent; +import org.primefaces.model.FilterMeta; /** * FilterValues @@ -33,14 +34,13 @@ public class TableViewState implements Serializable { private static final long serialVersionUID = 1L; - private Map filteredValues = new HashMap(); + private Map filteredValues = new HashMap(); public Object getFilterValue(String key) { - Object ret = filteredValues.computeIfAbsent(key, k -> ""); - return ret; + return filteredValues.computeIfAbsent(key, k -> new FilterMeta()); } - public void setFilterValue(String key, Object value) { + public void setFilterValue(String key, FilterMeta value) { filteredValues.put(key, value); } @@ -53,9 +53,9 @@ public void clearFilters() { } public void onFilter(FilterEvent event) { - Map filters = event.getFilters(); + Map filters = event.getFilterBy(); clearFilters(); - for (Entry entry : filters.entrySet()) { + for (Entry entry : filters.entrySet()) { setFilterValue(entry.getKey(), entry.getValue()); } } diff --git a/web/web_support/src/main/java/com/intuit/tank/project/FileUploadBean.java b/web/web_support/src/main/java/com/intuit/tank/project/FileUploadBean.java index 9ea1a2863..4582b4823 100644 --- a/web/web_support/src/main/java/com/intuit/tank/project/FileUploadBean.java +++ b/web/web_support/src/main/java/com/intuit/tank/project/FileUploadBean.java @@ -33,7 +33,6 @@ import org.picketlink.idm.IdentityManager; import org.picketlink.idm.model.basic.User; import org.primefaces.event.FileUploadEvent; -import org.primefaces.model.UploadedFile; import com.intuit.tank.ModifiedDatafileMessage; import com.intuit.tank.config.TsLoggedIn; @@ -42,6 +41,7 @@ import com.intuit.tank.qualifier.Modified; import com.intuit.tank.util.UploadedFileIterator; import com.intuit.tank.wrapper.FileInputStreamWrapper; +import org.primefaces.model.file.UploadedFile; @Named @RequestScoped @@ -98,7 +98,7 @@ public void handleFileUpload(FileUploadEvent event) throws Exception { messages.error("Error extracting zip file: " + e.toString()); throw new RuntimeException(e); } finally { - IOUtils.closeQuietly(item.getInputstream()); + IOUtils.closeQuietly(item.getInputStream()); } } } diff --git a/web/web_support/src/main/java/com/intuit/tank/project/JobTreeTableBean.java b/web/web_support/src/main/java/com/intuit/tank/project/JobTreeTableBean.java index 7e978c7ce..ad20eccfa 100644 --- a/web/web_support/src/main/java/com/intuit/tank/project/JobTreeTableBean.java +++ b/web/web_support/src/main/java/com/intuit/tank/project/JobTreeTableBean.java @@ -38,9 +38,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import com.intuit.tank.util.Messages; -import org.picketlink.Identity; -import org.picketlink.idm.IdentityManager; -import org.picketlink.idm.model.basic.User; import org.primefaces.event.NodeCollapseEvent; import org.primefaces.event.NodeExpandEvent; import org.primefaces.model.DefaultTreeNode; @@ -96,12 +93,6 @@ public abstract class JobTreeTableBean implements Serializable { @Inject private Messages messages; - @Inject - private Identity identity; - - @Inject - private IdentityManager identityManager; - @Inject private Security security; @@ -134,7 +125,6 @@ public abstract class JobTreeTableBean implements Serializable { @PostConstruct public void init() { - AWSXRay.getCurrentSegment().setUser(identityManager.lookupById(User.class, identity.getAccount().getId()).getLoginName()); tablePrefs = new TablePreferences(preferencesBean.getPreferences().getJobsTableColumns()); tablePrefs.registerListener(preferencesBean); } diff --git a/web/web_support/src/main/java/com/intuit/tank/script/LogicStepEditor.java b/web/web_support/src/main/java/com/intuit/tank/script/LogicStepEditor.java index 1630773f6..6f6c5d2ae 100644 --- a/web/web_support/src/main/java/com/intuit/tank/script/LogicStepEditor.java +++ b/web/web_support/src/main/java/com/intuit/tank/script/LogicStepEditor.java @@ -37,7 +37,6 @@ import com.intuit.tank.http.BaseResponse; import com.intuit.tank.project.RequestData; import com.intuit.tank.project.ScriptStep; -import com.intuit.tank.script.ScriptConstants; import com.intuit.tank.tools.script.ScriptIOBean; import com.intuit.tank.tools.script.ScriptRunner; import com.intuit.tank.tools.script.StringOutputLogger; diff --git a/web/web_support/src/main/java/com/intuit/tank/script/ScriptBean.java b/web/web_support/src/main/java/com/intuit/tank/script/ScriptBean.java index 8256f8c43..9dcd70993 100644 --- a/web/web_support/src/main/java/com/intuit/tank/script/ScriptBean.java +++ b/web/web_support/src/main/java/com/intuit/tank/script/ScriptBean.java @@ -106,7 +106,6 @@ public void setSaveAsName(String saveAsName) { @PostConstruct public void init() { - AWSXRay.getCurrentSegment().setUser(identityManager.lookupById(User.class, identity.getAccount().getId()).getLoginName()); tablePrefs = new TablePreferences(userPrefs.getPreferences().getScriptsTableColumns()); stepTablePrefs = new TablePreferences(userPrefs.getPreferences().getScriptStepTableColumns()); diff --git a/web/web_support/src/main/java/com/intuit/tank/script/ScriptCreationBean.java b/web/web_support/src/main/java/com/intuit/tank/script/ScriptCreationBean.java index b5a1725e1..77ecec46a 100644 --- a/web/web_support/src/main/java/com/intuit/tank/script/ScriptCreationBean.java +++ b/web/web_support/src/main/java/com/intuit/tank/script/ScriptCreationBean.java @@ -33,7 +33,6 @@ import org.picketlink.Identity; import org.picketlink.idm.IdentityManager; import org.picketlink.idm.model.basic.User; -import org.primefaces.model.UploadedFile; import com.intuit.tank.ModifiedScriptMessage; import com.intuit.tank.auth.Security; @@ -53,6 +52,7 @@ import com.intuit.tank.vm.settings.AccessRight; import com.intuit.tank.wrapper.FileInputStreamWrapper; import com.intuit.tank.wrapper.SelectableWrapper; +import org.primefaces.model.file.UploadedFile; @Named @ConversationScoped @@ -254,7 +254,7 @@ private boolean validate() { messages.error("Script name cannot be empty. Please enter a script name."); } if (getCreationMode().equalsIgnoreCase("upload script")) { - if (getFile() == null || getFile().getContents().length == 0) { + if (getFile() == null || getFile().getContent().length == 0) { messages.error("Please select a script file."); } } diff --git a/web/web_support/src/main/java/com/intuit/tank/script/ScriptEditor.java b/web/web_support/src/main/java/com/intuit/tank/script/ScriptEditor.java index 881ef6654..db4eec149 100644 --- a/web/web_support/src/main/java/com/intuit/tank/script/ScriptEditor.java +++ b/web/web_support/src/main/java/com/intuit/tank/script/ScriptEditor.java @@ -151,7 +151,6 @@ public class ScriptEditor implements Serializable { @PostConstruct public void init() { - AWSXRay.getCurrentSegment().setUser(identityManager.lookupById(User.class, identity.getAccount().getId()).getLoginName()); tablePrefs = new TablePreferences(userPrefs.getPreferences().getScriptStepTableColumns()); tablePrefs.registerListener(userPrefs); } diff --git a/web/web_support/src/main/java/com/intuit/tank/script/TankXmlUploadBean.java b/web/web_support/src/main/java/com/intuit/tank/script/TankXmlUploadBean.java index 3525d0398..edaa20607 100644 --- a/web/web_support/src/main/java/com/intuit/tank/script/TankXmlUploadBean.java +++ b/web/web_support/src/main/java/com/intuit/tank/script/TankXmlUploadBean.java @@ -19,10 +19,13 @@ import java.io.InputStream; import java.io.Serializable; +import javax.enterprise.context.RequestScoped; import javax.enterprise.event.Event; import javax.inject.Inject; import javax.inject.Named; import javax.xml.bind.JAXBContext; +import javax.xml.bind.JAXBException; +import javax.xml.parsers.ParserConfigurationException; import javax.xml.parsers.SAXParserFactory; import javax.xml.transform.Source; import javax.xml.transform.sax.SAXSource; @@ -36,7 +39,7 @@ import org.picketlink.idm.IdentityManager; import org.picketlink.idm.model.basic.User; import org.primefaces.event.FileUploadEvent; -import org.primefaces.model.UploadedFile; +import org.primefaces.model.file.UploadedFile; import org.xml.sax.InputSource; import com.intuit.tank.ModifiedScriptMessage; @@ -48,14 +51,17 @@ import com.intuit.tank.qualifier.Modified; import com.intuit.tank.util.UploadedFileIterator; import com.intuit.tank.wrapper.FileInputStreamWrapper; +import org.xml.sax.SAXException; +import org.xml.sax.SAXNotRecognizedException; +import org.xml.sax.SAXNotSupportedException; @Named +@RequestScoped public class TankXmlUploadBean implements Serializable { + private static final Logger LOG = LogManager.getLogger(TankXmlUploadBean.class); private static final long serialVersionUID = 1L; - private static final Logger LOG = LogManager.getLogger(TankXmlUploadBean.class); - private boolean useFlash = true; @Inject @@ -68,7 +74,6 @@ public class TankXmlUploadBean implements Serializable { @Inject private IdentityManager identityManager; - @Inject private Messages messages; @@ -95,7 +100,7 @@ public void handleFileUpload(FileUploadEvent event) throws Exception { messages.error("Error extracting zip file: " + e.toString()); throw new RuntimeException(e); } finally { - IOUtils.closeQuietly(item.getInputstream()); + IOUtils.closeQuietly(item.getInputStream()); } } } @@ -104,7 +109,7 @@ public void clearDialog() { messages.clear(); } - public void processScript(InputStream inputStream, String fileName) throws Exception { + public void processScript(InputStream inputStream, String fileName) { try { ScriptDao dao = new ScriptDao(); @@ -141,9 +146,10 @@ public void processScript(InputStream inputStream, String fileName) throws Excep script.setCreator(identityManager.lookupById(User.class, identity.getAccount().getId()).getLoginName()); } script = dao.saveOrUpdate(script); + LOG.info("Script " + script.getName() + " from file " + fileName + " has been added."); messages.info("Script " + script.getName() + " from file " + fileName + " has been added."); scriptEvent.fire(new ModifiedScriptMessage(script, this)); - } catch (Exception e) { + } catch (ParserConfigurationException | JAXBException | SAXException e) { LOG.error("Error unmarshalling script: " + e.getMessage() + " from file " + fileName, e); messages.error("Error unmarshalling script: " + e.toString() + " from file " + fileName); } diff --git a/web/web_support/src/main/java/com/intuit/tank/script/TestUploadBean.java b/web/web_support/src/main/java/com/intuit/tank/script/TestUploadBean.java index b8148e56e..c3aa7c742 100644 --- a/web/web_support/src/main/java/com/intuit/tank/script/TestUploadBean.java +++ b/web/web_support/src/main/java/com/intuit/tank/script/TestUploadBean.java @@ -26,7 +26,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import com.intuit.tank.util.Messages; -import org.primefaces.model.UploadedFile; +import org.primefaces.model.file.UploadedFile; @Named @ApplicationScoped @@ -68,7 +68,7 @@ public String save() { parent.mkdirs(); File f = new File(parent, fileName); LOG.info("Writing file to " + f.getAbsolutePath()); - FileUtils.writeByteArrayToFile(f, item.getContents()); + FileUtils.writeByteArrayToFile(f, item.getContent()); messages.info("Wrote file to " + f.getAbsolutePath()); } catch (Exception e) { messages.error(e.getMessage()); diff --git a/web/web_support/src/main/java/com/intuit/tank/util/UploadedFileIterator.java b/web/web_support/src/main/java/com/intuit/tank/util/UploadedFileIterator.java index 7e88745b8..c4135acaa 100644 --- a/web/web_support/src/main/java/com/intuit/tank/util/UploadedFileIterator.java +++ b/web/web_support/src/main/java/com/intuit/tank/util/UploadedFileIterator.java @@ -26,9 +26,9 @@ import org.apache.commons.io.IOUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.primefaces.model.UploadedFile; import com.intuit.tank.wrapper.FileInputStreamWrapper; +import org.primefaces.model.file.UploadedFile; /** * @@ -50,7 +50,7 @@ public UploadedFileIterator(UploadedFile item, String... extension) throws IOExc super(); this.extension = extension; isZip = item.getFileName().toLowerCase().endsWith(".zip"); - itemInputStream = item.getInputstream(); + itemInputStream = item.getInputStream(); if (isZip) { try { in = new MyInputStream( diff --git a/web/web_support/src/main/java/com/intuit/tank/util/UserNameFilter.java b/web/web_support/src/main/java/com/intuit/tank/util/UserNameFilter.java index 2668ff58d..5eadeb512 100644 --- a/web/web_support/src/main/java/com/intuit/tank/util/UserNameFilter.java +++ b/web/web_support/src/main/java/com/intuit/tank/util/UserNameFilter.java @@ -42,17 +42,15 @@ public class UserNameFilter implements Filter { @Inject private IdentityManager identityManager; - @Override - public void init(FilterConfig filterConfig) throws ServletException { - // TODO Auto-generated method stub - } + public void init(FilterConfig filterConfig) throws ServletException {} @Override public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) throws IOException, ServletException { if (identity != null && identity.getAccount() != null && identityManager.lookupById(User.class, identity.getAccount().getId()) != null) { - ThreadLocalUsernameProvider.getUsernameProvider().setUserName(identityManager.lookupById(User.class, identity.getAccount().getId()).getLoginName()); + ThreadLocalUsernameProvider.getUsernameProvider().setUserName( + identityManager.lookupById(User.class, identity.getAccount().getId()).getLoginName()); } else { ThreadLocalUsernameProvider.getUsernameProvider().setUserName(null); } @@ -60,8 +58,6 @@ public void doFilter(ServletRequest request, ServletResponse response, FilterCha } @Override - public void destroy() { - // TODO Auto-generated method stub - } + public void destroy() {} } diff --git a/web/web_support/src/test/java/com/intuit/tank/prefs/TableViewStateTest.java b/web/web_support/src/test/java/com/intuit/tank/prefs/TableViewStateTest.java index 9d095f578..1455e8249 100644 --- a/web/web_support/src/test/java/com/intuit/tank/prefs/TableViewStateTest.java +++ b/web/web_support/src/test/java/com/intuit/tank/prefs/TableViewStateTest.java @@ -18,6 +18,7 @@ import org.junit.jupiter.api.Test; import com.intuit.tank.prefs.TableViewState; +import org.primefaces.model.FilterMeta; /** * The class TableViewStateTest contains tests for the class {@link TableViewState}. @@ -129,7 +130,7 @@ public void testSetFilterValue_1() throws Exception { TableViewState fixture = new TableViewState(); String key = ""; - String value = ""; + FilterMeta value = new FilterMeta(); fixture.setFilterValue(key, value); diff --git a/web/web_support/src/test/java/com/intuit/tank/script/ScriptCreationBeanTest.java b/web/web_support/src/test/java/com/intuit/tank/script/ScriptCreationBeanTest.java index b1e9d5fe2..dffc16a19 100644 --- a/web/web_support/src/test/java/com/intuit/tank/script/ScriptCreationBeanTest.java +++ b/web/web_support/src/test/java/com/intuit/tank/script/ScriptCreationBeanTest.java @@ -27,8 +27,8 @@ import org.picketlink.Identity; import org.picketlink.extension.PicketLinkExtension; -import org.primefaces.model.DefaultUploadedFile; -import org.primefaces.model.UploadedFile; +import org.primefaces.model.file.CommonsUploadedFile; +import org.primefaces.model.file.UploadedFile; import com.intuit.tank.project.ScriptFilter; import com.intuit.tank.project.ScriptFilterGroup; @@ -75,7 +75,7 @@ public void testScriptCreationBean_1() @Disabled public void testCancel_1() throws Exception { - scriptCreationBean.setFile(new DefaultUploadedFile()); + scriptCreationBean.setFile(new CommonsUploadedFile()); scriptCreationBean.setProductName(""); scriptCreationBean.setCreationMode(""); scriptCreationBean.setFilterWrappers(new LinkedList<>()); @@ -100,7 +100,7 @@ public void testCancel_1() @Disabled public void testCreateNewScript_1() throws Exception { - scriptCreationBean.setFile(new DefaultUploadedFile()); + scriptCreationBean.setFile(new CommonsUploadedFile()); scriptCreationBean.setProductName(""); scriptCreationBean.setCreationMode(""); scriptCreationBean.setFilterWrappers(new LinkedList<>()); @@ -125,7 +125,7 @@ public void testCreateNewScript_1() @Test public void testGetCreationMode_1() throws Exception { - scriptCreationBean.setFile(new DefaultUploadedFile()); + scriptCreationBean.setFile(new CommonsUploadedFile()); scriptCreationBean.setProductName(""); scriptCreationBean.setCreationMode(""); scriptCreationBean.setFilterWrappers(new LinkedList<>()); @@ -150,7 +150,7 @@ public void testGetCreationMode_1() @Test public void testGetFile_1() throws Exception { - scriptCreationBean.setFile(new DefaultUploadedFile()); + scriptCreationBean.setFile(new CommonsUploadedFile()); scriptCreationBean.setProductName(""); scriptCreationBean.setCreationMode(""); scriptCreationBean.setFilterWrappers(new LinkedList<>()); @@ -176,7 +176,7 @@ public void testGetFile_1() @Test public void testGetFilterWrappers_2() throws Exception { - scriptCreationBean.setFile(new DefaultUploadedFile()); + scriptCreationBean.setFile(new CommonsUploadedFile()); scriptCreationBean.setProductName(""); scriptCreationBean.setCreationMode(""); scriptCreationBean.setFilterWrappers(new LinkedList<>()); @@ -202,7 +202,7 @@ public void testGetFilterWrappers_2() @Test public void testGetGroupWrappers_2() throws Exception { - scriptCreationBean.setFile(new DefaultUploadedFile()); + scriptCreationBean.setFile(new CommonsUploadedFile()); scriptCreationBean.setProductName(""); scriptCreationBean.setCreationMode(""); scriptCreationBean.setFilterWrappers(new LinkedList<>()); @@ -227,7 +227,7 @@ public void testGetGroupWrappers_2() @Test public void testGetName_1() throws Exception { - scriptCreationBean.setFile(new DefaultUploadedFile()); + scriptCreationBean.setFile(new CommonsUploadedFile()); scriptCreationBean.setProductName(""); scriptCreationBean.setCreationMode(""); scriptCreationBean.setFilterWrappers(new LinkedList<>()); @@ -252,7 +252,7 @@ public void testGetName_1() @Test public void testGetProductName_1() throws Exception { - scriptCreationBean.setFile(new DefaultUploadedFile()); + scriptCreationBean.setFile(new CommonsUploadedFile()); scriptCreationBean.setProductName(""); scriptCreationBean.setCreationMode(""); scriptCreationBean.setFilterWrappers(new LinkedList<>()); @@ -277,7 +277,7 @@ public void testGetProductName_1() @Test public void testSetCreationMode_1() throws Exception { - scriptCreationBean.setFile(new DefaultUploadedFile()); + scriptCreationBean.setFile(new CommonsUploadedFile()); scriptCreationBean.setProductName(""); scriptCreationBean.setCreationMode(""); scriptCreationBean.setFilterWrappers(new LinkedList<>()); @@ -302,13 +302,13 @@ public void testSetCreationMode_1() @Test public void testSetFile_1() throws Exception { - scriptCreationBean.setFile(new DefaultUploadedFile()); + scriptCreationBean.setFile(new CommonsUploadedFile()); scriptCreationBean.setProductName(""); scriptCreationBean.setCreationMode(""); scriptCreationBean.setFilterWrappers(new LinkedList<>()); scriptCreationBean.setName(""); scriptCreationBean.setGroupWrappers(new LinkedList<>()); - UploadedFile file = new DefaultUploadedFile(); + UploadedFile file = new CommonsUploadedFile(); scriptCreationBean.setFile(file); @@ -327,7 +327,7 @@ public void testSetFile_1() @Test public void testSetFilterWrappers_1() throws Exception { - scriptCreationBean.setFile(new DefaultUploadedFile()); + scriptCreationBean.setFile(new CommonsUploadedFile()); scriptCreationBean.setProductName(""); scriptCreationBean.setCreationMode(""); scriptCreationBean.setFilterWrappers(new LinkedList<>()); @@ -352,7 +352,7 @@ public void testSetFilterWrappers_1() @Test public void testSetGroupWrappers_1() throws Exception { - scriptCreationBean.setFile(new DefaultUploadedFile()); + scriptCreationBean.setFile(new CommonsUploadedFile()); scriptCreationBean.setProductName(""); scriptCreationBean.setCreationMode(""); scriptCreationBean.setFilterWrappers(new LinkedList<>()); @@ -377,7 +377,7 @@ public void testSetGroupWrappers_1() @Test public void testSetName_1() throws Exception { - scriptCreationBean.setFile(new DefaultUploadedFile()); + scriptCreationBean.setFile(new CommonsUploadedFile()); scriptCreationBean.setProductName(""); scriptCreationBean.setCreationMode(""); scriptCreationBean.setFilterWrappers(new LinkedList<>()); @@ -402,7 +402,7 @@ public void testSetName_1() @Test public void testSetProductName_1() throws Exception { - scriptCreationBean.setFile(new DefaultUploadedFile()); + scriptCreationBean.setFile(new CommonsUploadedFile()); scriptCreationBean.setProductName(""); scriptCreationBean.setCreationMode(""); scriptCreationBean.setFilterWrappers(new LinkedList<>()); diff --git a/web/web_support/src/test/java/com/intuit/tank/script/TankXmlUploadBeanTest.java b/web/web_support/src/test/java/com/intuit/tank/script/TankXmlUploadBeanTest.java index 74863238d..2f58e7749 100644 --- a/web/web_support/src/test/java/com/intuit/tank/script/TankXmlUploadBeanTest.java +++ b/web/web_support/src/test/java/com/intuit/tank/script/TankXmlUploadBeanTest.java @@ -13,22 +13,10 @@ * #L% */ -import java.io.InputStream; -import java.io.PipedInputStream; - -import javax.faces.component.UIComponent; - import org.junit.jupiter.api.*; import static org.junit.jupiter.api.Assertions.*; -import org.primefaces.event.FileUploadEvent; -import org.primefaces.extensions.component.dynaform.DynaForm; -import org.primefaces.model.DefaultUploadedFile; -import org.primefaces.model.UploadedFile; - -import com.intuit.tank.script.TankXmlUploadBean; - /** * The class TankXmlUploadBeanTest contains tests for the class * {@link TankXmlUploadBean}. diff --git a/web/web_support/src/test/java/com/intuit/tank/script/TestUploadBeanTest.java b/web/web_support/src/test/java/com/intuit/tank/script/TestUploadBeanTest.java index c63895d68..0171b46c4 100644 --- a/web/web_support/src/test/java/com/intuit/tank/script/TestUploadBeanTest.java +++ b/web/web_support/src/test/java/com/intuit/tank/script/TestUploadBeanTest.java @@ -18,8 +18,8 @@ import static org.junit.jupiter.api.Assertions.*; -import org.primefaces.model.DefaultUploadedFile; -import org.primefaces.model.UploadedFile; +import org.primefaces.model.file.CommonsUploadedFile; +import org.primefaces.model.file.UploadedFile; import com.intuit.tank.script.TestUploadBean; @@ -56,7 +56,7 @@ public void testTestUploadBean_1() @Test public void testGetFile_1() throws Exception { - testUploadBean.setFile(new DefaultUploadedFile()); + testUploadBean.setFile(new CommonsUploadedFile()); UploadedFile result = testUploadBean.getFile(); @@ -77,8 +77,8 @@ public void testGetFile_1() @Test public void testSetFile_1() throws Exception { - testUploadBean.setFile(new DefaultUploadedFile()); - UploadedFile file = new DefaultUploadedFile(); + testUploadBean.setFile(new CommonsUploadedFile()); + UploadedFile file = new CommonsUploadedFile(); testUploadBean.setFile(file); diff --git a/web/web_ui/pom.xml b/web/web_ui/pom.xml index 0ebde23c7..fce79a9b2 100644 --- a/web/web_ui/pom.xml +++ b/web/web_ui/pom.xml @@ -6,7 +6,7 @@ com.intuit.tank web-parent - 2.3.3 + 2.3.4 tank @@ -303,7 +303,7 @@ - jdom + org.jdom jdom diff --git a/web/web_ui/src/main/webapp/WEB-INF/.faces-config.xml.jsfdia b/web/web_ui/src/main/webapp/WEB-INF/.faces-config.xml.jsfdia deleted file mode 100644 index 7dc56a3d7..000000000 --- a/web/web_ui/src/main/webapp/WEB-INF/.faces-config.xml.jsfdia +++ /dev/null @@ -1,286 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/web/web_ui/src/main/webapp/WEB-INF/web.xml b/web/web_ui/src/main/webapp/WEB-INF/web.xml index 342433b3d..49bede8b3 100644 --- a/web/web_ui/src/main/webapp/WEB-INF/web.xml +++ b/web/web_ui/src/main/webapp/WEB-INF/web.xml @@ -43,14 +43,25 @@ PrimeFaces FileUpload Filter Faces Servlet + + - UserName Filter - com.intuit.tank.util.UserNameFilter - - - RestSecurity Filter - com.intuit.tank.util.RestSecurityFilter + AWSXRayServletFilter + com.amazonaws.xray.javax.servlet.AWSXRayServletFilter + + dynamicNamingRecognizedHosts + *.perf.a.intuit.com + + + dynamicNamingFallbackName + Tank + + + AWSXRayServletFilter + * + + LoginFilter @@ -86,30 +97,25 @@ /tools/* - - AWSXRayServletFilter - com.amazonaws.xray.javax.servlet.AWSXRayServletFilter - - dynamicNamingRecognizedHosts - *.perf.a.intuit.com - - - dynamicNamingFallbackName - Tank - + UserName Filter + com.intuit.tank.util.UserNameFilter - AWSXRayServletFilter - * + UserName Filter + Faces Servlet - Faces Servlet javax.faces.webapp.FacesServlet 1 + + + Faces Servlet + *.jsf + Intuit Tank REST API @@ -124,26 +130,19 @@ 1 - - Faces Servlet - *.jsf + Intuit Tank REST API + /rest/* + + RestSecurity Filter + com.intuit.tank.util.RestSecurityFilter + RestSecurity Filter Intuit Tank REST API - - - UserName Filter - Faces Servlet - - - - Intuit Tank REST API - /rest/* - org.jboss.weld.context.NonexistentConversationException @@ -171,4 +170,4 @@ javax.enterprise.inject.spi.BeanManager - + \ No newline at end of file diff --git a/web/web_ui/src/main/webapp/admin/logs.xhtml b/web/web_ui/src/main/webapp/admin/logs.xhtml index 2dbdb28e6..652de14d0 100644 --- a/web/web_ui/src/main/webapp/admin/logs.xhtml +++ b/web/web_ui/src/main/webapp/admin/logs.xhtml @@ -27,15 +27,19 @@
    Initial Lines: - + + +
    Poll seconds: - + + +
    diff --git a/web/web_ui/src/main/webapp/datafiles/index.xhtml b/web/web_ui/src/main/webapp/datafiles/index.xhtml index 48cd492ca..69962dd72 100644 --- a/web/web_ui/src/main/webapp/datafiles/index.xhtml +++ b/web/web_ui/src/main/webapp/datafiles/index.xhtml @@ -140,7 +140,7 @@ xmlns:ui="http://xmlns.jcp.org/jsf/facelets"
    - diff --git a/web/web_ui/src/main/webapp/projects/associateFiles.xhtml b/web/web_ui/src/main/webapp/projects/associateFiles.xhtml index 9afbd5008..ed1c9d3fc 100755 --- a/web/web_ui/src/main/webapp/projects/associateFiles.xhtml +++ b/web/web_ui/src/main/webapp/projects/associateFiles.xhtml @@ -12,7 +12,7 @@
    - diff --git a/web/web_ui/src/main/webapp/projects/index.xhtml b/web/web_ui/src/main/webapp/projects/index.xhtml index 9907313a9..53e5e5000 100644 --- a/web/web_ui/src/main/webapp/projects/index.xhtml +++ b/web/web_ui/src/main/webapp/projects/index.xhtml @@ -40,9 +40,8 @@ diff --git a/web/web_ui/src/main/webapp/scripts/index.xhtml b/web/web_ui/src/main/webapp/scripts/index.xhtml index f4407e5bb..0059d321b 100644 --- a/web/web_ui/src/main/webapp/scripts/index.xhtml +++ b/web/web_ui/src/main/webapp/scripts/index.xhtml @@ -59,13 +59,10 @@ - @@ -76,12 +73,10 @@ - - @@ -251,11 +246,9 @@
    - +
    diff --git a/web/web_ui/src/main/webapp/scripts/script-edit-view.xhtml b/web/web_ui/src/main/webapp/scripts/script-edit-view.xhtml index ab8820398..307edb4cb 100644 --- a/web/web_ui/src/main/webapp/scripts/script-edit-view.xhtml +++ b/web/web_ui/src/main/webapp/scripts/script-edit-view.xhtml @@ -209,7 +209,6 @@ - @@ -274,20 +273,17 @@ - -