Develop

apps/dashboard/web/polaris_web/web/src/util/request.js

@@ -99,8 +99,9 @@ service.interceptors.response.use((response) => {
   return response.data
 }, err)
 
+const black_list_apis = ['dashboard/accesstoken', 'api/fetchBurpPluginInfo', 'api/fetchActiveLoaders', 'api/fetchAllSubCategories']
 async function raiseMixpanelEvent(api) {
-  if (api && api.indexOf("/api/fetchActiveLoaders") == -1) {
+  if (api && !black_list_apis.some(black_list_api => api.includes(black_list_api))) {
     window.mixpanel.track(api)
   }
 }

apps/dashboard/web/src/util/request.js

@@ -135,9 +135,10 @@ service.interceptors.response.use((response) => {
   return response.data
 }, err)
 
+const black_list_apis = ['dashboard/accesstoken', 'api/fetchBurpPluginInfo', 'api/fetchActiveLoaders', 'api/fetchAllSubCategories']
 async function raiseMixpanelEvent(api){
-  if (api && api.indexOf("/api/fetchActiveLoaders")==-1) {
-    window.mixpanel.track(api)
+  if (api && !black_list_apis.some(black_list_api => api.includes(black_list_api))) {
+    window.mixpanel.track(api);
   }
 }
 

apps/testing/src/main/java/com/akto/test_editor/execution/Executor.java

@@ -50,6 +50,8 @@ public List<TestResult> execute(ExecutorNode node, RawApi rawApi, Map<String, Ob
 
         boolean requestSent = false;
 
+        List<String> error_messages = new ArrayList<>();
+
         for (ExecutorNode reqNode: reqNodes.getChildNodes()) {
             // make copy of varMap as well
             List<RawApi> sampleRawApis = new ArrayList<>();
@@ -59,6 +61,7 @@ public List<TestResult> execute(ExecutorNode node, RawApi rawApi, Map<String, Ob
             List<RawApi> testRawApis = new ArrayList<>();
             testRawApis = singleReq.getRawApis();
             if (testRawApis == null) {
+                error_messages.add(singleReq.getErrMsg());
                 continue;
             }
             boolean vulnerable = false;
@@ -77,17 +80,19 @@ public List<TestResult> execute(ExecutorNode node, RawApi rawApi, Map<String, Ob
                     }
                     vulnerable = res.getVulnerable();
                 } catch(Exception e) {
-                    loggerMaker.errorAndAddToDb("error executing test request " + logId + " " + e.getMessage(), LogDb.TESTING);
+                    error_messages.add("Error executing test request: " + e.getMessage());
+                    loggerMaker.errorAndAddToDb("Error executing test request " + logId + " " + e.getMessage(), LogDb.TESTING);
                 }
             }
         }
 
         if(result.isEmpty()){
             if(requestSent){
-                result.add(new TestResult(null, rawApi.getOriginalMessage(), Collections.singletonList(TestError.API_REQUEST_FAILED.getMessage()), 0, false, TestResult.Confidence.HIGH, null));
+                error_messages.add(TestError.API_REQUEST_FAILED.getMessage());
             } else {
-                result.add(new TestResult(null, rawApi.getOriginalMessage(), Collections.singletonList(TestError.NO_API_REQUEST.getMessage()), 0, false, TestResult.Confidence.HIGH, null));
+                error_messages.add(TestError.NO_API_REQUEST.getMessage());
             }
+            result.add(new TestResult(null, rawApi.getOriginalMessage(), error_messages, 0, false, TestResult.Confidence.HIGH, null));
         }
 
         return result;

apps/testing/src/main/java/com/akto/testing/ApiExecutor.java

@@ -21,6 +21,9 @@
 public class ApiExecutor {
     private static final LoggerMaker loggerMaker = new LoggerMaker(ApiExecutor.class);
 
+    // Load only first 1 MiB of response body into memory.
+    private static final int MAX_RESPONSE_SIZE = 1024*1024;
+
     private static OriginalHttpResponse common(Request request, boolean followRedirects) throws Exception {
 
         Integer accountId = Context.accountId.get();
@@ -51,7 +54,7 @@ private static OriginalHttpResponse common(Request request, boolean followRedire
         String body;
         try {
             response = call.execute();
-            ResponseBody responseBody = response.body();
+            ResponseBody responseBody = response.peekBody(MAX_RESPONSE_SIZE);
             if (responseBody == null) {
                 throw new Exception("Couldn't read response body");
             }

apps/testing/src/main/java/com/akto/testing/TestExecutor.java

@@ -36,9 +36,13 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.gson.Gson;
 import com.mongodb.BasicDBObject;
+import com.mongodb.client.model.Aggregates;
 import com.mongodb.client.model.Filters;
 import com.mongodb.client.model.Projections;
+import com.mongodb.client.model.Sorts;
 import com.mongodb.client.model.Updates;
+
+import org.bson.conversions.Bson;
 import org.bson.types.ObjectId;
 import org.json.JSONObject;
 import org.mortbay.util.ajax.JSON;
@@ -110,7 +114,7 @@ public void workflowInit (TestingRun testingRun, ObjectId summaryId) {
     public void apiWiseInit(TestingRun testingRun, ObjectId summaryId) {
         int accountId = Context.accountId.get();
         int now = Context.now();
-        int maxConcurrentRequests = testingRun.getMaxConcurrentRequests() > 0 ? testingRun.getMaxConcurrentRequests() : 100;
+        int maxConcurrentRequests = testingRun.getMaxConcurrentRequests() > 0 ? Math.min( testingRun.getMaxConcurrentRequests(), 100) : 10;
         TestingEndpoints testingEndpoints = testingRun.getTestingEndpoints();
 
         SampleMessageStore sampleMessageStore = SampleMessageStore.create();
@@ -165,7 +169,7 @@ public void apiWiseInit(TestingRun testingRun, ObjectId summaryId) {
 
         CountDownLatch latch = new CountDownLatch(apiInfoKeyList.size());
         ExecutorService threadPool = Executors.newFixedThreadPool(maxConcurrentRequests);
-        List<Future<List<TestingRunResult>>> futureTestingRunResults = new ArrayList<>();
+        List<Future<Void>> futureTestingRunResults = new ArrayList<>();
         Map<String, Integer> hostsToApiCollectionMap = new HashMap<>();
 
         ConcurrentHashMap<String, String> subCategoryEndpointMap = new ConcurrentHashMap<>();
@@ -200,7 +204,7 @@ public void apiWiseInit(TestingRun testingRun, ObjectId summaryId) {
                 loggerMaker.errorAndAddToDb("Error while finding host: " + e, LogDb.TESTING);
             }
             try {
-                 Future<List<TestingRunResult>> future = threadPool.submit(
+                 Future<Void> future = threadPool.submit(
                          () -> startWithLatch(apiInfoKey,
                                  testingRun.getTestIdConfig(),
                                  testingRun.getId(),testingRun.getTestingRunConfig(), testingUtil, summaryId,
@@ -225,20 +229,6 @@ public void apiWiseInit(TestingRun testingRun, ObjectId summaryId) {
 
         loggerMaker.infoAndAddToDb("Finished testing", LogDb.TESTING);
 
-        int totalResults = 0;
-        for (Future<List<TestingRunResult>> future: futureTestingRunResults) {
-            if (!future.isDone()) continue;
-            try {
-                if (!future.get().isEmpty()) {
-                    int resultSize = future.get().size();
-                    totalResults += resultSize;
-                }
-            } catch (InterruptedException | ExecutionException e) {
-                loggerMaker.errorAndAddToDb("Error while after running test : " + e, LogDb.TESTING);
-            }
-        }
-
-        loggerMaker.infoAndAddToDb("Finished adding " + totalResults + " testingRunResults", LogDb.TESTING);
     }
 
     public static void updateTestSummary(ObjectId summaryId){
@@ -269,8 +259,7 @@ public static void updateTestSummary(ObjectId summaryId){
                                     Filters.eq(TestingRunResult.VULNERABLE, true)),
                             limit,
                             skip,
-                            Projections.include(
-                                    TestingRunResult.TEST_RESULTS));
+                            Projections.include("testResults.confidence"));
 
             loggerMaker.infoAndAddToDb("Reading " + testingRunResults.size() + " vulnerable testingRunResults",
                     LogDb.TESTING);
@@ -463,7 +452,7 @@ public Map<String, Object> generateResponseMap(String payloadStr, Map<String, Li
         return respMap;
     }
 
-    public List<TestingRunResult> startWithLatch(
+    public Void startWithLatch(
             ApiInfo.ApiInfoKey apiInfoKey, int testIdConfig, ObjectId testRunId, TestingRunConfig testingRunConfig,
             TestingUtil testingUtil, ObjectId testRunResultSummaryId, int accountId, CountDownLatch latch, int startTime,
             int timeToKill, Map<String, TestConfig> testConfigMap, TestingRun testingRun, 
@@ -472,35 +461,20 @@ public List<TestingRunResult> startWithLatch(
         loggerMaker.infoAndAddToDb("Starting test for " + apiInfoKey, LogDb.TESTING);
 
         Context.accountId.set(accountId);
-        List<TestingRunResult> testingRunResults = new ArrayList<>();
         int now = Context.now();
         if ( timeToKill <= 0 || now - startTime <= timeToKill) {
             try {
                 // todo: commented out older one
 //                testingRunResults = start(apiInfoKey, testIdConfig, testRunId, testingRunConfig, testingUtil, testRunResultSummaryId, testConfigMap);
-                testingRunResults = startTestNew(apiInfoKey, testRunId, testingRunConfig, testingUtil, testRunResultSummaryId, testConfigMap, subCategoryEndpointMap, apiInfoKeyToHostMap);
-                String size = testingRunResults.size()+"";
-                loggerMaker.infoAndAddToDb("testingRunResults size: " + size, LogDb.TESTING);
-                if (!testingRunResults.isEmpty()) {
-                    trim(testingRunResults);
-                    TestingRunResultDao.instance.insertMany(testingRunResults);
-                    loggerMaker.infoAndAddToDb("Inserted testing results", LogDb.TESTING);
-                    //Creating issues from testingRunResults
-                   TestingIssuesHandler handler = new TestingIssuesHandler();
-                   boolean triggeredByTestEditor = false;
-                   if (testingRun.getTriggeredBy() != null) {
-                        triggeredByTestEditor = testingRun.getTriggeredBy().equals("test_editor");
-                   }
-                   handler.handleIssuesCreationFromTestingRunResults(testingRunResults, triggeredByTestEditor); // pass new field here
-                }
+                startTestNew(apiInfoKey, testRunId, testingRunConfig, testingUtil, testRunResultSummaryId, testConfigMap, subCategoryEndpointMap, apiInfoKeyToHostMap);
             } catch (Exception e) {
                 e.printStackTrace();
                 loggerMaker.errorAndAddToDb("error while running tests: " + e, LogDb.TESTING);
             }
         }
 
         latch.countDown();
-        return testingRunResults;
+        return null;
     }
 
     public static void trim(TestingRunResult testingRunResult) {
@@ -535,15 +509,30 @@ public void trim(List<TestingRunResult> testingRunResults) {
         }
     }
 
-    public List<TestingRunResult> startTestNew(ApiInfo.ApiInfoKey apiInfoKey, ObjectId testRunId,
+    public void insertResultsAndMakeIssues(List<TestingRunResult> testingRunResults) {
+        int resultSize = testingRunResults.size();
+        if (resultSize > 0) {
+            loggerMaker.infoAndAddToDb("testingRunResults size: " + resultSize, LogDb.TESTING);
+            trim(testingRunResults);
+            TestingRunResultDao.instance.insertMany(testingRunResults);
+            loggerMaker.infoAndAddToDb("Inserted testing results", LogDb.TESTING);
+            TestingIssuesHandler handler = new TestingIssuesHandler();
+            boolean triggeredByTestEditor = false;
+            handler.handleIssuesCreationFromTestingRunResults(testingRunResults, triggeredByTestEditor);
+            testingRunResults.clear();
+        }
+    }
+
+    public void startTestNew(ApiInfo.ApiInfoKey apiInfoKey, ObjectId testRunId,
                                                TestingRunConfig testingRunConfig, TestingUtil testingUtil,
                                                ObjectId testRunResultSummaryId, Map<String, TestConfig> testConfigMap,
                                                ConcurrentHashMap<String, String> subCategoryEndpointMap, Map<ApiInfoKey, String> apiInfoKeyToHostMap) {
-        List<TestingRunResult> testingRunResults = new ArrayList<>();
 
         List<String> testSubCategories = testingRunConfig == null ? new ArrayList<>() : testingRunConfig.getTestSubCategoryList();
 
         for (String testSubCategory: testSubCategories) {
+            List<TestingRunResult> testingRunResults = new ArrayList<>();
+
             TestConfig testConfig = testConfigMap.get(testSubCategory);
             if (testConfig == null) continue;
             TestingRunResult testingRunResult = null;
@@ -557,9 +546,10 @@ public List<TestingRunResult> startTestNew(ApiInfo.ApiInfoKey apiInfoKey, Object
                 e.printStackTrace();
             }
             if (testingRunResult != null) testingRunResults.add(testingRunResult);
+
+            insertResultsAndMakeIssues(testingRunResults);
         }
 
-        return testingRunResults;
     }
 
     public boolean applyRunOnceCheck(ApiInfoKey apiInfoKey, TestConfig testConfig, ConcurrentHashMap<String, String> subCategoryEndpointMap, Map<ApiInfoKey, String> apiInfoKeyToHostMap, String testSubCategory) {

libs/dao/src/main/java/com/akto/dao/testing/TestingRunResultDao.java

@@ -1,6 +1,7 @@
 package com.akto.dao.testing;
 
 import com.akto.dao.AccountsContextDao;
+import com.akto.dao.MCollection;
 import com.akto.dao.context.Context;
 import com.akto.dto.ApiInfo;
 import com.akto.dto.testing.TestingRunResult;
@@ -87,6 +88,8 @@ public void createIndicesIfAbsent() {
         Bson summaryIndex = Indexes.descending(Arrays.asList(TestingRunResult.TEST_RUN_RESULT_SUMMARY_ID, Constants.ID));
         createIndexIfAbsent(dbName, getCollName(), summaryIndex, new IndexOptions().name("testRunResultSummaryId_-1__id_-1"));
 
+        MCollection.createIndexIfAbsent(getDBName(), getCollName(),
+                new String[] { TestingRunResult.TEST_RUN_RESULT_SUMMARY_ID, TestingRunResult.VULNERABLE, Constants.ID }, false);
     }
 
 }

libs/dao/src/main/java/com/akto/dto/testing/GenericTestResult.java

@@ -5,6 +5,7 @@
 public class GenericTestResult {
 
     private boolean vulnerable;
+    public static final String _CONFIDENCE = "confidence";
     private Confidence confidence = Confidence.HIGH;
 
     public GenericTestResult() {