Merge pull request #199 from JupiterOne/INT-3904-fix-webapp-date-filter

Int 3904 fix webapp date filter
JupiterOne-Archives · Jun 1, 2022 · ff6a0fd · ff6a0fd
2 parents b7c7b57 + 5c9d218
commit ff6a0fd
Show file tree

Hide file tree

Showing 11 changed files with 677 additions and 479 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,6 +2,14 @@
 
 ## Unreleased
 
+## [5.11.9] - 2022-06-01
+
+### Fixed
+
+- Updated the filter for web app scans and web app findings to use `GREATER`
+  operator when querying using the last since date. Previously this was using
+  `EQUAL` which caused no data to be ingested for those steps
+
 ## [5.11.8] - 2022-05-24
 
 ## Added

diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@jupiterone/graph-qualys",
-  "version": "5.11.8",
+  "version": "5.11.9",
   "description": "Qualys integration for JupiterOne",
   "license": "MPL-2.0",
   "main": "src/index.js",

diff --git a/src/provider/__recordings__/fetchScannedWebAppIds_144896138/recording.har b/src/provider/__recordings__/fetchScannedWebAppIds_144896138/recording.har
@@ -4,15 +4,15 @@
     "creator": {
       "comment": "persister:JupiterOneIntegationFSPersister",
       "name": "Polly.JS",
-      "version": "4.3.0"
+      "version": "5.1.1"
     },
     "entries": [
       {
         "_id": "d32c77422391af117fcbeb901e925389",
         "_order": 0,
         "cache": {},
         "request": {
-          "bodySize": 285,
+          "bodySize": 228,
           "cookies": [],
           "headers": [
             {
@@ -38,7 +38,7 @@
             {
               "_fromType": "array",
               "name": "content-length",
-              "value": "285"
+              "value": "228"
             },
             {
               "_fromType": "array",
@@ -60,23 +60,23 @@
               "value": "qualysapi.qg3.apps.qualys.com"
             }
           ],
-          "headersSize": 398,
+          "headersSize": 390,
           "httpVersion": "HTTP/1.1",
           "method": "POST",
           "postData": {
             "mimeType": "text/xml",
             "params": [],
-            "text": "\n        <ServiceRequest>\n          <preferences>\n            <limitResults>100</limitResults>\n            <startFromOffset>1</startFromOffset>\n          </preferences>\n          <filters><Criteria field=\"isScanned\" operator=\"EQUALS\">true</Criteria></filters>\n        </ServiceRequest>"
+            "text": "<ServiceRequest>\n  <preferences>\n    <limitResults>100</limitResults>\n    <startFromOffset>1</startFromOffset>\n  </preferences>\n  <filters><Criteria field=\"isScanned\" operator=\"EQUALS\">true</Criteria></filters>\n</ServiceRequest>"
           },
           "queryString": [],
           "url": "https://qualysapi.qg3.apps.qualys.com/qps/rest/3.0/search/was/webapp"
         },
         "response": {
-          "bodySize": 854,
+          "bodySize": 1382,
           "content": {
             "mimeType": "application/xml",
-            "size": 854,
-            "text": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<ServiceResponse xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:noNamespaceSchemaLocation=\"https://qualysapi.qg3.apps.qualys.com/qps/xsd/3.0/was/webapp.xsd\">\n  <responseCode>SUCCESS</responseCode>\n  <count>1</count>\n  <hasMoreRecords>false</hasMoreRecords>\n  <data>\n    <WebApp>\n      <id>81221901</id>\n      <name><![CDATA[Ask JupiterOne]]></name>\n      <url><![CDATA[http://ask.us.jupiterone.io]]></url>\n      <owner>\n        <id>10748728</id>\n      </owner>\n      <tags>\n        <count>0</count>\n      </tags>\n      <createdDate>2020-09-10T00:01:33Z</createdDate>\n      <updatedDate>2020-09-10T00:29:13Z</updatedDate>\n    </WebApp>\n  </data>\n</ServiceResponse>"
+            "size": 1382,
+            "text": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<ServiceResponse xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:noNamespaceSchemaLocation=\"https://qualysapi.qg3.apps.qualys.com/qps/xsd/3.0/was/webapp.xsd\">\n  <responseCode>SUCCESS</responseCode>\n  <count>7</count>\n  <hasMoreRecords>false</hasMoreRecords>\n  <data>\n    <WebApp>\n      <id>238483924</id>\n      <name><![CDATA[http://rest.vulnweb.com/]]></name>\n      <url><![CDATA[http://rest.vulnweb.com/]]></url>\n      <owner>\n        <id>17729931</id>\n      </owner>\n      <tags>\n        <count>0</count>\n      </tags>\n      <createdDate>2022-04-12T21:12:52Z</createdDate>\n      <updatedDate>2022-04-12T21:39:40Z</updatedDate>\n    </WebApp>\n    <WebApp>\n      <id>238483925</id>\n      <name><![CDATA[http://testhtml5.vulnweb.com]]></name>\n      <url><![CDATA[http://testhtml5.vulnweb.com]]></url>\n      <owner>\n        <id>17729931</id>\n      </owner>\n      <tags>\n        <count>0</count>\n      </tags>\n      <createdDate>2022-04-12T21:13:24Z</createdDate>\n      <updatedDate>2022-04-12T21:14:51Z</updatedDate>\n    </WebApp>\n    <WebApp>\n      <id>238483926</id>\n      <name><![CDATA[http://testphp.vulnweb.com/]]></name>\n      <url><![CDATA[http://testphp.vulnweb.com/]]></url>\n      <owner>\n        <id>17729931</id>\n      </owner>\n      <tags>\n        <count>0</count>\n      </tags>\n      <createdDate>2022-04-12T21:16:33Z</createdDate>\n      <updatedDate>2022-04-12T21:20:39Z</updatedDate>\n    </WebApp>\n    <WebApp>\n      <id>238483927</id>\n      <name><![CDATA[http://testasp.vulnweb.com]]></name>\n      <url><![CDATA[http://testasp.vulnweb.com]]></url>\n      <owner>\n        <id>17729931</id>\n      </owner>\n      <tags>\n        <count>0</count>\n      </tags>\n      <createdDate>2022-04-12T21:16:58Z</createdDate>\n      <updatedDate>2022-04-12T21:23:51Z</updatedDate>\n    </WebApp>\n    <WebApp>\n      <id>238483928</id>\n      <name><![CDATA[http://testaspnet.vulnweb.com]]></name>\n      <url><![CDATA[http://testaspnet.vulnweb.com]]></url>\n      <owner>\n        <id>17729931</id>\n      </owner>\n      <tags>\n        <count>0</count>\n      </tags>\n      <createdDate>2022-04-12T21:18:09Z</createdDate>\n      <updatedDate>2022-04-12T21:22:51Z</updatedDate>\n    </WebApp>\n    <WebApp>\n      <id>251316885</id>\n      <name><![CDATA[Juice Shop]]></name>\n      <url><![CDATA[http://d218uuknkwwh79.cloudfront.net/#/]]></url>\n      <owner>\n        <id>17729931</id>\n      </owner>\n      <tags>\n        <count>0</count>\n      </tags>\n      <createdDate>2022-05-09T19:50:24Z</createdDate>\n      <updatedDate>2022-05-09T19:51:16Z</updatedDate>\n    </WebApp>\n    <WebApp>\n      <id>251316904</id>\n      <name><![CDATA[PyGoat]]></name>\n      <url><![CDATA[http://pygoat-dep.herokuapp.com/]]></url>\n      <owner>\n        <id>17729931</id>\n      </owner>\n      <tags>\n        <count>0</count>\n      </tags>\n      <createdDate>2022-05-10T00:59:15Z</createdDate>\n      <updatedDate>2022-05-10T01:01:21Z</updatedDate>\n    </WebApp>\n  </data>\n</ServiceResponse>"
           },
           "cookies": [],
           "headers": [
@@ -86,7 +86,7 @@
             },
             {
               "name": "x-powered-by",
-              "value": "Qualys:USPOD03:16cb332b-8708-5a0b-822b-364cd5ebd55b:0bd06b8b-d888-c7a8-8058-417c825e3976"
+              "value": "Qualys:USPOD03:b3f3a819-7884-e60e-81d0-9725801da546:51349a72-1971-7ba0-8252-5564ea1cc2fe"
             },
             {
               "name": "content-type",
@@ -98,45 +98,29 @@
             },
             {
               "name": "date",
-              "value": "Thu, 08 Oct 2020 18:41:44 GMT"
+              "value": "Wed, 01 Jun 2022 19:24:54 GMT"
             },
             {
               "name": "server",
               "value": "Apache"
-            },
-            {
-              "name": "strict-transport-security",
-              "value": "max-age=31536000"
-            },
-            {
-              "name": "x-frame-options",
-              "value": "SAMEORIGIN"
-            },
-            {
-              "name": "x-xss-protection",
-              "value": "1;mode=block"
-            },
-            {
-              "name": "x-content-type-options",
-              "value": "nosniff"
             }
           ],
-          "headersSize": 423,
+          "headersSize": 284,
           "httpVersion": "HTTP/1.1",
           "redirectURL": "",
           "status": 200,
           "statusText": "OK"
         },
-        "startedDateTime": "2020-10-08T18:41:44.173Z",
-        "time": 1060,
+        "startedDateTime": "2022-06-01T19:24:54.282Z",
+        "time": 759,
         "timings": {
           "blocked": -1,
           "connect": -1,
           "dns": -1,
           "receive": 0,
           "send": 0,
           "ssl": -1,
-          "wait": 1060
+          "wait": 759
         }
       }
     ],

diff --git a/src/provider/__recordings__/iterateWebAppFindingsBadId_3730797176/recording.har b/src/provider/__recordings__/iterateWebAppFindingsBadId_3730797176/recording.har
@@ -4,15 +4,15 @@
     "creator": {
       "comment": "persister:JupiterOneIntegationFSPersister",
       "name": "Polly.JS",
-      "version": "4.3.0"
+      "version": "5.1.1"
     },
     "entries": [
       {
         "_id": "21bbd5cfcfab3f2fc38f356e4ded9f77",
         "_order": 0,
         "cache": {},
         "request": {
-          "bodySize": 240,
+          "bodySize": 226,
           "cookies": [],
           "headers": [
             {
@@ -38,7 +38,7 @@
             {
               "_fromType": "array",
               "name": "content-length",
-              "value": "240"
+              "value": "226"
             },
             {
               "_fromType": "array",
@@ -60,13 +60,13 @@
               "value": "qualysapi.qg3.apps.qualys.com"
             }
           ],
-          "headersSize": 400,
+          "headersSize": 392,
           "httpVersion": "HTTP/1.1",
           "method": "POST",
           "postData": {
             "mimeType": "text/xml",
             "params": [],
-            "text": "\n      <ServiceRequest>\n        <preferences>\n          <limitResults>1</limitResults>\n        </preferences>\n        <filters>\n          <Criteria field=\"webApp.id\" operator=\"IN\">abc123</Criteria>\n        </filters>\n      </ServiceRequest>"
+            "text": "<ServiceRequest>\n  <preferences>\n    <limitResults>250</limitResults>\n    <startFromOffset>1</startFromOffset>\n  </preferences>\n  <filters><Criteria field=\"webApp.id\" operator=\"IN\">abc123</Criteria></filters>\n</ServiceRequest>"
           },
           "queryString": [],
           "url": "https://qualysapi.qg3.apps.qualys.com/qps/rest/3.0/search/was/finding/"
@@ -82,7 +82,7 @@
           "headers": [
             {
               "name": "x-powered-by",
-              "value": "Qualys:USPOD03:16cb332b-8708-5a0b-822b-364cd5ebd55b:0bd06b8b-d888-c7a8-8058-417c825e3976"
+              "value": "Qualys:USPOD03:b3f3a819-7884-e60e-81d0-9725801da546:51349a72-1971-7ba0-8252-5564ea1cc2fe"
             },
             {
               "name": "content-type",
@@ -94,45 +94,29 @@
             },
             {
               "name": "date",
-              "value": "Thu, 08 Oct 2020 18:41:05 GMT"
+              "value": "Wed, 01 Jun 2022 19:24:56 GMT"
             },
             {
               "name": "server",
               "value": "Apache"
-            },
-            {
-              "name": "strict-transport-security",
-              "value": "max-age=31536000"
-            },
-            {
-              "name": "x-frame-options",
-              "value": "SAMEORIGIN"
-            },
-            {
-              "name": "x-xss-protection",
-              "value": "1;mode=block"
-            },
-            {
-              "name": "x-content-type-options",
-              "value": "nosniff"
             }
           ],
-          "headersSize": 404,
+          "headersSize": 265,
           "httpVersion": "HTTP/1.1",
           "redirectURL": "",
           "status": 200,
           "statusText": "OK"
         },
-        "startedDateTime": "2020-10-08T18:41:04.957Z",
-        "time": 900,
+        "startedDateTime": "2022-06-01T19:24:55.747Z",
+        "time": 1001,
         "timings": {
           "blocked": -1,
           "connect": -1,
           "dns": -1,
           "receive": 0,
           "send": 0,
           "ssl": -1,
-          "wait": 900
+          "wait": 1001
         }
       }
     ],

diff --git a/src/provider/__recordings__/iterateWebAppFindingsUnknownId_373035765/recording.har b/src/provider/__recordings__/iterateWebAppFindingsUnknownId_373035765/recording.har
@@ -4,15 +4,15 @@
     "creator": {
       "comment": "persister:JupiterOneIntegationFSPersister",
       "name": "Polly.JS",
-      "version": "4.3.0"
+      "version": "5.1.1"
     },
     "entries": [
       {
         "_id": "21bbd5cfcfab3f2fc38f356e4ded9f77",
         "_order": 0,
         "cache": {},
         "request": {
-          "bodySize": 237,
+          "bodySize": 223,
           "cookies": [],
           "headers": [
             {
@@ -38,7 +38,7 @@
             {
               "_fromType": "array",
               "name": "content-length",
-              "value": "237"
+              "value": "223"
             },
             {
               "_fromType": "array",
@@ -60,13 +60,13 @@
               "value": "qualysapi.qg3.apps.qualys.com"
             }
           ],
-          "headersSize": 400,
+          "headersSize": 392,
           "httpVersion": "HTTP/1.1",
           "method": "POST",
           "postData": {
             "mimeType": "text/xml",
             "params": [],
-            "text": "\n      <ServiceRequest>\n        <preferences>\n          <limitResults>1</limitResults>\n        </preferences>\n        <filters>\n          <Criteria field=\"webApp.id\" operator=\"IN\">123</Criteria>\n        </filters>\n      </ServiceRequest>"
+            "text": "<ServiceRequest>\n  <preferences>\n    <limitResults>250</limitResults>\n    <startFromOffset>1</startFromOffset>\n  </preferences>\n  <filters><Criteria field=\"webApp.id\" operator=\"IN\">123</Criteria></filters>\n</ServiceRequest>"
           },
           "queryString": [],
           "url": "https://qualysapi.qg3.apps.qualys.com/qps/rest/3.0/search/was/finding/"
@@ -86,7 +86,7 @@
             },
             {
               "name": "x-powered-by",
-              "value": "Qualys:USPOD03:16cb332b-8708-5a0b-822b-364cd5ebd55b:0bd06b8b-d888-c7a8-8058-417c825e3976"
+              "value": "Qualys:USPOD03:b3f3a819-7884-e60e-81d0-9725801da546:51349a72-1971-7ba0-8252-5564ea1cc2fe"
             },
             {
               "name": "content-type",
@@ -98,45 +98,29 @@
             },
             {
               "name": "date",
-              "value": "Thu, 08 Oct 2020 18:41:04 GMT"
+              "value": "Wed, 01 Jun 2022 19:24:54 GMT"
             },
             {
               "name": "server",
               "value": "Apache"
-            },
-            {
-              "name": "strict-transport-security",
-              "value": "max-age=31536000"
-            },
-            {
-              "name": "x-frame-options",
-              "value": "SAMEORIGIN"
-            },
-            {
-              "name": "x-xss-protection",
-              "value": "1;mode=block"
-            },
-            {
-              "name": "x-content-type-options",
-              "value": "nosniff"
             }
           ],
-          "headersSize": 423,
+          "headersSize": 284,
           "httpVersion": "HTTP/1.1",
           "redirectURL": "",
           "status": 200,
           "statusText": "OK"
         },
-        "startedDateTime": "2020-10-08T18:41:03.376Z",
-        "time": 1569,
+        "startedDateTime": "2022-06-01T19:24:55.064Z",
+        "time": 675,
         "timings": {
           "blocked": -1,
           "connect": -1,
           "dns": -1,
           "receive": 0,
           "send": 0,
           "ssl": -1,
-          "wait": 1569
+          "wait": 675
         }
       }
     ],