JWT token verification check update

credsweeper/filters/value_base64_data_check.py

@@ -1,4 +1,3 @@
-import base64
 import contextlib
 import string
 
@@ -40,14 +39,6 @@ def run(self, line_data: LineData, target: AnalysisTarget) -> bool:
                 return True
         # check whether decoded bytes have enough entropy
         with contextlib.suppress(Exception):
-            value_len = len(value)
-            if 0x3 & value_len:
-                # Bitbucket client id is 18 chars length
-                pad_len = 4 - (0x3 & value_len)
-                value = value + ''.join(['='] * pad_len)
-            if '-' in value or '_' in value:
-                decoded = base64.urlsafe_b64decode(value)
-            else:
-                decoded = base64.standard_b64decode(value)
+            decoded = Util.decode_base64(value, padding_safe=True, urlsafe_detect=True)
             return Util.is_ascii_entropy_validate(decoded)
         return True

credsweeper/filters/value_grafana_check.py

@@ -1,11 +1,11 @@
-import base64
 import contextlib
 import json
 
 from credsweeper.config import Config
 from credsweeper.credentials import LineData
 from credsweeper.file_handler.analysis_target import AnalysisTarget
 from credsweeper.filters import Filter
+from credsweeper.utils import Util
 
 
 class ValueGrafanaCheck(Filter):
@@ -30,11 +30,11 @@ def run(self, line_data: LineData, target: AnalysisTarget) -> bool:
         with contextlib.suppress(Exception):
             if line_data.value.startswith("glc_"):
                 # Grafana Access Policy Token
-                decoded = base64.b64decode(line_data.value[4:])
+                decoded = Util.decode_base64(line_data.value[4:], padding_safe=True, urlsafe_detect=True)
                 keys = ["o", "n", "k", "m"]
             else:
                 # Grafana Provisioned API Key
-                decoded = base64.b64decode(line_data.value)
+                decoded = Util.decode_base64(line_data.value, padding_safe=True, urlsafe_detect=True)
                 keys = ["n", "k", "id"]
             if payload := json.loads(decoded):
                 for key in keys:

credsweeper/filters/value_json_web_token_check.py

@@ -1,11 +1,11 @@
-import base64
 import contextlib
 import json
 
 from credsweeper.config import Config
 from credsweeper.credentials import LineData
 from credsweeper.file_handler.analysis_target import AnalysisTarget
 from credsweeper.filters import Filter
+from credsweeper.utils import Util
 
 
 class ValueJsonWebTokenCheck(Filter):
@@ -33,9 +33,13 @@ def run(self, line_data: LineData, target: AnalysisTarget) -> bool:
             return True
         with contextlib.suppress(Exception):
             delimiter_pos = line_data.value.find(".")
-            # jwt token. '.' must be always in given data, according regex in rule
-            value = line_data.value[:delimiter_pos]
-            decoded = base64.b64decode(value)
+            # JWT token. '.' MAY be always in given data
+            if 0 <= delimiter_pos:
+                value = line_data.value[:delimiter_pos]
+            else:
+                value = line_data.value
+            # https://www.rfc-editor.org/rfc/rfc7515.txt - padding is optional
+            decoded = Util.decode_base64(value, padding_safe=True, urlsafe_detect=True)
             if header := json.loads(decoded):
                 if "alg" in header or "typ" in header:
                     return False

credsweeper/filters/value_structured_token_check.py

@@ -55,7 +55,7 @@ def check_crc32_struct(value: str) -> bool:
     @staticmethod
     def check_atlassian_struct(value: str) -> bool:
         """Returns False if value is valid for atlassian structure 'integer:bytes'"""
-        decoded = base64.b64decode(value)
+        decoded = Util.decode_base64(value, padding_safe=True, urlsafe_detect=True)
         delimiter_pos = decoded.find(b':')
         # there is limit for big integer value: math.log10(1<<64) = 19.265919722494797
         if 0 < delimiter_pos <= 20:

credsweeper/rules/config.yaml

@@ -268,7 +268,7 @@
   severity: medium
   type: pattern
   values:
-    - (^|[^.0-9A-Za-z_/+-])(?P<value>eyJ[A-Za-z0-9=_-]{13,}(\.[A-Za-z0-9-_.+\/=]+)?)
+    - (^|[^.0-9A-Za-z_/+-])(?P<value>eyJ[0-9A-Za-z_=-]{13,}(\.[0-9A-Za-z_=-]+){0,2})
   filter_type: GeneralPattern
   use_ml: true
   required_substrings:
@@ -622,7 +622,7 @@
   severity: high
   type: pattern
   values:
-    - (^|[^.0-9A-Za-z_/+-])(?P<value>eyJ[A-Za-z0-9_=-]{50,500}\.eyJ[A-Za-z0-9_=-]+\.[A-Za-z0-9_=-]+)
+    - (^|[^.0-9A-Za-z_/+-])(?P<value>eyJ[A-Za-z0-9_=-]{50,500}\.eyJ[A-Za-z0-9_=-]+\.[A-Za-z0-9_=-]+)([^.0-9A-Za-z_-]|$)
   filter_type:
     - ValueJsonWebTokenCheck
   required_substrings:
@@ -803,7 +803,7 @@
   severity: high
   type: pattern
   values:
-    - (^|[^.0-9A-Za-z_/+-])(?P<value>eyJ[a-zA-Z0-9=/-]{64,360})([^=0-9A-Za-z_/+-]|$)
+    - (^|[^.0-9A-Za-z_/+-])(?P<value>eyJ[0-9A-Za-z_=-]{64,360})([^=0-9A-Za-z_/+-]|$)
   filter_type:
     - ValueGrafanaCheck
   min_line_len: 67
@@ -814,7 +814,7 @@
   severity: high
   type: pattern
   values:
-    - (^|[^.0-9A-Za-z_/+-])(?P<value>glc_eyJ[a-zA-Z0-9=/-]{80,360})([^=0-9A-Za-z_/+-]|$)
+    - (^|[^.0-9A-Za-z_/+-])(?P<value>glc_eyJ[0-9A-Za-z_=-]{80,360})([^=0-9A-Za-z_/+-]|$)
   filter_type:
     - ValueGrafanaCheck
   min_line_len: 87

credsweeper/utils/util.py

@@ -1,4 +1,5 @@
 import ast
+import base64
 import json
 import logging
 import math
@@ -613,3 +614,17 @@ def parse_python(source: str) -> List[Any]:
         src = ast.parse(source)
         result = Util.ast_to_dict(src)
         return result
+
+    @staticmethod
+    def decode_base64(text: str, padding_safe: bool = False, urlsafe_detect=False) -> bytes:
+        """decode text to bytes with / without padding detect and urlsafe symbols"""
+        value = text
+        if padding_safe:
+            pad_num = 0x3 & len(value)
+            if pad_num:
+                value += '=' * (4 - pad_num)
+        if urlsafe_detect and '-' in value or '_' in value:
+            decoded = base64.urlsafe_b64decode(value)
+        else:
+            decoded = base64.standard_b64decode(value)
+        return decoded

tests/filters/test_value_json_web_token_check.py

@@ -11,17 +11,20 @@ class TestValueJsonWebTokenCheck:
 
     @pytest.mark.parametrize("line", ["12345:asbdsa:28yd"])
     def test_value_jwt_check_p(self, file_path: pytest.fixture, line: str) -> None:
+        jwt_like_line = base64.urlsafe_b64encode('{"typ":"JWT", "dummy": false}'.encode('ascii')).decode('ascii')
+        jwt_line_data = get_line_data(file_path, line=f"{jwt_like_line}", pattern=LINE_VALUE_PATTERN)
+        assert ValueJsonWebTokenCheck().run(jwt_line_data, DUMMY_ANALYSIS_TARGET) is False
         encoded_line = base64.b64encode(line.encode('ascii')).decode('ascii')
-        jwt_like_line = base64.b64encode('{"typ":"JWT", "dummy": false}'.encode('ascii')).decode('ascii')
         jwt_line_data = get_line_data(file_path, line=f"{jwt_like_line}.{encoded_line}", pattern=LINE_VALUE_PATTERN)
         assert ValueJsonWebTokenCheck().run(jwt_line_data, DUMMY_ANALYSIS_TARGET) is False
-        # partially line
+        assert '=' in jwt_like_line  # just demonstrate that encoded header contains padding symbol
+        jwt_like_line = jwt_like_line.replace('=', '')
         jwt_line_data = get_line_data(file_path, line=f"{jwt_like_line}.AnyTailOfString", pattern=LINE_VALUE_PATTERN)
         assert ValueJsonWebTokenCheck().run(jwt_line_data, DUMMY_ANALYSIS_TARGET) is False
 
     @pytest.mark.parametrize("line", ["1234f:asbdsa:28yd"])
     def test_value_jwt_check_n(self, file_path: pytest.fixture, line: str) -> None:
-        encoded_line = base64.b64encode(line.encode('ascii')).decode('ascii')
+        encoded_line = base64.urlsafe_b64encode(line.encode('ascii')).decode('ascii')
         jwt_line_data = get_line_data(file_path, line=f"eyJungle.{encoded_line}", pattern=LINE_VALUE_PATTERN)
         assert ValueJsonWebTokenCheck().run(jwt_line_data, DUMMY_ANALYSIS_TARGET) is True
         jwt_line_data = get_line_data(file_path, line="eyJungle", pattern=LINE_VALUE_PATTERN)