DocOnly rules updated, packages updated.

credsweeper/rules/config.yaml

@@ -2,11 +2,13 @@
   severity: medium
   type: pattern
   values:
-    - (?P<variable>[`'\"]?(?i:token|secret|key|키|암호|암호화|토큰)[`'\"]?)((\s)*[=:](\s)*)(?P<quote>[`'\"(])?(?P<value>\S{4,})(?(quote)[)`'\"])
+    - (?P<variable>[`'\"]?(?i:token|secret|key|키|암호|암호화|토큰)[`'\"]?)((\s)*[=:](\s)*)(?P<quote>[`'\"(])?(?P<value>(?-i:(?P<a>[A-Z])|(?P<b>[a-z])|(?P<c>[0-9/_+=~!@#$%^&*;:?-])){8,}(?(a)(?(b)(?(c)(.|$)|(?!x)x)|(?!x)x)|(?!x)x))(?(quote)[)`'\"])
   filter_type:
     - ValueAllowlistCheck
     - ValuePatternCheck
-  min_line_len: 10
+    - ValueEntropyBase64Check
+    - ValueCoupleKeywordCheck
+  min_line_len: 16
   required_substrings:
     - token
     - secret
@@ -24,10 +26,11 @@
   severity: medium
   type: pattern
   values:
-    - (?P<variable>[`'\"]?(?i:(?<!id[ :/])pa[as]swo?r?ds?|pwd?|p/w|비밀번호|비번|패스워드|암호)[`'\"]?)((\s)*[=:](\s)*)(?P<quote>[`'\"(])?(?P<value>\S{4,})(?(quote)[)`'\"])
+    - (?P<variable>[`'\"]?(?i:(?<!id[ :/])pa[as]swo?r?ds?|pwd?|p/w|비밀번호|비번|패스워드|암호)[`'\"]?)((\s)*[=:](\s)*)(?P<quote>[`'\"(])?(?P<value>(?-i:(?P<a>[A-Z])|(?P<b>[a-z])|(?P<c>[0-9/_+=~!@#$%^&*;:?-])){4,31}(?(a)(?(b)(?(c)(.|$)|(?!x)x)|(?!x)x)|(?!x)x))(?(quote)[)`'\"])
   filter_type:
     - ValueAllowlistCheck
     - ValuePatternCheck
+    - ValueDictionaryKeywordCheck
   min_line_len: 10
   required_substrings:
     - pass
@@ -45,10 +48,11 @@
   severity: medium
   type: pattern
   values:
-    - (^|(?P<variable>(?i:\bip[\s/]+id[\s/]+pw[\s/:]*))|(?P<url>://)|\s)(?P<ip>[0-2]?[0-9]{1,2}\.[0-2]?[0-9]{1,2}\.[0-2]?[0-9]{1,2}\.[0-2]?[0-9]{1,2})((?P<lpar>\s*(\w+\s+)?\()?\s*|(?(variable)[\s,/]+|\s*(?(url)[,]|[,/]))\s*)[\w.-]{3,}[\s,/]+(?P<value>(?(lpar)[^)\s]{4,}|[^\s/]{4,}))(?:\s|[^/]|$)
+    - (^|\s|(?P<variable>(?i:\bip[\s/]+id[\s/]+pw[\s/:]*))|(?P<url>://))(?P<ip>[0-2]?[0-9]{1,2}\.[0-2]?[0-9]{1,2}\.[0-2]?[0-9]{1,2}\.[0-2]?[0-9]{1,2})((?P<lpar>\s*\()?|(?(variable)[\s,/]+|(?(url)[,]|[,/])))\s*\w[\w.-]{3,}[\s,/]+(?P<value>(?(lpar)[^)\s]{4,}|(?(url)(?-i:(?P<a>[A-Z])|(?P<b>[a-z])|(?P<c>[0-9_+=~!@#$%^&*;?-])){4,31}(?(a)(?(b)(?(c)(.|$)|(?!x)x)|(?!x)x)|(?!x)x)|(?-i:(?P<e>[A-Z])|(?P<f>[a-z])|(?P<g>[0-9/_+=~!@#$%^&*;?-])){4,31}(?(e)(?(f)(?(g)(.|$)|(?!x)x)|(?!x)x)|(?!x)x))))(?:\s|[^/]|$)
   filter_type:
     - ValueAllowlistCheck
     - ValuePatternCheck
+    - ValueDictionaryKeywordCheck
   min_line_len: 10
   required_substrings:
     - "."
@@ -58,7 +62,7 @@
   severity: medium
   type: pattern
   values:
-    - (?P<ddash>--)?(?P<variable>\w*(?i:pa[as]swords?|passwd?|pwd|\bp/w|\bpw|비밀번호|비번|패스워드|암호))\s*?(?(ddash)[ =]|[:=/>-]{1,2})\s*?(?P<quote>[`'\"]+)?(?P<value>\S{3,}?)(?(quote)(?P=quote)|\b)
+    - (?P<ddash>--)?(?P<variable>\w*(?i:pa[as]swords?|passwd?|pwd|\bp/w|\bpw|비밀번호|비번|패스워드|암호))\s*?(?(ddash)[ =]|[:=/>-]{1,2})\s*(?P<quote>[`'\"]+)?(?P<value>(?-i:(?P<a>[A-Z])|(?P<b>[a-z])|(?P<c>[0-9/_+=~!@#$%^&*;:?-])){4,31}(?(a)(?(b)(?(c)(.|$)|(?!x)x)|(?!x)x)|(?!x)x))(?(quote)(?P=quote)|\b)
     - (?P<ddash>--)?(?P<variable>(?i:user\s*)?(?i:id|login|account|root|admin|user|name|wifi|role|host|default|계정|아이디))\s*?(?(ddash)[ =]|[ :=])\s*?(?P<value>\S+)
   filter_type:
     - ValueAllowlistCheck
@@ -79,11 +83,11 @@
   severity: medium
   type: pattern
   values:
-    - (?P<variable>[\w.-]*(?i:(?P<id>\bid\b)|id\b|user|name|계정|아이디)[\w.-]*(?(id)[ :(/]+|[:(/]+)(?i:pa[as]swo?r?ds?|pwd?|비밀번호|비번|패스워드|암호))\)?(\s*->\s*|[ =:)(/]+|\s+is\s+|\s+are\s+|\s*는\s*|\s*은\s*|\s*설정은\s*)\(?(?P<id_value>[\w.-]{2,31})[ :\(/\"',]+(?P<value>[^\s}\])\"']{4,31})
+    - (?P<variable>[\w.-]*(?i:(?P<id>\bid\b)|id\b|user|name|계정|아이디)[\w.-]*(?(id)[ :(/]+|[:(/]+)(?i:pa[as]swo?r?ds?|pwd?|비밀번호|비번|패스워드|암호))\)?(\s*->\s*|[ =:)(/]+|\s+is\s+|\s+are\s+|\s*는\s*|\s*은\s*|\s*설정은\s*)\(?(?P<id_value>[\w.-]{2,31})[ :\(/\"',]+(?P<value>(?-i:(?P<a>[A-Z])|(?P<b>[a-z])|(?P<c>[0-9/_+=~!@#$%^&*;:?-])){4,31}(?(a)(?(b)(?(c)(.|$)|(?!x)x)|(?!x)x)|(?!x)x))
   filter_type:
     - ValueAllowlistCheck
-    - ValueDictionaryKeywordCheck
     - ValuePatternCheck
+    - ValueDictionaryKeywordCheck
   min_line_len: 10
   required_substrings:
     - pw

requirements.txt

@@ -11,7 +11,7 @@ pandas==2.0.3
 PyYAML==6.0.1
 python-docx==1.0.1
 requests==2.31.0
-schwifty==2023.9.0
+schwifty==2023.10.0
 typing_extensions==4.8.0
 whatthepatch==1.0.5
 pdfminer.six==20221105
@@ -22,7 +22,7 @@ python-dateutil==2.8.2
 numpy==1.24.4
 # ^ the version supports python 3.8-3.11
 # ^ todo: check for py3.12 later https://github.com/numpy/numpy/issues/23808
-scikit-learn==1.3.1
+scikit-learn==1.3.2
 scipy==1.10.1
 # ^ the version supports python 3.8
 onnxruntime==1.16.1

tests/__init__.py

@@ -4,14 +4,14 @@
 SAMPLES_FILES_COUNT: int = 123
 
 # credentials count after scan
-SAMPLES_CRED_COUNT: int = 382
-SAMPLES_CRED_LINE_COUNT: int = 401
+SAMPLES_CRED_COUNT: int = 386
+SAMPLES_CRED_LINE_COUNT: int = 405
 
 # credentials count after post-processing
-SAMPLES_POST_CRED_COUNT: int = 296
+SAMPLES_POST_CRED_COUNT: int = 299
 
 # with option --doc
-SAMPLES_IN_DOC = 427
+SAMPLES_IN_DOC = 390
 
 # archived credentials that are not found without --depth
 SAMPLES_IN_DEEP_1 = SAMPLES_POST_CRED_COUNT + 21