diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index cf3857b417..3700b4b72f 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -22,7 +22,8 @@ jobs: - name: Checkout CredData uses: actions/checkout@v3 with: - repository: Samsung/CredData + ref: ecpkey + repository: babenek/CredData - name: Cache data id: cache-data diff --git a/credsweeper/app.py b/credsweeper/app.py index f19e17f74d..b50961a5f2 100644 --- a/credsweeper/app.py +++ b/credsweeper/app.py @@ -374,7 +374,6 @@ def export_results(self) -> None: if self.sort_output: credentials.sort(key=lambda x: ( # - x.line_data_list[0].line, # x.line_data_list[0].path, # x.line_data_list[0].line_num, # x.severity, # diff --git a/credsweeper/rules/config.yaml b/credsweeper/rules/config.yaml index 03e8301a70..b1e630fba8 100644 --- a/credsweeper/rules/config.yaml +++ b/credsweeper/rules/config.yaml @@ -383,7 +383,7 @@ min_line_len: 72 - name: PEM Private Key - severity: critical + severity: high type: pem_key values: - (?P-----BEGIN\s(?!ENCRYPTED)[^-]*PRIVATE[^-]*KEY[^-]*-----(.+-----END[^-]+-----)?) diff --git a/credsweeper/scanner/scanner.py b/credsweeper/scanner/scanner.py index dd804a1014..a91d2c3d81 100644 --- a/credsweeper/scanner/scanner.py +++ b/credsweeper/scanner/scanner.py @@ -149,8 +149,8 @@ def scan(self, provider: ContentProvider) -> List[Candidate]: if not (matched_keyword or matched_pem_key or matched_pattern or matched_multi): # target may be skipped only with length because not all rules have required_substrings - # logger.debug("Skip too short (%d) line %s:%d", target_line_stripped_len, target.file_path, - # target.line_num) + logger.debug("Skip too short (%d) line %s:%d", target_line_stripped_len, target.file_path, + target.line_num) continue # use lower case for required substring diff --git a/tests/__init__.py b/tests/__init__.py index 3e9d6241c4..1816b3dde6 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -4,19 +4,19 @@ SAMPLES_FILES_COUNT: int = 126 # credentials count after scan -SAMPLES_CRED_COUNT: int = 384 -SAMPLES_CRED_LINE_COUNT: int = 405 +SAMPLES_CRED_COUNT: int = 383 +SAMPLES_CRED_LINE_COUNT: int = 402 # credentials count after post-processing -SAMPLES_POST_CRED_COUNT: int = 298 +SAMPLES_POST_CRED_COUNT: int = 297 # with option --doc -SAMPLES_IN_DOC = 432 +SAMPLES_IN_DOC = 431 # archived credentials that are not found without --depth SAMPLES_IN_DEEP_1 = SAMPLES_POST_CRED_COUNT + 24 SAMPLES_IN_DEEP_2 = SAMPLES_IN_DEEP_1 + 16 -SAMPLES_IN_DEEP_3 = SAMPLES_IN_DEEP_2 + 3 +SAMPLES_IN_DEEP_3 = SAMPLES_IN_DEEP_2 + 2 # well known string with all latin letters AZ_DATA = b"The quick brown fox jumps over the lazy dog" diff --git a/tests/data/depth_3.json b/tests/data/depth_3.json index 86d10affd3..ff1b13c157 100644 --- a/tests/data/depth_3.json +++ b/tests/data/depth_3.json @@ -6763,280 +6763,22 @@ "severity": "high", "line_data_list": [ { - "line": "char pk[] = \"-----BEGIN RSA PRIVATE KEY-----\\n\\ni7aHavqQ9T2f2drU4N5WsM7aqQ97kbB/K4RUPdit+tIpqSb1GgY44yg6lckfoLMH\\najimpQ6sr9BuseERqELGE1U+Vll3izwuqr1UzCZ61gZnwPDBsD02jF038wYU6mZT\\n-----END RSA PRIVATE KEY-----\\n\";", + "line": "char pk[] = \"-----BEGIN OPENSSH PRIVATE KEY-----\\n\\ni7aHavqQ9T2f2drU4N5WsM7aqQ97kbB/K4RUPdit+tIpqSb1GgY44yg6lckfoLMH\\najimpQ6sr9BuseERqELGE1U+Vll3izwuqr1UzCZ61gZnwPDBsD02jF038wYU6mZT\\n-----END RSA PRIVATE KEY-----\\n\";", "line_num": 12, "path": "tests/samples/pem_key", "info": "tests/samples/pem_key|RAW", - "value": "-----BEGIN RSA PRIVATE KEY-----\\n\\ni7aHavqQ9T2f2drU4N5WsM7aqQ97kbB/K4RUPdit+tIpqSb1GgY44yg6lckfoLMH\\najimpQ6sr9BuseERqELGE1U+Vll3izwuqr1UzCZ61gZnwPDBsD02jF038wYU6mZT\\n-----END RSA PRIVATE KEY-----", + "value": "-----BEGIN OPENSSH PRIVATE KEY-----\\n\\ni7aHavqQ9T2f2drU4N5WsM7aqQ97kbB/K4RUPdit+tIpqSb1GgY44yg6lckfoLMH\\najimpQ6sr9BuseERqELGE1U+Vll3izwuqr1UzCZ61gZnwPDBsD02jF038wYU6mZT\\n-----END RSA PRIVATE KEY-----", "value_start": 13, - "value_end": 209, + "value_end": 213, "variable": null, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 5.029849490561501, + "entropy": 5.054323782040971, "valid": true } } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "PEM Private Key", - "severity": "high", - "line_data_list": [ - { - "line": "-----BEGIN EC PRIVATE KEY-----", - "line_num": 15, - "path": "tests/samples/pem_key", - "info": "tests/samples/pem_key|RAW", - "value": "-----BEGIN EC PRIVATE KEY-----", - "value_start": 0, - "value_end": 30, - "variable": null, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.447238004178161, - "valid": false - } - }, - { - "line": "MHcCAQEEINGSM49oAoGCCqAEi9Hdw6KvZcWxfg2IDhA7UkpDtzzt6mNAlLUqjShUsUBBngG0u2fZqJXSsFdLd+Kx4S3Sx4cVO+AwEHoUQDQgGWx0zo6fhJ/0EAfrPzVFyFC9s18lBt3cRoEDhS3ARo6/ZOXRnPqEqA==", - "line_num": 16, - "path": "tests/samples/pem_key", - "info": "tests/samples/pem_key|RAW", - "value": "MHcCAQEEINGSM49oAoGCCqAEi9Hdw6KvZcWxfg2IDhA7UkpDtzzt6mNAlLUqjShUsUBBngG0u2fZqJXSsFdLd+Kx4S3Sx4cVO+AwEHoUQDQgGWx0zo6fhJ/0EAfrPzVFyFC9s18lBt3cRoEDhS3ARo6/ZOXRnPqEqA==", - "value_start": 0, - "value_end": 164, - "variable": null, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 5.668600385988814, - "valid": true - } - }, - { - "line": "-----END EC PRIVATE KEY-----", - "line_num": 17, - "path": "tests/samples/pem_key", - "info": "tests/samples/pem_key|RAW", - "value": "-----END EC PRIVATE KEY-----", - "value_start": 0, - "value_end": 28, - "variable": null, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.2896544225308593, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "PEM Private Key", - "severity": "high", - "line_data_list": [ - { - "line": "-----BEGIN RSA PRIVATE KEY-----", - "line_num": 1, - "path": "tests/samples/pem_key.apk", - "info": "tests/samples/pem_key.apk|ZIP|3.zip|ZIP|2.zip|ZIP|2|RAW", - "value": "-----BEGIN RSA PRIVATE KEY-----", - "value_start": 0, - "value_end": 31, - "variable": null, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.529698260800009, - "valid": false - } - }, - { - "line": "MIICjdsnc34fdsjkgfdjvnkjcnvenr8vjsdjvsjhdfkshkehr34hr98whfuskhdfjhqllladhsjd", - "line_num": 2, - "path": "tests/samples/pem_key.apk", - "info": "tests/samples/pem_key.apk|ZIP|3.zip|ZIP|2.zip|ZIP|2|RAW", - "value": "MIICjdsnc34fdsjkgfdjvnkjcnvenr8vjsdjvsjhdfkshkehr34hr98whfuskhdfjhqllladhsjd", - "value_start": 0, - "value_end": 76, - "variable": null, - "entropy_validation": { - "iterator": "BASE36_CHARS", - "entropy": 3.880801520359916, - "valid": true - } - }, - { - "line": "wmJG8wVQZKjeGcjDOL5UlsuusFncCzWBQ7RKNUSesmQRMSGkVb1/3j+skZ6UtW+5u09lHNsj6tQ5", - "line_num": 3, - "path": "tests/samples/pem_key.apk", - "info": "tests/samples/pem_key.apk|ZIP|3.zip|ZIP|2.zip|ZIP|2|RAW", - "value": "wmJG8wVQZKjeGcjDOL5UlsuusFncCzWBQ7RKNUSesmQRMSGkVb1/3j+skZ6UtW+5u09lHNsj6tQ5", - "value_start": 0, - "value_end": 76, - "variable": null, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 5.213332901823973, - "valid": true - } - }, - { - "line": "1s1SPrCBkedbNf0Tp0GbMJDyR4e9T04ZZwIDAQABAoGAFijko56+qGyN8M0RVyaRAXz++xTqHBLh", - "line_num": 4, - "path": "tests/samples/pem_key.apk", - "info": "tests/samples/pem_key.apk|ZIP|3.zip|ZIP|2.zip|ZIP|2|RAW", - "value": "1s1SPrCBkedbNf0Tp0GbMJDyR4e9T04ZZwIDAQABAoGAFijko56+qGyN8M0RVyaRAXz++xTqHBLh", - "value_start": 0, - "value_end": 76, - "variable": null, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 5.325046388609145, - "valid": true - } - }, - { - "line": "3tx4VgMtrQ+WEgCjhoTwo23KMBAuJGSYnRmoBZM3lMfTKevIkAidPExvYCdm5dYq3XToLkkLv5L2", - "line_num": 5, - "path": "tests/samples/pem_key.apk", - "info": "tests/samples/pem_key.apk|ZIP|3.zip|ZIP|2.zip|ZIP|2|RAW", - "value": "3tx4VgMtrQ+WEgCjhoTwo23KMBAuJGSYnRmoBZM3lMfTKevIkAidPExvYCdm5dYq3XToLkkLv5L2", - "value_start": 0, - "value_end": 76, - "variable": null, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 5.267278500114894, - "valid": true - } - }, - { - "line": "pIIVOFMDG+KESnAFV7l2c+cnzRMW0+b6f8mR1CJzZuxVLL6Q02fvLi55/mbSYxECQQDeAw6fiIQX", - "line_num": 6, - "path": "tests/samples/pem_key.apk", - "info": "tests/samples/pem_key.apk|ZIP|3.zip|ZIP|2.zip|ZIP|2|RAW", - "value": "pIIVOFMDG+KESnAFV7l2c+cnzRMW0+b6f8mR1CJzZuxVLL6Q02fvLi55/mbSYxECQQDeAw6fiIQX", - "value_start": 0, - "value_end": 76, - "variable": null, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 5.293594289588578, - "valid": true - } - }, - { - "line": "GukBI4eMZZt4nscy2o12KyYner3VpoeE+Np2q+Z3pvAMd/aNzQ/W9WaI+NRfcxUJrmfPwIGm63il", - "line_num": 7, - "path": "tests/samples/pem_key.apk", - "info": "tests/samples/pem_key.apk|ZIP|3.zip|ZIP|2.zip|ZIP|2|RAW", - "value": "GukBI4eMZZt4nscy2o12KyYner3VpoeE+Np2q+Z3pvAMd/aNzQ/W9WaI+NRfcxUJrmfPwIGm63il", - "value_start": 0, - "value_end": 76, - "variable": null, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 5.405307776373748, - "valid": true - } - }, - { - "line": "AkEAxCL5HQb2bQr4ByorcMWm/hEP2MZzROV73yF41hPsRC9m66KrheO9HPTJuo3/9s5p+sqGxOlF", - "line_num": 8, - "path": "tests/samples/pem_key.apk", - "info": "tests/samples/pem_key.apk|ZIP|3.zip|ZIP|2.zip|ZIP|2|RAW", - "value": "AkEAxCL5HQb2bQr4ByorcMWm/hEP2MZzROV73yF41hPsRC9m66KrheO9HPTJuo3/9s5p+sqGxOlF", - "value_start": 0, - "value_end": 76, - "variable": null, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 5.372541658009631, - "valid": true - } - }, - { - "line": "L0NDt4SkosjgGwJAFklyR1uZ/wPJjj611cdBcztlPdqoxssQGnh85BzCj/u3WqBpE2vjvyyvyI5k", - "line_num": 9, - "path": "tests/samples/pem_key.apk", - "info": "tests/samples/pem_key.apk|ZIP|3.zip|ZIP|2.zip|ZIP|2|RAW", - "value": "L0NDt4SkosjgGwJAFklyR1uZ/wPJjj611cdBcztlPdqoxssQGnh85BzCj/u3WqBpE2vjvyyvyI5k", - "value_start": 0, - "value_end": 76, - "variable": null, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 5.308663329427085, - "valid": true - } - }, - { - "line": "X6zk7S0ljKtt2jny2+00VsBerQJBAJGC1Mg5Oydo5NwD6BiROrPxGo2bpTbu/fhrT8ebHkTz2epl", - "line_num": 10, - "path": "tests/samples/pem_key.apk", - "info": "tests/samples/pem_key.apk|ZIP|3.zip|ZIP|2.zip|ZIP|2|RAW", - "value": "X6zk7S0ljKtt2jny2+00VsBerQJBAJGC1Mg5Oydo5NwD6BiROrPxGo2bpTbu/fhrT8ebHkTz2epl", - "value_start": 0, - "value_end": 76, - "variable": null, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 5.425173236956998, - "valid": true - } - }, - { - "line": "U9VQQSQzY1oZMVX8i1m5WUTLPz2yLJIBQVdXqhMCQBGoiuSoSjafUhV7i1cEGpb88h5NBYZzWXGZ", - "line_num": 11, - "path": "tests/samples/pem_key.apk", - "info": "tests/samples/pem_key.apk|ZIP|3.zip|ZIP|2.zip|ZIP|2|RAW", - "value": "U9VQQSQzY1oZMVX8i1m5WUTLPz2yLJIBQVdXqhMCQBGoiuSoSjafUhV7i1cEGpb88h5NBYZzWXGZ", - "value_start": 0, - "value_end": 76, - "variable": null, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 5.107555270017291, - "valid": true - } - }, - { - "line": "37sJ5QsW+sJyoNde3xH8vdXhzU7eT82D6X/scw9RZz+/6rCJ4p0=", - "line_num": 12, - "path": "tests/samples/pem_key.apk", - "info": "tests/samples/pem_key.apk|ZIP|3.zip|ZIP|2.zip|ZIP|2|RAW", - "value": "37sJ5QsW+sJyoNde3xH8vdXhzU7eT82D6X/scw9RZz+/6rCJ4p0=", - "value_start": 0, - "value_end": 52, - "variable": null, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 5.0705380354071785, - "valid": true - } - }, - { - "line": "-----END RSA PRIVATE KEY-----", - "line_num": 13, - "path": "tests/samples/pem_key.apk", - "info": "tests/samples/pem_key.apk|ZIP|3.zip|ZIP|2.zip|ZIP|2|RAW", - "value": "-----END RSA PRIVATE KEY-----", - "value_start": 0, - "value_end": 29, - "variable": null, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.3783727041337137, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", diff --git a/tests/data/doc.json b/tests/data/doc.json index e32bb242bd..f33908c9a0 100644 --- a/tests/data/doc.json +++ b/tests/data/doc.json @@ -11053,76 +11053,22 @@ "severity": "high", "line_data_list": [ { - "line": "char pk[] = \"-----BEGIN RSA PRIVATE KEY-----\\n\\ni7aHavqQ9T2f2drU4N5WsM7aqQ97kbB/K4RUPdit+tIpqSb1GgY44yg6lckfoLMH\\najimpQ6sr9BuseERqELGE1U+Vll3izwuqr1UzCZ61gZnwPDBsD02jF038wYU6mZT\\n-----END RSA PRIVATE KEY-----\\n\";", + "line": "char pk[] = \"-----BEGIN OPENSSH PRIVATE KEY-----\\n\\ni7aHavqQ9T2f2drU4N5WsM7aqQ97kbB/K4RUPdit+tIpqSb1GgY44yg6lckfoLMH\\najimpQ6sr9BuseERqELGE1U+Vll3izwuqr1UzCZ61gZnwPDBsD02jF038wYU6mZT\\n-----END RSA PRIVATE KEY-----\\n\";", "line_num": 12, "path": "tests/samples/pem_key", "info": "tests/samples/pem_key|RAW", - "value": "-----BEGIN RSA PRIVATE KEY-----\\n\\ni7aHavqQ9T2f2drU4N5WsM7aqQ97kbB/K4RUPdit+tIpqSb1GgY44yg6lckfoLMH\\najimpQ6sr9BuseERqELGE1U+Vll3izwuqr1UzCZ61gZnwPDBsD02jF038wYU6mZT\\n-----END RSA PRIVATE KEY-----", + "value": "-----BEGIN OPENSSH PRIVATE KEY-----\\n\\ni7aHavqQ9T2f2drU4N5WsM7aqQ97kbB/K4RUPdit+tIpqSb1GgY44yg6lckfoLMH\\najimpQ6sr9BuseERqELGE1U+Vll3izwuqr1UzCZ61gZnwPDBsD02jF038wYU6mZT\\n-----END RSA PRIVATE KEY-----", "value_start": 13, - "value_end": 209, + "value_end": 213, "variable": null, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 5.029849490561501, + "entropy": 5.054323782040971, "valid": true } } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "PEM Private Key", - "severity": "high", - "line_data_list": [ - { - "line": "-----BEGIN EC PRIVATE KEY-----", - "line_num": 15, - "path": "tests/samples/pem_key", - "info": "tests/samples/pem_key|RAW", - "value": "-----BEGIN EC PRIVATE KEY-----", - "value_start": 0, - "value_end": 30, - "variable": null, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.447238004178161, - "valid": false - } - }, - { - "line": "MHcCAQEEINGSM49oAoGCCqAEi9Hdw6KvZcWxfg2IDhA7UkpDtzzt6mNAlLUqjShUsUBBngG0u2fZqJXSsFdLd+Kx4S3Sx4cVO+AwEHoUQDQgGWx0zo6fhJ/0EAfrPzVFyFC9s18lBt3cRoEDhS3ARo6/ZOXRnPqEqA==", - "line_num": 16, - "path": "tests/samples/pem_key", - "info": "tests/samples/pem_key|RAW", - "value": "MHcCAQEEINGSM49oAoGCCqAEi9Hdw6KvZcWxfg2IDhA7UkpDtzzt6mNAlLUqjShUsUBBngG0u2fZqJXSsFdLd+Kx4S3Sx4cVO+AwEHoUQDQgGWx0zo6fhJ/0EAfrPzVFyFC9s18lBt3cRoEDhS3ARo6/ZOXRnPqEqA==", - "value_start": 0, - "value_end": 164, - "variable": null, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 5.668600385988814, - "valid": true - } - }, - { - "line": "-----END EC PRIVATE KEY-----", - "line_num": 17, - "path": "tests/samples/pem_key", - "info": "tests/samples/pem_key|RAW", - "value": "-----END EC PRIVATE KEY-----", - "value_start": 0, - "value_end": 28, - "variable": null, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.2896544225308593, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", diff --git a/tests/data/ml_threshold_0.json b/tests/data/ml_threshold_0.json index 702ea30b37..42235ebe52 100644 --- a/tests/data/ml_threshold_0.json +++ b/tests/data/ml_threshold_0.json @@ -8467,76 +8467,22 @@ "severity": "high", "line_data_list": [ { - "line": "char pk[] = \"-----BEGIN RSA PRIVATE KEY-----\\n\\ni7aHavqQ9T2f2drU4N5WsM7aqQ97kbB/K4RUPdit+tIpqSb1GgY44yg6lckfoLMH\\najimpQ6sr9BuseERqELGE1U+Vll3izwuqr1UzCZ61gZnwPDBsD02jF038wYU6mZT\\n-----END RSA PRIVATE KEY-----\\n\";", + "line": "char pk[] = \"-----BEGIN OPENSSH PRIVATE KEY-----\\n\\ni7aHavqQ9T2f2drU4N5WsM7aqQ97kbB/K4RUPdit+tIpqSb1GgY44yg6lckfoLMH\\najimpQ6sr9BuseERqELGE1U+Vll3izwuqr1UzCZ61gZnwPDBsD02jF038wYU6mZT\\n-----END RSA PRIVATE KEY-----\\n\";", "line_num": 12, "path": "tests/samples/pem_key", "info": "", - "value": "-----BEGIN RSA PRIVATE KEY-----\\n\\ni7aHavqQ9T2f2drU4N5WsM7aqQ97kbB/K4RUPdit+tIpqSb1GgY44yg6lckfoLMH\\najimpQ6sr9BuseERqELGE1U+Vll3izwuqr1UzCZ61gZnwPDBsD02jF038wYU6mZT\\n-----END RSA PRIVATE KEY-----", + "value": "-----BEGIN OPENSSH PRIVATE KEY-----\\n\\ni7aHavqQ9T2f2drU4N5WsM7aqQ97kbB/K4RUPdit+tIpqSb1GgY44yg6lckfoLMH\\najimpQ6sr9BuseERqELGE1U+Vll3izwuqr1UzCZ61gZnwPDBsD02jF038wYU6mZT\\n-----END RSA PRIVATE KEY-----", "value_start": 13, - "value_end": 209, + "value_end": 213, "variable": null, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 5.029849490561501, + "entropy": 5.054323782040971, "valid": true } } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "PEM Private Key", - "severity": "high", - "line_data_list": [ - { - "line": "-----BEGIN EC PRIVATE KEY-----", - "line_num": 15, - "path": "tests/samples/pem_key", - "info": "", - "value": "-----BEGIN EC PRIVATE KEY-----", - "value_start": 0, - "value_end": 30, - "variable": null, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.447238004178161, - "valid": false - } - }, - { - "line": "MHcCAQEEINGSM49oAoGCCqAEi9Hdw6KvZcWxfg2IDhA7UkpDtzzt6mNAlLUqjShUsUBBngG0u2fZqJXSsFdLd+Kx4S3Sx4cVO+AwEHoUQDQgGWx0zo6fhJ/0EAfrPzVFyFC9s18lBt3cRoEDhS3ARo6/ZOXRnPqEqA==", - "line_num": 16, - "path": "tests/samples/pem_key", - "info": "", - "value": "MHcCAQEEINGSM49oAoGCCqAEi9Hdw6KvZcWxfg2IDhA7UkpDtzzt6mNAlLUqjShUsUBBngG0u2fZqJXSsFdLd+Kx4S3Sx4cVO+AwEHoUQDQgGWx0zo6fhJ/0EAfrPzVFyFC9s18lBt3cRoEDhS3ARo6/ZOXRnPqEqA==", - "value_start": 0, - "value_end": 164, - "variable": null, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 5.668600385988814, - "valid": true - } - }, - { - "line": "-----END EC PRIVATE KEY-----", - "line_num": 17, - "path": "tests/samples/pem_key", - "info": "", - "value": "-----END EC PRIVATE KEY-----", - "value_start": 0, - "value_end": 28, - "variable": null, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.2896544225308593, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", diff --git a/tests/data/output.json b/tests/data/output.json index 1c65da62f9..0e330617c1 100644 --- a/tests/data/output.json +++ b/tests/data/output.json @@ -6319,7 +6319,7 @@ "ml_validation": "NOT_AVAILABLE", "ml_probability": null, "rule": "PEM Private Key", - "severity": "critical", + "severity": "high", "line_data_list": [ { "line": "value = \"-----BEGIN OPENSSH PRIVATE KEY-----\" \\", @@ -6448,7 +6448,7 @@ "ml_validation": "NOT_AVAILABLE", "ml_probability": null, "rule": "PEM Private Key", - "severity": "critical", + "severity": "high", "line_data_list": [ { "line": "char pk[] = \"-----BEGIN OPENSSH PRIVATE KEY-----\\n\\ni7aHavqQ9T2f2drU4N5WsM7aqQ97kbB/K4RUPdit+tIpqSb1GgY44yg6lckfoLMH\\najimpQ6sr9BuseERqELGE1U+Vll3izwuqr1UzCZ61gZnwPDBsD02jF038wYU6mZT\\n-----END RSA PRIVATE KEY-----\\n\";", @@ -6467,249 +6467,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "PEM Private Key", - "severity": "critical", - "line_data_list": [ - { - "line": "-----BEGIN EC PRIVATE KEY-----", - "line_num": 15, - "path": "tests/samples/pem_key", - "info": "", - "value": "-----BEGIN EC PRIVATE KEY-----", - "value_start": 0, - "value_end": 30, - "variable": null, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.447238004178161, - "valid": false - } - }, - { - "line": "MHcCAQEEINGSM49oAoGCCqAEi9Hdw6KvZcWxfg2IDhA7UkpDtzzt6mNAlLUqjShUsUBBngG0u2fZqJXSsFdLd+Kx4S3Sx4cVO+AwEHoUQDQgGWx0zo6fhJ/0EAfrPzVFyFC9s18lBt3cRoEDhS3ARo6/ZOXRnPqEqA==", - "line_num": 16, - "path": "tests/samples/pem_key", - "info": "", - "value": "MHcCAQEEINGSM49oAoGCCqAEi9Hdw6KvZcWxfg2IDhA7UkpDtzzt6mNAlLUqjShUsUBBngG0u2fZqJXSsFdLd+Kx4S3Sx4cVO+AwEHoUQDQgGWx0zo6fhJ/0EAfrPzVFyFC9s18lBt3cRoEDhS3ARo6/ZOXRnPqEqA==", - "value_start": 0, - "value_end": 164, - "variable": null, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 5.668600385988814, - "valid": true - } - }, - { - "line": "-----END EC PRIVATE KEY-----", - "line_num": 17, - "path": "tests/samples/pem_key", - "info": "", - "value": "-----END EC PRIVATE KEY-----", - "value_start": 0, - "value_end": 28, - "variable": null, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.2896544225308593, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "PEM Private Key", - "severity": "critical", - "line_data_list": [ - { - "line": "-----BEGIN PRIVATE KEY-----", - "line_num": 43, - "path": "tests/samples/pem_key", - "info": "", - "value": "-----BEGIN PRIVATE KEY-----", - "value_start": 0, - "value_end": 27, - "variable": null, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.391423149269947, - "valid": false - } - }, - { - "line": "MIIByQIBADCCAZsGCSqGSIb3DQEDATCCAYwCggGBAP//////////rfhUWKK7Spqv", - "line_num": 44, - "path": "tests/samples/pem_key", - "info": "", - "value": "MIIByQIBADCCAZsGCSqGSIb3DQEDATCCAYwCggGBAP//////////rfhUWKK7Spqv", - "value_start": 0, - "value_end": 64, - "variable": null, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 4.510027399432949, - "valid": true - } - }, - { - "line": "3FYgJz088di5xYPOLTaVqeE2QRRkM/vMk53OJJs++X0v42NjDHXY9oGyAq7EYXrT", - "line_num": 45, - "path": "tests/samples/pem_key", - "info": "", - "value": "3FYgJz088di5xYPOLTaVqeE2QRRkM/vMk53OJJs++X0v42NjDHXY9oGyAq7EYXrT", - "value_start": 0, - "value_end": 64, - "variable": null, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 5.288909765557392, - "valid": true - } - }, - { - "line": "3x7V1f1lYSQz9R9fBm7QhWNlVT3tGvO1VxNef1fJNZhPDHDg5ot34qaJ2vPv6HId", - "line_num": 46, - "path": "tests/samples/pem_key", - "info": "", - "value": "3x7V1f1lYSQz9R9fBm7QhWNlVT3tGvO1VxNef1fJNZhPDHDg5ot34qaJ2vPv6HId", - "value_start": 0, - "value_end": 64, - "variable": null, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 5.109069531114784, - "valid": true - } - }, - { - "line": "8VihNq3nNTCsyk9IOnl6vAqxgrMk+2HRCKlLssjj+7lq2rdg1/RoHU9Co945TfSu", - "line_num": 47, - "path": "tests/samples/pem_key", - "info": "", - "value": "8VihNq3nNTCsyk9IOnl6vAqxgrMk+2HRCKlLssjj+7lq2rdg1/RoHU9Co945TfSu", - "value_start": 0, - "value_end": 64, - "variable": null, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 5.25352441389348, - "valid": true - } - }, - { - "line": "Vu3nY3K7GQsHp8juCm1wngL84c334uzANATNKDQvYZFy/pzphYP/jk8SMu7ygYPD", - "line_num": 48, - "path": "tests/samples/pem_key", - "info": "", - "value": "Vu3nY3K7GQsHp8juCm1wngL84c334uzANATNKDQvYZFy/pzphYP/jk8SMu7ygYPD", - "value_start": 0, - "value_end": 64, - "variable": null, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 5.038909765557392, - "valid": true - } - }, - { - "line": "/jsbTG+tczu1/LwuwiAFxY7xg30Wg7LG80omwbLv+ohrQjhhH8/c3jVbO2UZA1u8", - "line_num": 49, - "path": "tests/samples/pem_key", - "info": "", - "value": "/jsbTG+tczu1/LwuwiAFxY7xg30Wg7LG80omwbLv+ohrQjhhH8/c3jVbO2UZA1u8", - "value_start": 0, - "value_end": 64, - "variable": null, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 5.061889062229568, - "valid": true - } - }, - { - "line": "NPTe+ZwCOGG0b8nW5skHetkdJpH39+5ZjLD6wYbZHK7+EwmFE5JwtBMMk7xDeUT0", - "line_num": 50, - "path": "tests/samples/pem_key", - "info": "", - "value": "NPTe+ZwCOGG0b8nW5skHetkdJpH39+5ZjLD6wYbZHK7+EwmFE5JwtBMMk7xDeUT0", - "value_start": 0, - "value_end": 64, - "variable": null, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 5.116729296672176, - "valid": true - } - }, - { - "line": "/URS4tdN02Ty4h5x9Uv/XK6Cq5yd9p7obSvFIjY6DavFIZebDeraHb+aQtXESE4K", - "line_num": 51, - "path": "tests/samples/pem_key", - "info": "", - "value": "/URS4tdN02Ty4h5x9Uv/XK6Cq5yd9p7obSvFIjY6DavFIZebDeraHb+aQtXESE4K", - "value_start": 0, - "value_end": 64, - "variable": null, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 5.15977441389348, - "valid": true - } - }, - { - "line": "vNBr+lPd7zwbIO4/1Z18JeQdK2bGLjf//////////wIBAgICARMEJQIjB8TRLx6q", - "line_num": 52, - "path": "tests/samples/pem_key", - "info": "", - "value": "vNBr+lPd7zwbIO4/1Z18JeQdK2bGLjf//////////wIBAgICARMEJQIjB8TRLx6q", - "value_start": 0, - "value_end": 64, - "variable": null, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 4.86236507332541, - "valid": true - } - }, - { - "line": "XYQJ0RAM+5ztVLhy9EXNdjY0EYODS7TFi5RZLE4=", - "line_num": 53, - "path": "tests/samples/pem_key", - "info": "", - "value": "XYQJ0RAM+5ztVLhy9EXNdjY0EYODS7TFi5RZLE4=", - "value_start": 0, - "value_end": 40, - "variable": null, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 4.834183719779189, - "valid": true - } - }, - { - "line": "-----END PRIVATE KEY-----", - "line_num": 54, - "path": "tests/samples/pem_key", - "info": "", - "value": "-----END PRIVATE KEY-----", - "value_start": 0, - "value_end": 25, - "variable": null, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.224609718596318, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", diff --git a/tests/filters/test_value_pattern_check.py b/tests/filters/test_value_pattern_check.py index 8c9112cc13..41c011f3cb 100644 --- a/tests/filters/test_value_pattern_check.py +++ b/tests/filters/test_value_pattern_check.py @@ -4,7 +4,7 @@ import pytest from credsweeper.config import Config -from credsweeper.filters import ValuePatternCheck, ValuePemPatternCheck +from credsweeper.filters import ValuePatternCheck from tests.filters.conftest import LINE_VALUE_PATTERN, DUMMY_ANALYSIS_TARGET from tests.test_utils.dummy_line_data import get_line_data @@ -20,12 +20,9 @@ def test_equal_pattern_check_n(self) -> None: self.assertFalse(ValuePatternCheck(self.config).equal_pattern_check("Crackle123")) self.assertFalse(ValuePatternCheck(self.config).equal_pattern_check("IEEE32441")) self.assertFalse(ValuePatternCheck(self.config).equal_pattern_check("Pass...")) - self.assertFalse(ValuePemPatternCheck(self.config).equal_pattern_check("AAAABCD")) def test_equal_pattern_check_p(self) -> None: self.assertTrue(ValuePatternCheck(self.config).equal_pattern_check("AAAABCD")) - self.assertFalse(ValuePemPatternCheck(self.config).equal_pattern_check("AAAABCD")) - self.assertTrue(ValuePemPatternCheck(self.config).equal_pattern_check("AAAAABCD")) self.assertTrue(ValuePatternCheck(self.config).equal_pattern_check("-------BEGIN")) self.config.pattern_len = 8 self.assertFalse(ValuePatternCheck(self.config).equal_pattern_check("-------BEGIN")) diff --git a/tests/samples/pem_key b/tests/samples/pem_key index c11344732e..81d6cd1dc8 100644 --- a/tests/samples/pem_key +++ b/tests/samples/pem_key @@ -1,6 +1,27 @@ ------BEGIN PRIVATE KEY----- -MIGiAgEAMBQGByqGSM49AgEGCSskAwMCCAEBCQSBhjCBgwIBAQQoRBvhspL4HZA6 - YcfouziNWFuE1e1RcN69xlsJ8OGZEMobV29pntQlSaFUA1IABFDmVKAu6UDU1doe - 67Fzp3WeLAKNcaLjX13W0X704bqNvSU7q747RCCbd46miRYsiqYejnsX0A/A5bOi - I1LJixCGcAgm1pnGajKeok/ecsl+ ------END PRIVATE KEY----- +# valious line ending for sanitize +value = "-----BEGIN OPENSSH PRIVATE KEY-----" \ +"Proc-Type: 4,ENCRYPTED\nDEK-Info: AES-128-CBC,91ABCDB07DE3D352A7A59A3A7427C7E4" + ++"b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW", +// "QyNTUxOQAAACBqIPMG94HL7zedFzsvi45m" + + "HS8ZuyLQXqvHpHobcdNCJAAAAJimRM7VpkTO" + "QAAAAtzc2gtZWQyNTUxOQAAACBqIPMG94HL7zedFzsvi45mHS8ZuyLQXqvHpHo12 \n" + " AAAEBvVc8FVPGUs3LZ1o+LnjW4uUlEnk/5LQQ9yO2eiI3SFGog8wb3gcvvN50XOy+LjmYd" + +"Lxm7ItBeq8ekehtx00IkAAAAEWlvYW5uaXNAc2VjdXJlYm94AQID\nBA== \n -----END OPENSSH PRIVATE KEY-----" + +# all private key in single line +char pk[] = "-----BEGIN OPENSSH PRIVATE KEY-----\n\ni7aHavqQ9T2f2drU4N5WsM7aqQ97kbB/K4RUPdit+tIpqSb1GgY44yg6lckfoLMH\najimpQ6sr9BuseERqELGE1U+Vll3izwuqr1UzCZ61gZnwPDBsD02jF038wYU6mZT\n-----END RSA PRIVATE KEY-----\n"; + +# fake with spaces inside +-----BEGIN OPENSSH FAKE PRIVATE KEY----- +i7aHavqQ9T2f2drWsM7aqQ97kbB/K4RUPdit+tIpqSb1GgY44yg6lckfoLMH not a real key ajimpQ6sr9BuseERqELGE1U+Vll3izwuqr1UzCZ61gZn +-----END OPENSSH FAKE PRIVATE KEY----- + +# fake with ellipsis +-----BEGIN SAMPLE PRIVATE KEY----- +i7aHavqQ9T2f2drU4N5WsM7aqQ97kbB/K4RUPdit+tIpqSb1GgYdbj4...impQ6sr9BuseERqELGE1U+Vll3izwuqr1UzCZ61gZnwPDBsD02jF038wYU6mZT +-----END SAMPLE PRIVATE KEY----- + +# low entropy fake key +-----BEGIN OPENSSH LOW ENTROPY PRIVATE KEY----- +12345678901231278634987284736283548102438723941563428762374129402103402394932746672734543664375t7323341253845186253784== +-----END LOW ENTROPY PRIVATE KEY-----