Merge branch 'main' into dev

.github/workflows/integration_sh.yaml

@@ -23,8 +23,8 @@ jobs:
             run: |
               pip install requests
               mkdir actions-runner && cd actions-runner
-              curl -o actions-runner-linux-x64-2.300.2.tar.gz -L https://github.com/actions/runner/releases/download/v2.300.2/actions-runner-linux-x64-2.300.2.tar.gz
-              tar xzf ./actions-runner-linux-x64-2.300.2.tar.gz
+              curl -o actions-runner-linux-x64-2.312.0.tar.gz -L https://github.com/actions/runner/releases/download/v2.312.0/actions-runner-linux-x64-2.312.0.tar.gz
+              tar xzf ./actions-runner-linux-x64-2.312.0.tar.gz
               python ../test/runner_helper.py register
               nohup ./run.sh &
           - name: Sleep

README.md

@@ -8,28 +8,43 @@
 
 
 Gato, or GitHub Attack Toolkit, is an enumeration and attack tool that allows both 
-blue teamers and offensive security practitioners to evaluate the blast radius 
-of a compromised personal access token within a GitHub organization.
+blue teamers and offensive security practitioners to identify and exploit 
+pipeline vulnerabilities within a GitHub organization's public and private 
+repositories.
 
-The tool also allows searching for and thoroughly enumerating public
-repositories that utilize self-hosted runners. GitHub recommends that
-self-hosted runners only be utilized for private repositories, however, there
-are thousands of organizations that utilize self-hosted runners.
+The tool has post-exploitation features to leverage a compromised personal
+access token in addition to enumeration features to identify poisoned pipeline
+execution vulnerabilities against public repositories that use self-hosted GitHub Actions 
+runners.
 
-## Version 1.5 Released
+GitHub recommends that self-hosted runners only be utilized for private repositories, however, there are thousands of organizations that utilize self-hosted runners. Default configurations are often vulnerable, and Gato uses a mix of workflow file analysis and run-log analysis to identify potentially vulnerable repositories at scale.
 
-Gato version 1.5 was released on June 27th, 2023!
+## Version 1.6
 
-#### New Features
+Gato version 1.6 improves the public repository enumeration feature set.
 
-* Secrets Enumeration
-* Secrets Exfiltration
-* API-only Enumeration
-* JSON Output
-* Improved Code Search
-* GitHub Enterprise Server Support
-* PAT Validation Only Mode
-* Quality of life and UX improvements
+Previously, Gato's code search functionality by default only looked for
+yaml files that explicitly had "self-hosted" in the name. Now, the
+code search functionality supports a SourceGraph query. This query has a 
+lower false negative rate and is not limited by GitHub's code search limit.
+
+For example, the following query will identify public repositories that use 
+self-hosted runners:
+
+`gato search --sourcegraph --output-text public_repos.txt`
+
+This can be fed back into Gato's enumeration feature:
+
+`gato enumerate --repositories public_repos.txt --output-json enumeration_results.json`
+
+Additionally the release contains several improvements under the hood to speed up the enumeration process. This includes changes to limit redundant run-log downloads (which are the slowest part of Gato's enumeration process) and using the GraphQL API to download workflow files when enumerating an entire organization. Finally, Gato will use a heuristic to detect if an attached runner is non-ephemeral. Most poisoned pipeline execution attacks require a non-ephemeral runner in order to exploit.
+
+### New Features
+
+* SourceGraph Search Functionality
+* Improved Public Repository Enumeration Speed
+* Improved Workflow File Analysis
+* Non-ephemeral self-hosted runner detection
 
 ## Who is it for?
 
@@ -44,6 +59,7 @@ Gato version 1.5 was released on June 27th, 2023!
 
 * GitHub Classic PAT Privilege Enumeration
 * GitHub Code Search API-based enumeration
+* SourceGraph Search enumeration
 * GitHub Action Run Log Parsing to identify Self-Hosted Runners
 * Bulk Repo Sparse Clone Features
 * GitHub Action Workflow Parsing

gato/attack/attack.py

@@ -582,7 +582,7 @@ def secrets_dump(
                 if len(blob) == 2:
                     cleartext = Attacker.__decrypt_secrets(priv_key, blob)
                     Output.owned("Decrypted and Decoded Secrets:")
-                    print(cleartext)
+                    print(cleartext.decode())
 
                 else:
                     Output.error(

gato/enumerate/repository.py

@@ -188,7 +188,7 @@ def construct_workflow_cache(self, yml_results):
 
         Args:
             yml_results (list): List of results from individual GraphQL queries
-            (100 nodes at atime).)
+            (100 nodes at a time).
         """
         for result in yml_results:
             owner = result['nameWithOwner']

gato/github/api.py

@@ -729,7 +729,6 @@ def retrieve_run_logs(self, repo_name: str, short_circuit: str = True):
                 run_log = self.call_get(
                     f'/repos/{repo_name}/actions/runs/{run["id"]}/'
                     f'attempts/{run["run_attempt"]}/logs')
-
                 if run_log.status_code == 200:
                     run_log = self.__process_run_log(run_log.content, run)
                     if run_log:
@@ -1205,4 +1204,4 @@ def commit_workflow(self, repo_name: str,
         if self.__verify_result(r, 201) is False:
             return None
 
-        return new_commit_sha
+        return new_commit_sha

gato/github/gql_queries.py

@@ -31,7 +31,7 @@ class GqlQueries():
 
     @staticmethod
     def get_workflow_ymls(repos: list):
-        """Retrieve workflow yml files for ea
+        """Retrieve workflow yml files for each repository.
 
         Args:
             repos (List[Repository]): List of repository objects
@@ -51,4 +51,4 @@ def get_workflow_ymls(repos: list):
             }
 
             queries.append(query)
-        return queries
+        return queries

gato/search/search.py

@@ -107,13 +107,24 @@ def use_sourcegraph_api(
                 if line and line.decode().startswith("data:"):
                     json_line = line.decode().replace("data:", "").strip()
                     event = json.loads(json_line)
+
+                    if "title" in event and event["title"] == "Unable To Process Query":
+                        Output.error("SourceGraph was unable to process the query!")
+                        Output.error(f"Error: {Output.bright(event['description'])}")
+                        return False
+
                     for element in event:
                         if "repository" in element:
                             results.add(
                                 element["repository"].replace("github.com/", "")
                             )
+        else:
+            Output.error(
+                f"SourceGraph returned an error: {Output.bright(response.status_code)}"
+            )
+            return False
 
-        return results
+        return sorted(results)
 
     def use_search_api(self, organization: str, query=None):
         """Utilize GitHub Code Search API to try and identify repositories
@@ -153,7 +164,7 @@ def use_search_api(self, organization: str, query=None):
             organization, custom_query=query
         )
 
-        return candidates
+        return sorted(candidates)
 
     def present_results(self, results, output_text=None):
         """

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "praetorian-gato"
-version = "1.5.1"
+version = "1.6.0"
 description = "GitHub Actions Enumeration and Attack Framework"
 readme = "readme.md"
 authors = [

test/test_cases.json

@@ -23,7 +23,7 @@
                 "type": "stdout"
             },
             {
-                "expect": "About to enumerate 4 repos within the GHRunnerPlayground organization!",
+                "expect": "About to enumerate 3 repos within the GHRunnerPlayground organization!",
                 "type": "stdout"
             },
             {
@@ -54,7 +54,7 @@
                 "type": "stdout"
             },
             {
-                "expect": "About to enumerate 4 repos within the GHRunnerPlayground organization!",
+                "expect": "About to enumerate 3 repos within the GHRunnerPlayground organization!",
                 "type": "stdout"
             }
         ],
@@ -73,7 +73,7 @@
                 "type": "stdout"
             },
             {
-                "expect": "About to enumerate 3 repos within the GHRunnerPlayground organization!",
+                "expect": "About to enumerate 1 repos within the GHRunnerPlayground organization!",
                 "type": "stdout"
             }
         ],
@@ -88,7 +88,7 @@
                 "type": "stdout"
             },
             {
-                "expect": "About to enumerate 4 repos within the GHRunnerPlayground organization!",
+                "expect": "About to enumerate 3 repos within the GHRunnerPlayground organization!",
                 "type": "stdout"
             },
             {
@@ -111,7 +111,7 @@
                 "type": "stdout"
             },
             {
-                "expect": "About to enumerate 4 repos within the GHRunnerPlayground organization!",
+                "expect": "About to enumerate 3 repos within the GHRunnerPlayground organization!",
                 "type": "stdout"
             },
             {
@@ -242,26 +242,26 @@
                 "type": "stdout"
             },
             {
-                "expect": "INTEGRATION_SECRET, last updated 2023-06-22T01:47:43Z",
+                "expect": "INTEGRATION_SECRET, last updated",
                 "type": "stdout"
             }
         ],
         "extra_validation": {"type":"none"}
     },
     {
         "PAT": "ORG_ADMIN_REPO_WORKFLOW_TOKEN",
-        "invocation": "gato -s e -r GHRunnerPlayground/md-test -sr",
+        "invocation": "gato -s e -r GHRunnerPlayground/TestSecrets -sr",
         "assertions": [
             {
                 "expect": "The repository can access 2 secrets and the token can use a workflow to read",
                 "type": "stdout"
             },
             {
-                "expect": "REPO_SECRET, last updated 2023-04-20T18:01:37Z",
+                "expect": "REPO_SECRET, last updated",
                 "type": "stdout"
             },
             {
-                "expect": "INTEGRATION_SECRET, last updated 2023-06-22T01:47:43Z",
+                "expect": "INTEGRATION_SECRET, last updated",
                 "type": "stdout"
             }
         ],
@@ -276,7 +276,7 @@
                 "type": "stdout"
             },
             {
-                "expect": "About to enumerate 4 repos within the GHRunnerPlayground organization!",
+                "expect": "About to enumerate 3 repos within the GHRunnerPlayground organization!",
                 "type": "stdout"
             },
             {
@@ -299,7 +299,7 @@
         "invocation": "gato -s enum --target GHRunnerPlayground -oJ integration_test.json",
         "assertions": [
             {
-                "expect": "About to enumerate 4 repos within the GHRunnerPlayground organization!",
+                "expect": "About to enumerate 3 repos within the GHRunnerPlayground organization!",
                 "type": "stdout"
             },
             {