diff --git a/README.md b/README.md index f6cf175..3633a38 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,7 @@ You have two options. I'd recommend the first as it will give you access to the 4. Bring up the stack: `docker-compose up` 5. Open up http://localhost:8080/ + ### via Go get _Note_: this method does not include the shhgit web interface @@ -40,7 +41,8 @@ _Note_: this method does not include the shhgit web interface shhgit can work in two ways: consuming the public APIs of GitHub, Gist, GitLab and BitBucket or by processing files in a local directory. -By default, shhgit will run in the former 'public mode'. For GitHub and Gist, you will need to obtain and provide an access token (see [this guide](https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line); it doesn't require any scopes or permissions. And then place it under `github_access_tokens` in `config.yaml`). GitLab and BitBucket do not require any API tokens. +By default, shhgit will run in the former 'public mode'. For GitHub and Gist, you will need to obtain and provide an access token (see [this guide](https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line); it doesn't require any scopes or permissions. And then place it under `github_access_tokens` in `config.yaml`). +For GitHub Enterprise (GHE) a token is also needed, but rate limiting is configurable and your token might need read access to the repositories in questions depending on the configuration of your GHE instance. GitLab and BitBucket do not require any API tokens. You can also forgo the signatures and use shhgit with your own custom search query, e.g. to find all AWS keys you could use `shhgit --search-query AWS_ACCESS_KEY_ID=AKIA`. And to run in local mode (and perhaps integrate in to your CI pipelines) you can pass the `--local` flag (see usage below). @@ -87,6 +89,7 @@ The `config.yaml` file has 7 elements. A [default is provided](https://github.co github_access_tokens: # provide at least one token - 'token one' - 'token two' +github_enterprise_url: '' # url to your github enterprise (optional) webhook: '' # URL to a POST webhook. webhook_payload: '' # Payload to POST to the webhook URL blacklisted_strings: [] # list of strings to ignore diff --git a/config.yaml b/config.yaml index 359e81a..c405d36 100644 --- a/config.yaml +++ b/config.yaml @@ -1,5 +1,6 @@ github_access_tokens: - '' +github_enterprise_url: '' webhook: '' # URL to which the payload is POSTed # This default payload will work for Slack and MatterMost. diff --git a/core/config.go b/core/config.go index 9170835..56e7622 100644 --- a/core/config.go +++ b/core/config.go @@ -13,6 +13,7 @@ import ( type Config struct { GitHubAccessTokens []string `yaml:"github_access_tokens"` + GitHubEnterpriseUrl string `yaml:"github_enterprise_url"` Webhook string `yaml:"webhook,omitempty"` WebhookPayload string `yaml:"webhook_payload,omitempty"` BlacklistedStrings []string `yaml:"blacklisted_strings"` diff --git a/core/git.go b/core/git.go index a7b24fe..44726ea 100644 --- a/core/git.go +++ b/core/git.go @@ -2,6 +2,7 @@ package core import ( "context" + "net/url" "strings" "time" @@ -24,22 +25,33 @@ type GitResource struct { Url string } -func CloneRepository(session *Session, url string, dir string) (*git.Repository, error) { +func CloneRepository(session *Session, rawUrl string, dir string) (*git.Repository, error) { timeout := time.Duration(*session.Options.CloneRepositoryTimeout) * time.Second localCtx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() + + if len(session.Config.GitHubEnterpriseUrl) > 0 { + githubUrl, err := url.Parse(rawUrl) + if err != nil { + return nil, err + } + + userInfo := url.User(session.Config.GitHubAccessTokens[0]) + githubUrl.User = userInfo + rawUrl = githubUrl.String() + } - session.Log.Debug("[%s] Cloning in to %s", url, strings.Replace(dir, *session.Options.TempDirectory, "", -1)) + session.Log.Debug("[%s] Cloning in to %s", rawUrl, strings.Replace(dir, *session.Options.TempDirectory, "", -1)) repository, err := git.PlainCloneContext(localCtx, dir, false, &git.CloneOptions{ Depth: 1, RecurseSubmodules: git.NoRecurseSubmodules, - URL: url, + URL: rawUrl, SingleBranch: true, Tags: git.NoTags, }) if err != nil { - session.Log.Debug("[%s] Cloning failed: %s", url, err.Error()) + session.Log.Debug("[%s] Cloning failed: %s", rawUrl, err.Error()) return nil, err } diff --git a/core/github.go b/core/github.go index d4a8d08..ea2a475 100644 --- a/core/github.go +++ b/core/github.go @@ -50,7 +50,7 @@ func GetRepositories(session *Session) { GetSession().Log.Warn("Error getting GitHub events... trying again", err) } - if opt.Page == 0 { + if opt.Page == 0 && resp.Rate.Limit > 0 { session.Log.Warn("Token %s[..] has %d/%d calls remaining.", client.Token[:10], resp.Rate.Remaining, resp.Rate.Limit) } @@ -152,10 +152,11 @@ func GetRepository(session *Session, id int64) (*github.Repository, error) { repo, resp, err := client.Repositories.GetByID(session.Context, id) if err != nil { + session.Log.Warn("Got error %s", err) return nil, err } - if resp.Rate.Remaining <= 1 { + if resp.Rate.Remaining <= 1 && resp.Rate.Limit > 0 { session.Log.Warn("Token %s[..] rate limited. Reset at %s", client.Token[:10], resp.Rate.Reset) client.RateLimitedUntil = resp.Rate.Reset.Time } diff --git a/core/session.go b/core/session.go index bd06e1b..4cd304b 100644 --- a/core/session.go +++ b/core/session.go @@ -5,8 +5,10 @@ import ( "encoding/csv" "fmt" "math/rand" + "net/url" "os" "runtime" + "strings" "sync" "time" @@ -65,7 +67,23 @@ func (s *Session) InitGitHubClients() { ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token}) tc := oauth2.NewClient(s.Context, ts) - client := github.NewClient(tc) + client := github.NewClient(tc) + enterpriseUrl := s.Config.GitHubEnterpriseUrl + + if len(enterpriseUrl) > 0 { + baseEndpoint, err := url.Parse(enterpriseUrl) + + if err != nil { + s.Log.Warn("Failed to parse GitHubEnterpriseUrl %s[..]: %s", enterpriseUrl, err) + return + } + + if !strings.HasSuffix(baseEndpoint.Path, "/api/v3/") { + baseEndpoint.Path += "api/v3/" + } + + client.BaseURL = baseEndpoint + } client.UserAgent = fmt.Sprintf("%s v%s", Name, Version) _, _, err := client.Users.Get(s.Context, "")