Merge pull request #14 from hueristiq/dev

Development v0.6.0
hueristiq · Nov 27, 2023 · 89e5bc0 · 89e5bc0
2 parents 57bc303 + f1d3d3e
commit 89e5bc0
Show file tree

Hide file tree

Showing 22 changed files with 83 additions and 110 deletions.
diff --git a/README.md b/README.md
@@ -111,7 +111,7 @@ Example `config.yaml`:
 > **NOTE:** The keys/tokens below are invalid and used as examples, use your own keys/tokens!
 
 ```yaml
-version: 0.5.0
+version: 0.6.0
 sources:
     - alienvault
     - anubis
@@ -170,7 +170,7 @@ __  _____ _   _| |__  / _(_)_ __   __| |___ / _ __
 \ \/ / __| | | | '_ \| |_| | '_ \ / _` | |_ \| '__|
  >  <\__ \ |_| | |_) |  _| | | | | (_| |___) | |
 /_/\_\___/\__,_|_.__/|_| |_|_| |_|\__,_|____/|_|
-                                             v0.5.0
+                                             v0.6.0
 
                    with <3 by Hueristiq Open Source
 

diff --git a/go.mod b/go.mod
@@ -16,9 +16,9 @@ require (
 require (
 	github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 // indirect
 	github.com/hueristiq/hqgoutils v0.0.0-20231024005153-bd2c47932440 // indirect
-	golang.org/x/net v0.18.0 // indirect
-	golang.org/x/sys v0.14.0 // indirect
-	golang.org/x/term v0.14.0 // indirect
+	golang.org/x/net v0.19.0 // indirect
+	golang.org/x/sys v0.15.0 // indirect
+	golang.org/x/term v0.15.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
 )
diff --git a/go.sum b/go.sum
@@ -35,12 +35,12 @@ github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcU
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 h1:nrZ3ySNYwJbSpD6ce9duiP+QkD3JuLCcWkdaehUS/3Y=
 github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80/go.mod h1:iFyPdL66DjUD96XmzVL3ZntbzcflLnznH0fr99w5VqE=
-golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg=
-golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ=
-golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q=
-golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/term v0.14.0 h1:LGK9IlZ8T9jvdy6cTdfKUCltatMFOehAQo9SRC46UQ8=
-golang.org/x/term v0.14.0/go.mod h1:TySc+nGkYR6qt8km8wUhuFRTVSMIX3XPR58y2lC8vww=
+golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c=
+golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U=
+golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
+golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4=
+golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
 golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

diff --git a/internal/configuration/configuration.go b/internal/configuration/configuration.go
@@ -47,7 +47,7 @@ func (cfg *Configuration) Write(path string) (err error) {
 
 const (
 	NAME    string = "xsubfind3r"
-	VERSION string = "0.5.0"
+	VERSION string = "0.6.0"
 )
 
 var (

diff --git a/pkg/httpclient/client.go b/pkg/httpclient/client.go
@@ -74,3 +74,14 @@ func SimpleGet(URL string) (*http.Response, error) {
 func Post(URL, cookies string, headers map[string]string, body io.Reader) (*http.Response, error) {
 	return HTTPRequest(methods.Post, URL, cookies, headers, body)
 }
+
+func DiscardResponse(response *http.Response) {
+	if response != nil {
+		_, err := io.Copy(io.Discard, response.Body)
+		if err != nil {
+			return
+		}
+
+		response.Body.Close()
+	}
+}
diff --git a/pkg/scraper/sources/anubis/anubis.go b/pkg/scraper/sources/anubis/anubis.go
@@ -33,7 +33,7 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source
 
 			results <- result
 
-			getSubdomainsRes.Body.Close()
+			httpclient.DiscardResponse(getSubdomainsRes)
 
 			return
 		}
@@ -56,9 +56,7 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source
 
 		getSubdomainsRes.Body.Close()
 
-		for index := range getSubdomainsResData {
-			subdomain := getSubdomainsResData[index]
-
+		for _, subdomain := range getSubdomainsResData {
 			result := sources.Result{
 				Type:   sources.Subdomain,
 				Source: source.Name(),

diff --git a/pkg/scraper/sources/bevigil/bevigil.go b/pkg/scraper/sources/bevigil/bevigil.go
@@ -59,7 +59,7 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s
 
 			results <- result
 
-			getSubdomainsRes.Body.Close()
+			httpclient.DiscardResponse(getSubdomainsRes)
 
 			return
 		}
@@ -82,9 +82,7 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s
 
 		getSubdomainsRes.Body.Close()
 
-		for index := range getSubdomainsResData.Subdomains {
-			subdomain := getSubdomainsResData.Subdomains[index]
-
+		for _, subdomain := range getSubdomainsResData.Subdomains {
 			result := sources.Result{
 				Type:   sources.Subdomain,
 				Source: source.Name(),

diff --git a/pkg/scraper/sources/bufferover/bufferover.go b/pkg/scraper/sources/bufferover/bufferover.go
@@ -62,7 +62,7 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s
 
 			results <- result
 
-			getTLSLogsSearchRes.Body.Close()
+			httpclient.DiscardResponse(getTLSLogsSearchRes)
 
 			return
 		}
@@ -119,13 +119,10 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s
 			entries = getTLSLogsSearchResData.Results
 		}
 
-		for index := range entries {
-			entry := entries[index]
+		for _, entry := range entries {
 			subdomains := regex.FindAllString(entry, -1)
 
-			for index := range subdomains {
-				subdomain := subdomains[index]
-
+			for _, subdomain := range subdomains {
 				result := sources.Result{
 					Type:   sources.Subdomain,
 					Source: source.Name(),

diff --git a/pkg/scraper/sources/builtwith/builtwith.go b/pkg/scraper/sources/builtwith/builtwith.go
@@ -60,7 +60,7 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s
 
 			results <- result
 
-			getDomainInfoRes.Body.Close()
+			httpclient.DiscardResponse(getDomainInfoRes)
 
 			return
 		}
@@ -83,12 +83,8 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s
 
 		getDomainInfoRes.Body.Close()
 
-		for index := range getDomainInfoResData.Results {
-			item := getDomainInfoResData.Results[index]
-
-			for index := range item.Result.Paths {
-				path := item.Result.Paths[index]
-
+		for _, item := range getDomainInfoResData.Results {
+			for _, path := range item.Result.Paths {
 				value := path.Domain
 
 				if path.SubDomain != "" {

diff --git a/pkg/scraper/sources/certspotter/certspotter.go b/pkg/scraper/sources/certspotter/certspotter.go
@@ -60,7 +60,7 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s
 
 			results <- result
 
-			getCTLogsSearchRes.Body.Close()
+			httpclient.DiscardResponse(getCTLogsSearchRes)
 
 			return
 		}
@@ -87,12 +87,8 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s
 			return
 		}
 
-		for index := range getCTLogsSearchResData {
-			cert := getCTLogsSearchResData[index]
-
-			for index := range cert.DNSNames {
-				subdomain := cert.DNSNames[index]
-
+		for _, cert := range getCTLogsSearchResData {
+			for _, subdomain := range cert.DNSNames {
 				if subdomain != domain && !strings.HasSuffix(subdomain, "."+domain) {
 					continue
 				}
@@ -124,9 +120,9 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s
 
 				results <- result
 
-				getCTLogsSearchRes.Body.Close()
+				httpclient.DiscardResponse(getCTLogsSearchRes)
 
-				continue
+				break
 			}
 
 			var getCTLogsSearchResData []getCTLogsSearchResponse
@@ -142,21 +138,17 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s
 
 				getCTLogsSearchRes.Body.Close()
 
-				continue
+				break
 			}
 
 			getCTLogsSearchRes.Body.Close()
 
 			if len(getCTLogsSearchResData) == 0 {
-				return
+				break
 			}
 
-			for index := range getCTLogsSearchResData {
-				cert := getCTLogsSearchResData[index]
-
-				for index := range cert.DNSNames {
-					subdomain := cert.DNSNames[index]
-
+			for _, cert := range getCTLogsSearchResData {
+				for _, subdomain := range cert.DNSNames {
 					if subdomain != domain && !strings.HasSuffix(subdomain, "."+domain) {
 						continue
 					}

diff --git a/pkg/scraper/sources/chaos/chaos.go b/pkg/scraper/sources/chaos/chaos.go
@@ -56,7 +56,7 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s
 
 			results <- result
 
-			getSubdomainsRes.Body.Close()
+			httpclient.DiscardResponse(getSubdomainsRes)
 
 			return
 		}
@@ -79,9 +79,7 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s
 
 		getSubdomainsRes.Body.Close()
 
-		for index := range getSubdomainsResData.Subdomains {
-			subdomain := getSubdomainsResData.Subdomains[index]
-
+		for _, subdomain := range getSubdomainsResData.Subdomains {
 			result := sources.Result{
 				Type:   sources.Subdomain,
 				Source: source.Name(),

diff --git a/pkg/scraper/sources/commoncrawl/commoncrawl.go b/pkg/scraper/sources/commoncrawl/commoncrawl.go
@@ -70,7 +70,7 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source
 
 			results <- result
 
-			getIndexesRes.Body.Close()
+			httpclient.DiscardResponse(getIndexesRes)
 
 			return
 		}
@@ -103,12 +103,8 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source
 
 		searchIndexes := make(map[string]string)
 
-		for index := range years {
-			year := years[index]
-
-			for index := range getIndexesResData {
-				CCIndex := getIndexesResData[index]
-
+		for _, year := range years {
+			for _, CCIndex := range getIndexesResData {
 				if strings.Contains(CCIndex.ID, year) {
 					if _, ok := searchIndexes[year]; !ok {
 						searchIndexes[year] = CCIndex.API
@@ -138,7 +134,7 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source
 
 				results <- result
 
-				getPaginationRes.Body.Close()
+				httpclient.DiscardResponse(getPaginationRes)
 
 				continue
 			}
@@ -180,7 +176,7 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source
 
 					results <- result
 
-					getURLsRes.Body.Close()
+					httpclient.DiscardResponse(getURLsRes)
 
 					continue
 				}
@@ -219,11 +215,9 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source
 						continue
 					}
 
-					match := regex.FindAllString(getURLsResData.URL, -1)
-
-					for index := range match {
-						subdomain := match[index]
+					subdomains := regex.FindAllString(getURLsResData.URL, -1)
 
+					for _, subdomain := range subdomains {
 						result := sources.Result{
 							Type:   sources.Subdomain,
 							Source: source.Name(),

diff --git a/pkg/scraper/sources/crtsh/crtsh.go b/pkg/scraper/sources/crtsh/crtsh.go
@@ -39,7 +39,7 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source
 
 			results <- result
 
-			getNameValuesRes.Body.Close()
+			httpclient.DiscardResponse(getNameValuesRes)
 
 			return
 		}
@@ -62,13 +62,10 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source
 
 		getNameValuesRes.Body.Close()
 
-		for index := range getNameValuesResData {
-			record := getNameValuesResData[index]
+		for _, record := range getNameValuesResData {
 			subdomains := strings.Split(record.NameValue, "\n")
 
-			for index := range subdomains {
-				subdomain := subdomains[index]
-
+			for _, subdomain := range subdomains {
 				if subdomain != domain && !strings.HasSuffix(subdomain, "."+domain) {
 					continue
 				}

diff --git a/pkg/scraper/sources/fullhunt/fullhunt.go b/pkg/scraper/sources/fullhunt/fullhunt.go
@@ -60,7 +60,7 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s
 
 			results <- result
 
-			getSubdomainsRes.Body.Close()
+			httpclient.DiscardResponse(getSubdomainsRes)
 
 			return
 		}
@@ -84,9 +84,7 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s
 
 		getSubdomainsRes.Body.Close()
 
-		for index := range getSubdomainsResData.Hosts {
-			subdomain := getSubdomainsResData.Hosts[index]
-
+		for _, subdomain := range getSubdomainsResData.Hosts {
 			result := sources.Result{
 				Type:   sources.Subdomain,
 				Source: source.Name(),

diff --git a/pkg/scraper/sources/github/github.go b/pkg/scraper/sources/github/github.go
@@ -98,7 +98,7 @@ func (source *Source) Enumerate(searchReqURL string, domainRegexp *regexp.Regexp
 
 		results <- result
 
-		searchRes.Body.Close()
+		httpclient.DiscardResponse(searchRes)
 
 		return
 	}
@@ -145,7 +145,7 @@ func (source *Source) Enumerate(searchReqURL string, domainRegexp *regexp.Regexp
 
 			results <- result
 
-			getRawContentRes.Body.Close()
+			httpclient.DiscardResponse(getRawContentRes)
 
 			continue
 		}

diff --git a/pkg/scraper/sources/hackertarget/hackertarget.go b/pkg/scraper/sources/hackertarget/hackertarget.go
@@ -35,7 +35,7 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source
 
 			results <- result
 
-			hostSearchRes.Body.Close()
+			httpclient.DiscardResponse(hostSearchRes)
 
 			return
 		}
@@ -66,9 +66,7 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source
 
 			match := regex.FindAllString(line, -1)
 
-			for index := range match {
-				subdomain := match[index]
-
+			for _, subdomain := range match {
 				result := sources.Result{
 					Type:   sources.Subdomain,
 					Source: source.Name(),
-Original file line number
+Diff line change
@@ Expand Up / @@ -47,7 +47,7 @@ func (cfg *Configuration) Write(path string) (err error) { @@
     const (
     	NAME    string = "xsubfind3r"
-    	VERSION string = "0.5.0"
+    	VERSION string = "0.6.0"
     )
     var (
@@ Expand Down @@