diff --git a/.github/dependabot.yml b/.github/dependabot.yml index d0b098d1c..120bb02ca 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -17,8 +17,9 @@ updates: include: "scope" labels: - "Type: Maintenance" - allow: - - dependency-name: "github.com/projectdiscovery/*" + groups: + modules: + patterns: ["github.com/projectdiscovery/*"] # # Maintain dependencies for GitHub Actions # - package-ecosystem: "github-actions" @@ -42,4 +43,4 @@ updates: # prefix: "chore" # include: "scope" # labels: -# - "Type: Maintenance" \ No newline at end of file +# - "Type: Maintenance" diff --git a/common/httpx/httpx.go b/common/httpx/httpx.go index d5027b79d..256066b2b 100644 --- a/common/httpx/httpx.go +++ b/common/httpx/httpx.go @@ -164,7 +164,13 @@ func New(options *Options) (*HTTPX, error) { } if httpx.Options.HTTPProxy != "" { - proxyURL, parseErr := url.Parse(httpx.Options.HTTPProxy) + httpx.Options.Proxy = httpx.Options.HTTPProxy + } else if httpx.Options.SocksProxy != "" { + httpx.Options.Proxy = httpx.Options.SocksProxy + } + + if httpx.Options.Proxy != "" { + proxyURL, parseErr := url.Parse(httpx.Options.Proxy) if parseErr != nil { return nil, parseErr } diff --git a/common/httpx/option.go b/common/httpx/option.go index 6ef619193..6386213bb 100644 --- a/common/httpx/option.go +++ b/common/httpx/option.go @@ -13,12 +13,15 @@ import ( type Options struct { RandomAgent bool DefaultUserAgent string - HTTPProxy string - SocksProxy string - Threads int - CdnCheck string - ExcludeCdn bool - ExtractFqdn bool + Proxy string + // Deprecated: use Proxy + HTTPProxy string + // Deprecated: use Proxy + SocksProxy string + Threads int + CdnCheck string + ExcludeCdn bool + ExtractFqdn bool // Timeout is the maximum time to wait for the request Timeout time.Duration // RetryMax is the maximum number of retries diff --git a/go.mod b/go.mod index 0ef8d41f8..16fc76f7a 100644 --- a/go.mod +++ b/go.mod @@ -22,31 +22,31 @@ require ( github.com/projectdiscovery/asnmap v1.1.1 github.com/projectdiscovery/cdncheck v1.1.0 github.com/projectdiscovery/clistats v0.1.1 - github.com/projectdiscovery/dsl v0.2.5 + github.com/projectdiscovery/dsl v0.3.13 github.com/projectdiscovery/fastdialer v0.3.0 github.com/projectdiscovery/fdmax v0.0.4 github.com/projectdiscovery/goconfig v0.0.1 - github.com/projectdiscovery/goflags v0.1.64 - github.com/projectdiscovery/gologger v1.1.33 - github.com/projectdiscovery/hmap v0.0.70 + github.com/projectdiscovery/goflags v0.1.67 + github.com/projectdiscovery/gologger v1.1.41 + github.com/projectdiscovery/hmap v0.0.76 github.com/projectdiscovery/mapcidr v1.1.34 github.com/projectdiscovery/networkpolicy v0.1.1 - github.com/projectdiscovery/ratelimit v0.0.54 - github.com/projectdiscovery/rawhttp v0.1.68 - github.com/projectdiscovery/retryablehttp-go v1.0.88 - github.com/projectdiscovery/tlsx v1.1.7 - github.com/projectdiscovery/useragent v0.0.69 - github.com/projectdiscovery/utils v0.3.0 - github.com/projectdiscovery/wappalyzergo v0.1.22 + github.com/projectdiscovery/ratelimit v0.0.68 + github.com/projectdiscovery/rawhttp v0.1.84 + github.com/projectdiscovery/retryablehttp-go v1.0.96 + github.com/projectdiscovery/tlsx v1.1.8 + github.com/projectdiscovery/useragent v0.0.87 + github.com/projectdiscovery/utils v0.4.8 + github.com/projectdiscovery/wappalyzergo v0.2.12 github.com/rs/xid v1.5.0 github.com/spaolacci/murmur3 v1.1.0 - github.com/stretchr/testify v1.9.0 + github.com/stretchr/testify v1.10.0 github.com/zmap/zcrypto v0.0.0-20240512203510-0fef58d9a9db go.etcd.io/bbolt v1.3.10 // indirect go.uber.org/multierr v1.11.0 golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 - golang.org/x/net v0.33.0 - golang.org/x/sys v0.28.0 // indirect + golang.org/x/net v0.34.0 + golang.org/x/sys v0.29.0 // indirect golang.org/x/text v0.21.0 ) @@ -119,7 +119,7 @@ require ( github.com/projectdiscovery/freeport v0.0.7 // indirect github.com/projectdiscovery/gostruct v0.0.2 // indirect github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect - github.com/projectdiscovery/retryabledns v1.0.87 // indirect + github.com/projectdiscovery/retryabledns v1.0.94 // indirect github.com/refraction-networking/utls v1.6.7 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/rogpeppe/go-internal v1.12.0 // indirect @@ -151,11 +151,11 @@ require ( github.com/yusufpapurcu/wmi v1.2.4 // indirect github.com/zcalusic/sysinfo v1.0.2 // indirect github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 // indirect - golang.org/x/crypto v0.31.0 // indirect + golang.org/x/crypto v0.32.0 // indirect golang.org/x/mod v0.17.0 // indirect golang.org/x/oauth2 v0.18.0 // indirect golang.org/x/sync v0.10.0 // indirect - golang.org/x/term v0.27.0 // indirect + golang.org/x/term v0.28.0 // indirect golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect google.golang.org/appengine v1.6.8 // indirect diff --git a/go.sum b/go.sum index 6b0c43be8..e47c26a58 100644 --- a/go.sum +++ b/go.sum @@ -250,8 +250,8 @@ github.com/projectdiscovery/cdncheck v1.1.0 h1:qDITidmJsejzpk3rMkauCh6sjI2GH9hW/ github.com/projectdiscovery/cdncheck v1.1.0/go.mod h1:sZ8U4MjHSsyaTVjBbYWHT1cwUVvUYwDX1W+WvWRicIc= github.com/projectdiscovery/clistats v0.1.1 h1:8mwbdbwTU4aT88TJvwIzTpiNeow3XnAB72JIg66c8wE= github.com/projectdiscovery/clistats v0.1.1/go.mod h1:4LtTC9Oy//RiuT1+76MfTg8Hqs7FQp1JIGBM3nHK6a0= -github.com/projectdiscovery/dsl v0.2.5 h1:SJzIPGryApvNuAMa/VCe2yPzIDbkoz/pyH8Zt3Wlk+0= -github.com/projectdiscovery/dsl v0.2.5/go.mod h1:aLyfr+br+cgwhHUWdQkIYJz+riWABaJ76jKcPYkcWYE= +github.com/projectdiscovery/dsl v0.3.13 h1:3/2Cd7bl5qS3bpVIKK4Qns2f8/XKPlj82SsztHplHg0= +github.com/projectdiscovery/dsl v0.3.13/go.mod h1:nWkkkfreA+P0nc0O9lalqu5lieIirSYNAD4W9daKd+U= github.com/projectdiscovery/fastdialer v0.3.0 h1:/wMptjdsrAU/wiaA/U3lSgYGaYCGJH6xm0mLei6oMxk= github.com/projectdiscovery/fastdialer v0.3.0/go.mod h1:Q0YLArvpx9GAfY/NcTPMCA9qZuVOGnuVoNYWzKBwxdQ= github.com/projectdiscovery/fdmax v0.0.4 h1:K9tIl5MUZrEMzjvwn/G4drsHms2aufTn1xUdeVcmhmc= @@ -260,38 +260,38 @@ github.com/projectdiscovery/freeport v0.0.7 h1:Q6uXo/j8SaV/GlAHkEYQi8WQoPXyJWxys github.com/projectdiscovery/freeport v0.0.7/go.mod h1:cOhWKvNBe9xM6dFJ3RrrLvJ5vXx2NQ36SecuwjenV2k= github.com/projectdiscovery/goconfig v0.0.1 h1:36m3QjohZvemqh9bkJAakaHsm9iEZ2AcQSS18+0QX/s= github.com/projectdiscovery/goconfig v0.0.1/go.mod h1:CPO25zR+mzTtyBrsygqsHse0sp/4vB/PjaHi9upXlDw= -github.com/projectdiscovery/goflags v0.1.64 h1:FDfwdt9N97Hi8OuhbkDlKtVttpc/CRMIWQVa08VsHsI= -github.com/projectdiscovery/goflags v0.1.64/go.mod h1:3FyHIVQtnycNOc1LE3O1jj/XR5XuMdF9QfHd0ujhnX4= -github.com/projectdiscovery/gologger v1.1.33 h1:wQxaQ8p/0Rx89lowBp0PnY2QSWiqf9QW1vGYAllsVJ4= -github.com/projectdiscovery/gologger v1.1.33/go.mod h1:P/WwqKstshQATJxN39V0KJ9ZuiGLOizmSqHIYrrz1T4= +github.com/projectdiscovery/goflags v0.1.67 h1:WOS1gdirH1Xrig8tYM9QC5UD/dfgABUcvrL0d8/fCPA= +github.com/projectdiscovery/goflags v0.1.67/go.mod h1:cg6+yrLlaekP1hnefBc/UXbH1YGWa0fuzEW9iS1aG4g= +github.com/projectdiscovery/gologger v1.1.41 h1:s5iAOhYDoKnUay5CtvhFpZdGzQKXeARJbpQubThT1Bs= +github.com/projectdiscovery/gologger v1.1.41/go.mod h1:68ltkt+p8WquBsi4CQ2Z2nfrDXhuraC7GQbtH15B0PI= github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M= github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE= -github.com/projectdiscovery/hmap v0.0.70 h1:1TtvmzJNntKbU9CJI7W5auchg62lzgFYaG4BljwrS1o= -github.com/projectdiscovery/hmap v0.0.70/go.mod h1:sugZdHVusZTc45CKtZw7y460C7+JS5SY6ODwlE1i8tI= +github.com/projectdiscovery/hmap v0.0.76 h1:m13e18HEJYkCddcnHucn0XxO/tG8Av6TX+U1dw+PQzQ= +github.com/projectdiscovery/hmap v0.0.76/go.mod h1:AmLVOcAQKX4Iq4vTE/07LONuctMNUOveofI5EHb4oMQ= github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 h1:ZScLodGSezQVwsQDtBSMFp72WDq0nNN+KE/5DHKY5QE= github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983/go.mod h1:3G3BRKui7nMuDFAZKR/M2hiOLtaOmyukT20g88qRQjI= github.com/projectdiscovery/mapcidr v1.1.34 h1:udr83vQ7oz3kEOwlsU6NC6o08leJzSDQtls1wmXN/kM= github.com/projectdiscovery/mapcidr v1.1.34/go.mod h1:1+1R6OkKSAKtWDXE9RvxXtXPoajXTYX0eiEdkqlhQqQ= github.com/projectdiscovery/networkpolicy v0.1.1 h1:iv9gECukD5KAZp98KVh+T3TEPTkY6dr3sKsdbh9XyZU= github.com/projectdiscovery/networkpolicy v0.1.1/go.mod h1:/Hg2ieLewSe/BagFF+UYXAQo3NwmVMq16MSAl492XkU= -github.com/projectdiscovery/ratelimit v0.0.54 h1:VwrFLPSfWle5Hg7AvwQkDd4EWDzYWTzd2mDTXTrqvmA= -github.com/projectdiscovery/ratelimit v0.0.54/go.mod h1:yGIqMaT8vKG+4mlCqkWOWAvBtvWUlQeanZae31E09cY= -github.com/projectdiscovery/rawhttp v0.1.68 h1:6oXatkSkO9a8ybtbhgowQ5wXG0QxmH8a/Ulk4DRsLMM= -github.com/projectdiscovery/rawhttp v0.1.68/go.mod h1:gEF17M4VdXbU9Gn9sZXOY75/bXSduq6XuvZTGxw2/2A= -github.com/projectdiscovery/retryabledns v1.0.87 h1:MPEXVKdu89FEW23xIMpBzzvdegvtcAs7osSqHimBVOs= -github.com/projectdiscovery/retryabledns v1.0.87/go.mod h1:snDTjRcmBj+iveber/o0jC0iLEkM6c0Sdo2IXe2O+fE= -github.com/projectdiscovery/retryablehttp-go v1.0.88 h1:uR6T+i8Sy1isfG1KClhhsXnOqkOR6E8MAvuyOFq3T10= -github.com/projectdiscovery/retryablehttp-go v1.0.88/go.mod h1:ktjiIKyej+plUeK9vksqRf3wGicqY3E1rW84V/O7p0M= +github.com/projectdiscovery/ratelimit v0.0.68 h1:gMLD1aB4R8w7BIpKvtQf6TNb6+5zsJO9WSRWZ9pxwe4= +github.com/projectdiscovery/ratelimit v0.0.68/go.mod h1:ieU9nNu9Ie8nVMKdj3bsX3JA3kfNI8qn4pkNXsyRxsw= +github.com/projectdiscovery/rawhttp v0.1.84 h1:91ctBCABSR2dIMj+hoQz3E0EsmPDNv2YmsjMOwjyJnk= +github.com/projectdiscovery/rawhttp v0.1.84/go.mod h1:F3mb+qlkh/pI+CiaoddmMmgeD2TSPM7WUMcm9DU9KBY= +github.com/projectdiscovery/retryabledns v1.0.94 h1:MvxtRcmvxhxikxT7p/E40hcYRWRiL5fg/JQ8bpBaz+0= +github.com/projectdiscovery/retryabledns v1.0.94/go.mod h1:croGTyMM4yNlrSWA/X7xNe3c0c7mDmCdbm8goLd8Bak= +github.com/projectdiscovery/retryablehttp-go v1.0.96 h1:/88LMgEGVx5rpqdLYIqxyJHBluuREKxJxPVIWi1zOlA= +github.com/projectdiscovery/retryablehttp-go v1.0.96/go.mod h1:YIN8SpckCijt5J1AV+nQApUWyCC3Y+8nAwDfK/RwhWs= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= -github.com/projectdiscovery/tlsx v1.1.7 h1:eSsl/SmTDL/z2CMeSrbssk4f/9oOotMP1SgXl3yynSM= -github.com/projectdiscovery/tlsx v1.1.7/go.mod h1:g66QQ4/y4tLVjoGbzWIv+Q6xwFzxfJbEDx86Y1dYHDM= -github.com/projectdiscovery/useragent v0.0.69 h1:6TkVMKjOBDqecUBBkXlTlBUfAUM8Bgn/58E3ZasndBs= -github.com/projectdiscovery/useragent v0.0.69/go.mod h1:Ss+Kf6XEjSQqhm7LmbEtOxKvGnnnR88Fx+0PG2PXuBY= -github.com/projectdiscovery/utils v0.3.0 h1:JNRMUPMx2gSzdk3bOQPtXSucoz+qC1JVkibIn7+kPTk= -github.com/projectdiscovery/utils v0.3.0/go.mod h1:k2XlmfaYO4k6T4vAyUa3Kn/0BxPTIlNiBFpM6nVCbz0= -github.com/projectdiscovery/wappalyzergo v0.1.22 h1:7QnpeZy/txsvVcQNlL1n4xIu8IXgMxEsda3w6k6bShY= -github.com/projectdiscovery/wappalyzergo v0.1.22/go.mod h1:wnvmbC10pQTOoCKnCTmWKP20rpEtqrMJZvzuTuleeyw= +github.com/projectdiscovery/tlsx v1.1.8 h1:Y+VkOp6JmUBb4tci1Fbz9U7ELEQ2irFhm+HS58tHruM= +github.com/projectdiscovery/tlsx v1.1.8/go.mod h1:6u/dbLuMsLzmux58AWnAB24qh2+Trk0auCK2I9B17Vo= +github.com/projectdiscovery/useragent v0.0.87 h1:XkFxYqwvFaA7Aw3/tnDmXyMuQpBXplTrm0GvrM+aaHY= +github.com/projectdiscovery/useragent v0.0.87/go.mod h1:lsKq3LoUWXnn0+Iir8hv+vS3zfMOrKn/aNb33Ik/9CI= +github.com/projectdiscovery/utils v0.4.8 h1:/Xd38fP8xc6kifZayjrhcYALenJrjO3sHO7lg+I8ZGk= +github.com/projectdiscovery/utils v0.4.8/go.mod h1:S314NzLcXVCbLbwYCoorAJYcnZEwv7Uhw2d3aF5fJ4s= +github.com/projectdiscovery/wappalyzergo v0.2.12 h1:A3oBpnEbTHOa3Q9m4w/5LLXsmCEiu0mJcwyjf3M9xnc= +github.com/projectdiscovery/wappalyzergo v0.2.12/go.mod h1:3vtvQCSYpU+Ilk0qy09WYT9BH0Stut5Qon7KJJ78GKw= github.com/refraction-networking/utls v1.6.7 h1:zVJ7sP1dJx/WtVuITug3qYUq034cDq9B2MR1K67ULZM= github.com/refraction-networking/utls v1.6.7/go.mod h1:BC3O4vQzye5hqpmDTWUqi4P5DDhzJfkV1tdqtawQIH0= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= @@ -330,8 +330,8 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/tidwall/assert v0.1.0 h1:aWcKyRBUAdLoVebxo95N7+YZVTFF/ASTr7BN4sLP6XI= @@ -423,8 +423,8 @@ golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= -golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= -golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc= +golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= @@ -453,8 +453,8 @@ golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= -golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= -golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= +golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0= +golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= @@ -496,8 +496,8 @@ golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= -golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= +golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -508,8 +508,8 @@ golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= -golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= -golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= +golang.org/x/term v0.28.0 h1:/Ts8HFuMR2E6IP/jlo7QVLZHggjKQbhu/7H0LJFr3Gg= +golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= diff --git a/runner/banner.go b/runner/banner.go index d546076c3..6343300d3 100644 --- a/runner/banner.go +++ b/runner/banner.go @@ -16,7 +16,7 @@ const banner = ` ` // Version is the current Version of httpx -const Version = `v1.6.9` +const Version = `v1.6.10` // showBanner is used to show the banner to the user func showBanner() { diff --git a/runner/options.go b/runner/options.go index c09944d18..ce558f7ec 100644 --- a/runner/options.go +++ b/runner/options.go @@ -165,18 +165,21 @@ func (s *ScanOptions) Clone() *ScanOptions { // Options contains configuration options for httpx. type Options struct { - CustomHeaders customheader.CustomHeaders - CustomPorts customport.CustomPorts - matchStatusCode []int - matchContentLength []int - filterStatusCode []int - filterContentLength []int - Output string - OutputAll bool - StoreResponseDir string - OmitBody bool - HTTPProxy string + CustomHeaders customheader.CustomHeaders + CustomPorts customport.CustomPorts + matchStatusCode []int + matchContentLength []int + filterStatusCode []int + filterContentLength []int + Output string + OutputAll bool + StoreResponseDir string + OmitBody bool + // Deprecated: use Proxy + HTTPProxy string + // Deprecated: use Proxy SocksProxy string + Proxy string InputFile string InputTargetHost goflags.StringSlice Methods string @@ -474,7 +477,7 @@ func ParseOptions() *Options { flagSet.StringVarP(&options.SniName, "sni-name", "sni", "", "custom TLS SNI name"), flagSet.BoolVar(&options.RandomAgent, "random-agent", true, "enable Random User-Agent to use"), flagSet.VarP(&options.CustomHeaders, "header", "H", "custom http headers to send with request"), - flagSet.StringVarP(&options.HTTPProxy, "proxy", "http-proxy", "", "http proxy to use (eg http://127.0.0.1:8080)"), + flagSet.StringVarP(&options.Proxy, "proxy", "http-proxy", "", "proxy (http|socks) to use (eg http://127.0.0.1:8080)"), flagSet.BoolVar(&options.Unsafe, "unsafe", false, "send raw requests skipping golang normalization"), flagSet.BoolVar(&options.Resume, "resume", false, "resume scan using resume.cfg"), flagSet.BoolVarP(&options.FollowRedirects, "follow-redirects", "fr", false, "follow http redirects"), diff --git a/runner/runner.go b/runner/runner.go index 4b8061507..de8e6495a 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -151,7 +151,12 @@ func New(options *Options) (*Runner, error) { httpxOptions.FollowHostRedirects = options.FollowHostRedirects httpxOptions.RespectHSTS = options.RespectHSTS httpxOptions.MaxRedirects = options.MaxRedirects - httpxOptions.HTTPProxy = options.HTTPProxy + if options.HTTPProxy != "" { + options.Proxy = options.HTTPProxy + } else if options.SocksProxy != "" { + options.Proxy = options.SocksProxy + } + httpxOptions.Proxy = options.Proxy httpxOptions.Unsafe = options.Unsafe httpxOptions.UnsafeURI = options.RequestURI httpxOptions.CdnCheck = options.OutputCDN @@ -293,6 +298,7 @@ func New(options *Options) (*Runner, error) { scanopts.NoHeadlessBody = options.NoHeadlessBody scanopts.UseInstalledChrome = options.UseInstalledChrome scanopts.ScreenshotTimeout = options.ScreenshotTimeout + scanopts.ScreenshotIdle = options.ScreenshotIdle if options.OutputExtractRegexs != nil { for _, regex := range options.OutputExtractRegexs { @@ -1051,94 +1057,92 @@ func (r *Runner) RunEnumeration() { gologger.Silent().Msgf("%s\n", resp.str) } - if resp.Err != nil { - continue - } - // store responses or chain in directory - URL, _ := urlutil.Parse(resp.URL) - domainFile := resp.Method + ":" + URL.EscapedString() - hash := hashes.Sha1([]byte(domainFile)) - domainResponseFile := fmt.Sprintf("%s.txt", hash) - screenshotResponseFile := fmt.Sprintf("%s.png", hash) - hostFilename := strings.ReplaceAll(URL.Host, ":", "_") - domainResponseBaseDir := filepath.Join(r.options.StoreResponseDir, "response") - domainScreenshotBaseDir := filepath.Join(r.options.StoreResponseDir, "screenshot") - responseBaseDir := filepath.Join(domainResponseBaseDir, hostFilename) - screenshotBaseDir := filepath.Join(domainScreenshotBaseDir, hostFilename) - - var responsePath, screenshotPath, screenshotPathRel string - // store response - if r.scanopts.StoreResponse || r.scanopts.StoreChain { - if r.scanopts.OmitBody { - resp.Raw = strings.Replace(resp.Raw, resp.ResponseBody, "", -1) - } + if resp.Err == nil { + URL, _ := urlutil.Parse(resp.URL) + domainFile := resp.Method + ":" + URL.EscapedString() + hash := hashes.Sha1([]byte(domainFile)) + domainResponseFile := fmt.Sprintf("%s.txt", hash) + screenshotResponseFile := fmt.Sprintf("%s.png", hash) + hostFilename := strings.ReplaceAll(URL.Host, ":", "_") + domainResponseBaseDir := filepath.Join(r.options.StoreResponseDir, "response") + domainScreenshotBaseDir := filepath.Join(r.options.StoreResponseDir, "screenshot") + responseBaseDir := filepath.Join(domainResponseBaseDir, hostFilename) + screenshotBaseDir := filepath.Join(domainScreenshotBaseDir, hostFilename) + + var responsePath, screenshotPath, screenshotPathRel string + // store response + if r.scanopts.StoreResponse || r.scanopts.StoreChain { + if r.scanopts.OmitBody { + resp.Raw = strings.Replace(resp.Raw, resp.ResponseBody, "", -1) + } - responsePath = fileutilz.AbsPathOrDefault(filepath.Join(responseBaseDir, domainResponseFile)) - // URL.EscapedString returns that can be used as filename - respRaw := resp.Raw - reqRaw := resp.RequestRaw - if len(respRaw) > r.scanopts.MaxResponseBodySizeToSave { - respRaw = respRaw[:r.scanopts.MaxResponseBodySizeToSave] - } - data := reqRaw - if r.options.StoreChain && resp.Response != nil && resp.Response.HasChain() { - data = append(data, append([]byte("\n"), []byte(resp.Response.GetChain())...)...) - } - data = append(data, respRaw...) - data = append(data, []byte("\n\n\n")...) - data = append(data, []byte(resp.URL)...) - _ = fileutil.CreateFolder(responseBaseDir) - writeErr := os.WriteFile(responsePath, data, 0644) - if writeErr != nil { - gologger.Error().Msgf("Could not write response at path '%s', to disk: %s", responsePath, writeErr) + responsePath = fileutilz.AbsPathOrDefault(filepath.Join(responseBaseDir, domainResponseFile)) + // URL.EscapedString returns that can be used as filename + respRaw := resp.Raw + reqRaw := resp.RequestRaw + if len(respRaw) > r.scanopts.MaxResponseBodySizeToSave { + respRaw = respRaw[:r.scanopts.MaxResponseBodySizeToSave] + } + data := reqRaw + if r.options.StoreChain && resp.Response != nil && resp.Response.HasChain() { + data = append(data, append([]byte("\n"), []byte(resp.Response.GetChain())...)...) + } + data = append(data, respRaw...) + data = append(data, []byte("\n\n\n")...) + data = append(data, []byte(resp.URL)...) + _ = fileutil.CreateFolder(responseBaseDir) + writeErr := os.WriteFile(responsePath, data, 0644) + if writeErr != nil { + gologger.Error().Msgf("Could not write response at path '%s', to disk: %s", responsePath, writeErr) + } + resp.StoredResponsePath = responsePath } - resp.StoredResponsePath = responsePath - } - if r.scanopts.Screenshot { - screenshotPath = fileutilz.AbsPathOrDefault(filepath.Join(screenshotBaseDir, screenshotResponseFile)) - screenshotPathRel = filepath.Join(hostFilename, screenshotResponseFile) - _ = fileutil.CreateFolder(screenshotBaseDir) - err := os.WriteFile(screenshotPath, resp.ScreenshotBytes, 0644) - if err != nil { - gologger.Error().Msgf("Could not write screenshot at path '%s', to disk: %s", screenshotPath, err) - } + if r.scanopts.Screenshot { + screenshotPath = fileutilz.AbsPathOrDefault(filepath.Join(screenshotBaseDir, screenshotResponseFile)) + screenshotPathRel = filepath.Join(hostFilename, screenshotResponseFile) + _ = fileutil.CreateFolder(screenshotBaseDir) + err := os.WriteFile(screenshotPath, resp.ScreenshotBytes, 0644) + if err != nil { + gologger.Error().Msgf("Could not write screenshot at path '%s', to disk: %s", screenshotPath, err) + } - resp.ScreenshotPath = screenshotPath - resp.ScreenshotPathRel = screenshotPathRel - if r.scanopts.NoScreenshotBytes { - resp.ScreenshotBytes = []byte{} + resp.ScreenshotPath = screenshotPath + resp.ScreenshotPathRel = screenshotPathRel + if r.scanopts.NoScreenshotBytes { + resp.ScreenshotBytes = []byte{} + } } - } - if indexFile != nil { - indexData := fmt.Sprintf("%s %s (%d %s)\n", resp.StoredResponsePath, resp.URL, resp.StatusCode, http.StatusText(resp.StatusCode)) - _, _ = indexFile.WriteString(indexData) - } - if indexScreenshotFile != nil && resp.ScreenshotPathRel != "" { - indexData := fmt.Sprintf("%s %s (%d %s)\n", resp.ScreenshotPathRel, resp.URL, resp.StatusCode, http.StatusText(resp.StatusCode)) - _, _ = indexScreenshotFile.WriteString(indexData) - } + if indexFile != nil { + indexData := fmt.Sprintf("%s %s (%d %s)\n", resp.StoredResponsePath, resp.URL, resp.StatusCode, http.StatusText(resp.StatusCode)) + _, _ = indexFile.WriteString(indexData) + } + if indexScreenshotFile != nil && resp.ScreenshotPathRel != "" { + indexData := fmt.Sprintf("%s %s (%d %s)\n", resp.ScreenshotPathRel, resp.URL, resp.StatusCode, http.StatusText(resp.StatusCode)) + _, _ = indexScreenshotFile.WriteString(indexData) + } - if r.scanopts.StoreVisionReconClusters { - foundCluster := false - pHash, _ := resp.KnowledgeBase["pHash"].(uint64) - for i, cluster := range r.pHashClusters { - distance, _ := goimagehash.NewImageHash(pHash, goimagehash.PHash).Distance(goimagehash.NewImageHash(cluster.BasePHash, goimagehash.PHash)) - if distance <= hammingDistanceThreshold { - r.pHashClusters[i].Hashes = append(r.pHashClusters[i].Hashes, pHashUrl{PHash: pHash, Url: resp.URL}) - foundCluster = true - break + if r.scanopts.StoreVisionReconClusters { + foundCluster := false + pHash, _ := resp.KnowledgeBase["pHash"].(uint64) + for i, cluster := range r.pHashClusters { + distance, _ := goimagehash.NewImageHash(pHash, goimagehash.PHash).Distance(goimagehash.NewImageHash(cluster.BasePHash, goimagehash.PHash)) + if distance <= hammingDistanceThreshold { + r.pHashClusters[i].Hashes = append(r.pHashClusters[i].Hashes, pHashUrl{PHash: pHash, Url: resp.URL}) + foundCluster = true + break + } } - } - if !foundCluster { - newCluster := pHashCluster{ - BasePHash: pHash, - Hashes: []pHashUrl{{PHash: pHash, Url: resp.URL}}, + if !foundCluster { + newCluster := pHashCluster{ + BasePHash: pHash, + Hashes: []pHashUrl{{PHash: pHash, Url: resp.URL}}, + } + r.pHashClusters = append(r.pHashClusters, newCluster) } - r.pHashClusters = append(r.pHashClusters, newCluster) } } diff --git a/runner/types.go b/runner/types.go index 7bdb3a9c6..45d9d3406 100644 --- a/runner/types.go +++ b/runner/types.go @@ -53,20 +53,20 @@ type Result struct { Scheme string `json:"scheme,omitempty" csv:"scheme" mapstructure:"scheme"` Error string `json:"error,omitempty" csv:"error" mapstructure:"error"` WebServer string `json:"webserver,omitempty" csv:"webserver" mapstructure:"webserver"` - ResponseBody string `json:"body,omitempty" csv:"body" mapstructure:"body"` + ResponseBody string `json:"body,omitempty" csv:"-" mapstructure:"body"` BodyPreview string `json:"body_preview,omitempty" csv:"body_preview" mapstructure:"body_preview"` ContentType string `json:"content_type,omitempty" csv:"content_type" mapstructure:"content_type"` Method string `json:"method,omitempty" csv:"method" mapstructure:"method"` Host string `json:"host,omitempty" csv:"host" mapstructure:"host"` Path string `json:"path,omitempty" csv:"path" mapstructure:"path"` FavIconMMH3 string `json:"favicon,omitempty" csv:"favicon" mapstructure:"favicon"` - FavIconMD5 string `json:"favicon_md5,omitempty" csv:"favicon" mapstructure:"favicon_md5"` + FavIconMD5 string `json:"favicon_md5,omitempty" csv:"favicon_md5" mapstructure:"favicon_md5"` FaviconPath string `json:"favicon_path,omitempty" csv:"favicon_path" mapstructure:"favicon_path"` FaviconURL string `json:"favicon_url,omitempty" csv:"favicon_url" mapstructure:"favicon_url"` FinalURL string `json:"final_url,omitempty" csv:"final_url" mapstructure:"final_url"` - ResponseHeaders map[string]interface{} `json:"header,omitempty" csv:"header" mapstructure:"header"` - RawHeaders string `json:"raw_header,omitempty" csv:"raw_header" mapstructure:"raw_header"` - Request string `json:"request,omitempty" csv:"request" mapstructure:"request"` + ResponseHeaders map[string]interface{} `json:"header,omitempty" csv:"-" mapstructure:"header"` + RawHeaders string `json:"raw_header,omitempty" csv:"-" mapstructure:"raw_header"` + Request string `json:"request,omitempty" csv:"-" mapstructure:"request"` ResponseTime string `json:"time,omitempty" csv:"time" mapstructure:"time"` JarmHash string `json:"jarm_hash,omitempty" csv:"jarm_hash" mapstructure:"jarm_hash"` ChainStatusCodes []int `json:"chain_status_codes,omitempty" csv:"chain_status_codes" mapstructure:"chain_status_codes"` diff --git a/static/html-summary.html b/static/html-summary.html index dee56c482..9961f92c9 100644 --- a/static/html-summary.html +++ b/static/html-summary.html @@ -120,7 +120,7 @@ - Screenshot + Screenshot