Improve proxy client

ahobsonsayers · Sep 27, 2024 · 2e2fda6 · 2e2fda6
1 parent 6b47a79
commit 2e2fda6
Show file tree

Hide file tree

Showing 3 changed files with 87 additions and 39 deletions.
diff --git a/test/testutils/env.go b/test/testutils/env.go
@@ -6,18 +6,22 @@ import (
 	"testing"
 )
 
-func SkipIfCI(t *testing.T, args ...any) {
+func IsCI() bool {
 	env, ok := os.LookupEnv("CI")
 	if !ok {
-		return
+		return false
 	}
 
 	isCI, err := strconv.ParseBool(env)
 	if err != nil {
-		return
+		return false
 	}
 
-	if isCI {
+	return isCI
+}
+
+func SkipIfCI(t *testing.T, args ...any) {
+	if IsCI() {
 		t.Skip(args...)
 	}
 }
diff --git a/test/testutils/proxy.go b/test/testutils/proxy.go
@@ -10,37 +10,71 @@ import (
 	"math/rand"
 	"net/http"
 	"net/url"
-	"strings"
+	"regexp"
 	"sync"
 	"time"
+
+	"github.com/samber/lo"
 )
 
 const (
 	RoosterKidProxyListURL = "https://raw.githubusercontent.com/roosterkid/openproxylist/main/HTTPS_RAW.txt"
 	ProxlifyProxyListURL   = "https://raw.githubusercontent.com/proxifly/free-proxy-list/main/proxies/protocols/http/data.txt" // nolint: revive
 	TheSpeedXProxyListURL  = "https://raw.githubusercontent.com/TheSpeedX/PROXY-List/refs/heads/master/http.txt"
+	HideIPMeProxyListURL   = "https://raw.githubusercontent.com/zloi-user/hideip.me/refs/heads/main/http.txt"
+
+	DefaultProxyTestTimeout = 5 * time.Second
+	DefaultProxyTestURL     = "https://example.com"
+)
+
+var (
+	proxyIPRegex = regexp.MustCompile(`\b(?:\d{1,3}\.){3}\d{1,3}(?::\d+)?\b`)
+
+	ProxyListUrls = []string{
+		RoosterKidProxyListURL,
+		ProxlifyProxyListURL,
+		TheSpeedXProxyListURL,
+		HideIPMeProxyListURL,
+	}
 )
 
-func NewProxyClient(proxyListUrls ...string) (*http.Client, error) {
-	if len(proxyListUrls) == 0 {
+type ProxyClientConfig struct {
+	ProxyListUrls []string
+	TestURL       string
+	TestTimeout   time.Duration
+}
+
+func (c *ProxyClientConfig) applyDefaults() {
+	if c.TestURL == "" {
+		c.TestURL = DefaultProxyTestURL
+	}
+	if c.TestTimeout == time.Duration(0) {
+		c.TestTimeout = DefaultProxyTestTimeout
+	}
+}
+
+func NewProxyClient(config ProxyClientConfig) (*http.Client, error) {
+	config.applyDefaults()
+
+	if len(config.ProxyListUrls) == 0 {
 		return nil, fmt.Errorf("at least one proxy list url must be passed")
 	}
 
-	proxyTransport, err := newProxyTransport(proxyListUrls)
+	proxyTransport, err := newProxyTransport(config)
 	if err != nil {
 		return nil, err
 	}
 
 	return &http.Client{Transport: proxyTransport}, nil
 }
 
-func newProxyTransport(proxyListUrls []string) (http.RoundTripper, error) {
-	proxyLists, err := downloadProxyLists(proxyListUrls)
+func newProxyTransport(config ProxyClientConfig) (http.RoundTripper, error) {
+	proxyLists, err := downloadProxyLists(config.ProxyListUrls)
 	if err != nil {
 		return nil, fmt.Errorf("error downloading proxy list: %w", err)
 	}
 
-	proxyLists = getWorkingProxies(proxyLists, 2*time.Second)
+	proxyLists = getWorkingProxies(proxyLists, config)
 	if len(proxyLists) == 0 {
 		return nil, errors.New("none of the proxies in the proxy list are working")
 	}
@@ -54,7 +88,7 @@ func newProxyTransport(proxyListUrls []string) (http.RoundTripper, error) {
 }
 
 type proxyTransport struct {
-	proxyList []*url.URL
+	proxyList []url.URL
 	index     int
 	mu        sync.Mutex
 }
@@ -69,16 +103,16 @@ func (t *proxyTransport) RoundTrip(request *http.Request) (*http.Response, error
 
 	// Create a new transport and us it for the request
 	transport := &http.Transport{
-		Proxy: http.ProxyURL(proxyURL),
+		Proxy: http.ProxyURL(&proxyURL),
 		TLSClientConfig: &tls.Config{
 			MinVersion: tls.VersionTLS12,
 		},
 	}
 	return transport.RoundTrip(request)
 }
 
-func downloadProxyLists(proxyListUrls []string) ([]*url.URL, error) {
-	proxyUrls := []*url.URL{}
+func downloadProxyLists(proxyListUrls []string) ([]url.URL, error) {
+	proxyUrls := []url.URL{}
 	for _, proxyListUrl := range proxyListUrls {
 		proxyListProxyUrls, err := downloadProxyList(proxyListUrl)
 		if err != nil {
@@ -87,10 +121,10 @@ func downloadProxyLists(proxyListUrls []string) ([]*url.URL, error) {
 		proxyUrls = append(proxyUrls, proxyListProxyUrls...)
 	}
 
-	return proxyUrls, nil
+	return lo.Uniq(proxyUrls), nil
 }
 
-func downloadProxyList(proxyListUrl string) ([]*url.URL, error) {
+func downloadProxyList(proxyListUrl string) ([]url.URL, error) {
 	resp, err := http.Get(proxyListUrl)
 	if err != nil {
 		return nil, err
@@ -99,41 +133,42 @@ func downloadProxyList(proxyListUrl string) ([]*url.URL, error) {
 	return parseProxyList(resp.Body)
 }
 
-func parseProxyList(reader io.Reader) ([]*url.URL, error) {
-	var proxyUrls []*url.URL
+func parseProxyList(reader io.Reader) ([]url.URL, error) {
+	var proxyUrls []url.URL
 	scanner := bufio.NewScanner(reader)
 	for scanner.Scan() {
-		proxyAddress := scanner.Text()
-		if !strings.HasPrefix(proxyAddress, "http://") {
-			proxyAddress = "http://" + proxyAddress
+		line := scanner.Text()
+		ipPort := proxyIPRegex.FindString(line)
+		if ipPort == "" {
+			continue
 		}
 
+		proxyAddress := "http://" + ipPort
 		parsedURL, err := url.Parse(proxyAddress)
 		if err != nil {
 			fmt.Printf("Skipping invalid proxy URL: %s\n", proxyAddress)
 			continue
 		}
 
-		proxyUrls = append(proxyUrls, parsedURL)
+		proxyUrls = append(proxyUrls, *parsedURL)
 	}
 
-	err := scanner.Err()
-	if err != nil {
+	if err := scanner.Err(); err != nil {
 		return nil, err
 	}
 
 	return proxyUrls, nil
 }
 
-func getWorkingProxies(proxyUrls []*url.URL, timeout time.Duration) []*url.URL {
-	resultsChan := make(chan *url.URL, len(proxyUrls))
+func getWorkingProxies(proxyUrls []url.URL, config ProxyClientConfig) []url.URL {
+	resultsChan := make(chan url.URL, len(proxyUrls))
 
 	var wg sync.WaitGroup
 	for _, proxyUrl := range proxyUrls {
 		wg.Add(1)
-		go func(proxyUrl *url.URL) {
+		go func(proxyUrl url.URL) {
 			defer wg.Done()
-			ok := checkProxy(proxyUrl, timeout)
+			ok := checkProxy(&proxyUrl, config)
 			if ok {
 				resultsChan <- proxyUrl
 			}
@@ -143,16 +178,16 @@ func getWorkingProxies(proxyUrls []*url.URL, timeout time.Duration) []*url.URL {
 	wg.Wait()
 	close(resultsChan)
 
-	results := make([]*url.URL, 0, len(proxyUrls))
+	results := make([]url.URL, 0, len(proxyUrls))
 	for result := range resultsChan {
 		results = append(results, result)
 	}
 
 	return results
 }
 
-func checkProxy(proxyUrl *url.URL, timeout time.Duration) bool {
-	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+func checkProxy(proxyUrl *url.URL, config ProxyClientConfig) bool {
+	ctx, cancel := context.WithTimeout(context.Background(), config.TestTimeout)
 	defer cancel()
 
 	client := &http.Client{
@@ -163,7 +198,7 @@ func checkProxy(proxyUrl *url.URL, timeout time.Duration) bool {
 
 	request, err := http.NewRequestWithContext(
 		ctx, http.MethodGet,
-		"https://example.com",
+		config.TestURL,
 		http.NoBody,
 	)
 	if err != nil {

diff --git a/twickets/twickets_test.go b/twickets/twickets_test.go
@@ -2,8 +2,10 @@ package twickets_test
 
 import (
 	"context"
+	"net/http"
 	"os"
 	"testing"
+	"time"
 
 	"github.com/ahobsonsayers/twitchets/test/testutils"
 	"github.com/ahobsonsayers/twitchets/twickets"
@@ -17,12 +19,19 @@ func TestGetLatestTickets(t *testing.T) {
 	twicketsAPIKey := os.Getenv("TWICKETS_API_KEY")
 	require.NotEmpty(t, twicketsAPIKey, "TWICKETS_API_KEY is not set")
 
-	httpClient, err := testutils.NewProxyClient(
-		testutils.RoosterKidProxyListURL,
-		testutils.ProxlifyProxyListURL,
-		testutils.TheSpeedXProxyListURL,
-	)
-	require.NoError(t, err)
+	var httpClient *http.Client
+	if !testutils.IsCI() {
+		httpClient = http.DefaultClient
+	} else {
+		var err error
+		httpClient, err = testutils.NewProxyClient(
+			testutils.ProxyClientConfig{
+				ProxyListUrls: testutils.ProxyListUrls,
+				TestTimeout:   5 * time.Second,
+			},
+		)
+		require.NoError(t, err)
+	}
 
 	twicketsClient := twickets.NewClient(httpClient)
 	tickets, err := twicketsClient.FetchTickets(