From 35af3925accd6ec0a152305959ba155e49bbb8cc Mon Sep 17 00:00:00 2001 From: Ahmed Elfakharany Date: Sat, 22 Oct 2022 21:56:41 +0200 Subject: [PATCH] Adds the rotating proxies project --- projects/httpbin/httpbin/proxies.txt | 300 ++++++++++++++++++++ projects/httpbin/httpbin/settings.py | 5 + projects/httpbin/httpbin/spiders/proxies.py | 13 + 3 files changed, 318 insertions(+) create mode 100644 projects/httpbin/httpbin/proxies.txt create mode 100644 projects/httpbin/httpbin/spiders/proxies.py diff --git a/projects/httpbin/httpbin/proxies.txt b/projects/httpbin/httpbin/proxies.txt new file mode 100644 index 0000000..c5974bb --- /dev/null +++ b/projects/httpbin/httpbin/proxies.txt @@ -0,0 +1,300 @@ +80.48.119.28:8080 +198.59.191.234:8080 +139.59.1.14:8080 +198.49.68.80 +169.57.1.85:8123 +146.196.48.2 +45.79.110.81 +219.78.228.211 +83.229.73.175 +46.18.48.18:8080 +165.154.226.12 +139.99.237.62 +83.229.72.174 +165.154.243.53 +221.132.18.26:8090 +20.111.54.16 +42.3.182.149 +151.181.91.10 +181.214.91.244:3128 +165.154.225.65 +49.207.36.81 +81.200.123.74 +110.164.3.7:8888 +112.120.41.171 +216.137.184.253 +51.250.80.131 +20.210.113.32:8123 +20.24.43.214:8123 +174.138.16.96:8888 +216.240.140.81 +165.232.149.87:8888 +45.142.106.3 +8.209.246.6 +42.3.27.14 +172.104.241.29:8081 +85.159.214.61:1080 +193.122.71.184:3128 +89.107.197.165:3128 +185.61.152.137:8080 +149.129.239.170:8080 +110.238.74.184:8080 +173.230.153.163:8080 +172.104.111.212 +204.185.204.64:8080 +47.254.195.196:8080 +47.254.237.222:8080 +41.188.149.79 +66.175.223.147:4153 +45.33.12.251:8080 +192.53.163.144:3128 +43.154.233.149:8080 +20.54.56.26:8080 +173.255.209.155:1080 +38.94.109.12 +149.129.232.50:8080 +47.88.11.3:2080 +54.75.75.45:8888 +198.11.175.180:8080 +139.59.88.145:8888 +118.26.110.48:8080 +172.105.172.220:3128 +191.103.219.225:4861 +113.23.237.130:8888 +5.45.103.133:3128 +103.215.207.74:83 +94.181.48.171:1256 +41.33.3.35:1981 +68.183.191.179:4429 +51.15.242.202:8888 +172.105.25.190:8020 +47.74.152.29:8888 +201.229.250.22:8080 +221.132.28.18:8090 +43.154.216.109:80 +210.14.104.230:8080 +104.248.194.17:80 +193.141.65.48:808 +78.46.27.131:8080 +118.172.43.60:8080 +174.81.78.64:4867 +75.50.42.115:3129 +187.217.54.84:80 +85.70.210.30:80 +143.198.40.24:8888 +185.86.162.107:3111 +179.1.129.54:999 +103.166.10.15:8181 +223.204.9.111:8080 +45.79.90.143:4455 +66.29.154.103:3128 +165.22.252.212:3357 +103.78.36.163:4697 +103.143.195.255:8080 +177.192.115.14:8080 +101.109.4.120:8080 +103.181.245.146:8080 +191.97.9.86:999 +110.238.113.119:8080 +170.187.141.197:8080 +159.138.169.48:8080 +8.219.74.58:8080 +110.238.111.229:8080 +149.129.187.190:3128 +45.79.212.97:2080 +198.11.175.192:8080 +38.94.109.7:80 +84.54.185.203:8080 +157.100.12.138:999 +121.101.133.70:8080 +202.152.12.202:8080 +177.75.97.192:3128 +46.38.43.20:8082 +45.79.111.38:9994 +117.54.114.32 +172.104.252.86 +34.223.105.122 +85.183.140.35:8080 +185.51.10.19 +3.1.248.232 +47.91.44.217:8000 +117.54.114.35 +162.0.226.218 +45.79.27.210:4455 +157.100.26.69 +135.181.199.92:9300 +41.32.12.190 +186.68.101.146:6969 +103.73.102.74:3127 +196.27.106.112:8080 +162.155.10.150:5544 +80.66.81.40:8080 +128.199.202.122:8080 +51.159.207.156:3128 +110.78.81.107:8080 +190.107.237.25:999 +116.203.20.249:8888 +45.79.208.64:4455 +47.254.47.61:8080 +8.213.128.6:8080 +212.71.255.43:3861 +110.238.109.146:8080 +159.138.255.141:8080 +159.138.252.45:8080 +176.236.146.174:8080 +47.253.214.60"5711 +177.12.238.1"3128 +8.209.249.96"8080 +58.27.59.249"80 +45.169.162.1"3128 +177.12.238.100"3128 +103.167.134.31 +155.138.197.162 +193.3.20.13 +172.105.184.208:8001 +5.189.184.6 +194.233.84.239 +194.195.216.153:4145 +47.74.64.65 8080 +49.0.253.51 2080 +8.209.240.66:2080 +47.91.45.198:8080 +47.74.71.208:8080 +5.253.247.30:2019 +201.77.108.48:999 +116.58.166.194:8080 +44.226.199.133:80 +216.176.187.99:3000 +157.230.97.17:8888 +177.135.247.119:8080 +107.152.197.63:8085 +45.88.163.184:5964 +183.89.76.207:8080 +194.195.240.60:8080 +149.129.213.200:8080 +172.105.190.51:8017 +172.105.107.25:999 +159.203.31.27:8118 +212.98.152.65:9094 +190.110.99.104:999 +96.126.124.197:81 +198.27.74.6:9300 +47.252.1.180:3128 +161.35.140.31:8888 +190.2.215.210:999 +181.209.111.147:999 +129.213.69.94:80 +159.138.158.36:8888 +149.129.184.250:8080 +8.209.68.1:8080 +8.209.64.208:8080 +47.245.34.161:8080 +75.84.210.2 +103.7.27.186:8080 +204.48.31.203:80 +139.59.115.146:3128 +159.65.63.209:8888 +5.2.75.58:8118 +82.165.21.59:80 +200.105.215.18:3363 +218.158.198.89:8001 +190.113.41.163:999 +36.93.5.25:9812 +139.255.109.27:8080 +206.62.161.175:999 +77.236.237.177:8080 +181.65.169.37:999 +45.167.95.184:8085 +182.253.112.194:8080 +185.189.186.19:8080 +103.156.15.68:8080 +45.56.83.46:8012 +149.129.131.46:8080 +43.255.113.232:8082 +178.79.191.47:5441 +47.91.56.120:8080 +94.102.196.89:1500 +181.209.105.156:8080 +190.14.249.217:999 +204.199.129.38:999 +93.145.17.218:8080 +113.252.44.63:80 +47.241.165.133:443 +121.1.41.162:111 +147.182.207.176:8080 +209.97.152.208:8888 +103.255.241.190:5544 +117.102.87.66:8080 +109.68.148.60:3128 +47.91.15.175:80 +200.25.254.193:5424 +198.12.85.211:80 +35.86.232.240:80 +37.152.172.198:8080 +52.172.249.189:8080 +45.134.187.72:7109 +88.218.66.100:8085 +104.227.173.134:8197 +138.122.194.242:7318 +193.23.253.218:7790 +104.168.126.68:3128 +154.202.117.51:3128 +154.202.116.102:3128 +91.188.247.208:8085 +107.172.37.190:3128 +154.202.125.232:3128 +154.202.118.99:3128 +154.38.18.25:8800 +23.254.101.108:3128 +185.223.40.102:8800 +193.9.33.11:6784 +45.57.225.167:9249 +193.39.91.150:8800 +194.146.134.16: 5218 +185.228.216.3: 8800 +107.172.34.25: 3128 +38.15.153.32:3128 +23.94.227.141:3128 +154.38.16.252:8800 +193.9.33.30:6803 +192.186.151.111:8612 +157.52.174.6:6215 +154.202.114.146:3128 +185.77.221.148:8085 +154.38.16.214:8800 +134.73.188.252:5342 +185.101.169.24:6568 +85.208.86.181:8085 +154.202.124.11:3128 +161.123.208.91:6335 +193.233.83.122:8085 +85.208.86.159:8085 +66.151.50.35:6838 +45.72.108.68:6122 +107.172.37.218:3128 +185.228.217.172:8800 +185.223.43.31:8800 +154.202.127.122:3128 +185.228.216.199:8800 +154.202.114.182:3128 +154.202.114.104:3128 +104.144.72.63:6095 +154.202.118.103:3128 +144.168.255.85:3128 +144.168.255.159:3128 +144.168.253.20:3128 +94.131.81.135:6275 +38.15.155.106:3128 +107.172.34.8:3128 +107.172.38.182:3128 +107.172.38.188:3128 +154.202.115.101:3128 +213.166.79.30:8085 +194.31.162.146:7662 +185.223.43.82:8800 +107.173.107.39:3128 +23.254.101.116:3128 +193.233.230.53:8085 +185.228.217.105:8800 +138.128.78.134:7220 +107.173.107.203:3128 \ No newline at end of file diff --git a/projects/httpbin/httpbin/settings.py b/projects/httpbin/httpbin/settings.py index 9825cb4..17eb350 100644 --- a/projects/httpbin/httpbin/settings.py +++ b/projects/httpbin/httpbin/settings.py @@ -89,4 +89,9 @@ DOWNLOADER_MIDDLEWARES = { 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': None, 'scrapy_user_agents.middlewares.RandomUserAgentMiddleware': 400, + 'rotating_free_proxies.middlewares.RotatingProxyMiddleware': 610, + 'rotating_free_proxies.middlewares.BanDetectionMiddleware': 620, } +# Path that this library uses to store list of proxies +ROTATING_PROXY_LIST_PATH = 'proxies.txt' +NUMBER_OF_PROXIES_TO_FETCH = 5 # Controls how many proxies to use diff --git a/projects/httpbin/httpbin/spiders/proxies.py b/projects/httpbin/httpbin/spiders/proxies.py new file mode 100644 index 0000000..2ba023b --- /dev/null +++ b/projects/httpbin/httpbin/spiders/proxies.py @@ -0,0 +1,13 @@ +import scrapy +import json + + +class UseragentSpider(scrapy.Spider): + name = 'proxies' + allowed_domains = ['httpbin.org'] + start_urls = ['https://httpbin.org/ip'] + + def parse(self, response): + payload = json.loads(response.body) + yield (payload) + yield scrapy.Request(self.start_urls[0], dont_filter=True)