Adds the rotating proxies project

projects/httpbin/httpbin/proxies.txt

@@ -0,0 +1,300 @@
+80.48.119.28:8080
+198.59.191.234:8080
+139.59.1.14:8080	
+198.49.68.80	
+169.57.1.85:8123	
+146.196.48.2	
+45.79.110.81	
+219.78.228.211
+83.229.73.175	
+46.18.48.18:8080	
+165.154.226.12	
+139.99.237.62	
+83.229.72.174	
+165.154.243.53	
+221.132.18.26:8090
+20.111.54.16		
+42.3.182.149		
+151.181.91.10		
+181.214.91.244:3128
+165.154.225.65		
+49.207.36.81		
+81.200.123.74		
+110.164.3.7:8888
+112.120.41.171
+216.137.184.253
+51.250.80.131
+20.210.113.32:8123
+20.24.43.214:8123
+174.138.16.96:8888
+216.240.140.81
+165.232.149.87:8888
+45.142.106.3
+8.209.246.6
+42.3.27.14
+172.104.241.29:8081
+85.159.214.61:1080
+193.122.71.184:3128
+89.107.197.165:3128
+185.61.152.137:8080
+149.129.239.170:8080
+110.238.74.184:8080
+173.230.153.163:8080
+172.104.111.212
+204.185.204.64:8080
+47.254.195.196:8080
+47.254.237.222:8080
+41.188.149.79	
+66.175.223.147:4153
+45.33.12.251:8080
+192.53.163.144:3128
+43.154.233.149:8080
+20.54.56.26:8080	
+173.255.209.155:1080
+38.94.109.12	
+149.129.232.50:8080
+47.88.11.3:2080	
+54.75.75.45:8888	
+198.11.175.180:8080
+139.59.88.145:8888
+118.26.110.48:8080
+172.105.172.220:3128
+191.103.219.225:4861
+113.23.237.130:8888
+5.45.103.133:3128
+103.215.207.74:83	
+94.181.48.171:1256
+41.33.3.35:1981	
+68.183.191.179:4429
+51.15.242.202:8888
+172.105.25.190:8020
+47.74.152.29:8888
+201.229.250.22:8080
+221.132.28.18:8090
+43.154.216.109:80	
+210.14.104.230:8080
+104.248.194.17:80	
+193.141.65.48:808	
+78.46.27.131:8080
+118.172.43.60:8080
+174.81.78.64:4867
+75.50.42.115:3129
+187.217.54.84:80	
+85.70.210.30:80	
+143.198.40.24:8888
+185.86.162.107:3111
+179.1.129.54:999	
+103.166.10.15:8181
+223.204.9.111:8080
+45.79.90.143:4455
+66.29.154.103:3128
+165.22.252.212:3357
+103.78.36.163:4697
+103.143.195.255:8080
+177.192.115.14:8080
+101.109.4.120:8080
+103.181.245.146:8080
+191.97.9.86:999
+110.238.113.119:8080
+170.187.141.197:8080
+159.138.169.48:8080
+8.219.74.58:8080	
+110.238.111.229:8080
+149.129.187.190:3128
+45.79.212.97:2080
+198.11.175.192:8080
+38.94.109.7:80
+84.54.185.203:8080
+157.100.12.138:999	
+121.101.133.70:8080
+202.152.12.202:8080
+177.75.97.192:3128
+46.38.43.20:8082	
+45.79.111.38:9994
+117.54.114.32	
+172.104.252.86	
+34.223.105.122	
+85.183.140.35:8080
+185.51.10.19
+3.1.248.232	
+47.91.44.217:8000
+117.54.114.35
+162.0.226.218
+45.79.27.210:4455
+157.100.26.69
+135.181.199.92:9300
+41.32.12.190
+186.68.101.146:6969
+103.73.102.74:3127
+196.27.106.112:8080
+162.155.10.150:5544
+80.66.81.40:8080	
+128.199.202.122:8080
+51.159.207.156:3128
+110.78.81.107:8080
+190.107.237.25:999	
+116.203.20.249:8888
+45.79.208.64:4455
+47.254.47.61:8080
+8.213.128.6:8080	
+212.71.255.43:3861
+110.238.109.146:8080
+159.138.255.141:8080
+159.138.252.45:8080
+176.236.146.174:8080
+47.253.214.60"5711
+177.12.238.1"3128
+8.209.249.96"8080
+58.27.59.249"80	
+45.169.162.1"3128
+177.12.238.100"3128
+103.167.134.31	
+155.138.197.162	
+193.3.20.13	
+172.105.184.208:8001
+5.189.184.6
+194.233.84.239	
+194.195.216.153:4145
+47.74.64.65	8080	
+49.0.253.51	2080	
+8.209.240.66:2080
+47.91.45.198:8080
+47.74.71.208:8080
+5.253.247.30:2019
+201.77.108.48:999	
+116.58.166.194:8080
+44.226.199.133:80	
+216.176.187.99:3000
+157.230.97.17:8888
+177.135.247.119:8080
+107.152.197.63:8085
+45.88.163.184:5964
+183.89.76.207:8080
+194.195.240.60:8080
+149.129.213.200:8080
+172.105.190.51:8017
+172.105.107.25:999	
+159.203.31.27:8118
+212.98.152.65:9094
+190.110.99.104:999	
+96.126.124.197:81	
+198.27.74.6:9300	
+47.252.1.180:3128
+161.35.140.31:8888
+190.2.215.210:999	
+181.209.111.147:999	
+129.213.69.94:80	
+159.138.158.36:8888
+149.129.184.250:8080
+8.209.68.1:8080	
+8.209.64.208:8080
+47.245.34.161:8080
+75.84.210.2
+103.7.27.186:8080
+204.48.31.203:80	
+139.59.115.146:3128
+159.65.63.209:8888
+5.2.75.58:8118	
+82.165.21.59:80	
+200.105.215.18:3363
+218.158.198.89:8001
+190.113.41.163:999	
+36.93.5.25:9812	
+139.255.109.27:8080
+206.62.161.175:999	
+77.236.237.177:8080
+181.65.169.37:999	
+45.167.95.184:8085
+182.253.112.194:8080
+185.189.186.19:8080
+103.156.15.68:8080
+45.56.83.46:8012	
+149.129.131.46:8080
+43.255.113.232:8082
+178.79.191.47:5441
+47.91.56.120:8080
+94.102.196.89:1500
+181.209.105.156:8080
+190.14.249.217:999	
+204.199.129.38:999	
+93.145.17.218:8080
+113.252.44.63:80	
+47.241.165.133:443	
+121.1.41.162:111	
+147.182.207.176:8080
+209.97.152.208:8888
+103.255.241.190:5544
+117.102.87.66:8080
+109.68.148.60:3128
+47.91.15.175:80	
+200.25.254.193:5424
+198.12.85.211:80	
+35.86.232.240:80	
+37.152.172.198:8080
+52.172.249.189:8080
+45.134.187.72:7109
+88.218.66.100:8085
+104.227.173.134:8197
+138.122.194.242:7318
+193.23.253.218:7790
+104.168.126.68:3128
+154.202.117.51:3128
+154.202.116.102:3128
+91.188.247.208:8085
+107.172.37.190:3128
+154.202.125.232:3128
+154.202.118.99:3128
+154.38.18.25:8800
+23.254.101.108:3128
+185.223.40.102:8800
+193.9.33.11:6784	
+45.57.225.167:9249
+193.39.91.150:8800
+194.146.134.16:	5218
+185.228.216.3:	8800
+107.172.34.25:	3128
+38.15.153.32:3128
+23.94.227.141:3128
+154.38.16.252:8800
+193.9.33.30:6803	
+192.186.151.111:8612
+157.52.174.6:6215
+154.202.114.146:3128
+185.77.221.148:8085
+154.38.16.214:8800
+134.73.188.252:5342
+185.101.169.24:6568
+85.208.86.181:8085
+154.202.124.11:3128
+161.123.208.91:6335
+193.233.83.122:8085
+85.208.86.159:8085
+66.151.50.35:6838
+45.72.108.68:6122
+107.172.37.218:3128
+185.228.217.172:8800
+185.223.43.31:8800
+154.202.127.122:3128
+185.228.216.199:8800
+154.202.114.182:3128
+154.202.114.104:3128
+104.144.72.63:6095
+154.202.118.103:3128
+144.168.255.85:3128
+144.168.255.159:3128
+144.168.253.20:3128
+94.131.81.135:6275
+38.15.155.106:3128
+107.172.34.8:3128
+107.172.38.182:3128
+107.172.38.188:3128
+154.202.115.101:3128
+213.166.79.30:8085
+194.31.162.146:7662
+185.223.43.82:8800
+107.173.107.39:3128
+23.254.101.116:3128
+193.233.230.53:8085
+185.228.217.105:8800
+138.128.78.134:7220
+107.173.107.203:3128

projects/httpbin/httpbin/settings.py

@@ -89,4 +89,9 @@
 DOWNLOADER_MIDDLEWARES = {
     'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': None,
     'scrapy_user_agents.middlewares.RandomUserAgentMiddleware': 400,
+    'rotating_free_proxies.middlewares.RotatingProxyMiddleware': 610,
+    'rotating_free_proxies.middlewares.BanDetectionMiddleware': 620,
 }
+# Path that this library uses to store list of proxies
+ROTATING_PROXY_LIST_PATH = 'proxies.txt'
+NUMBER_OF_PROXIES_TO_FETCH = 5  # Controls how many proxies to use

projects/httpbin/httpbin/spiders/proxies.py

@@ -0,0 +1,13 @@
+import scrapy
+import json
+
+
+class UseragentSpider(scrapy.Spider):
+    name = 'proxies'
+    allowed_domains = ['httpbin.org']
+    start_urls = ['https://httpbin.org/ip']
+
+    def parse(self, response):
+        payload = json.loads(response.body)
+        yield (payload)
+        yield scrapy.Request(self.start_urls[0], dont_filter=True)