nbulaj#14: allow to set custom proxy for Client

CHANGELOG.md

@@ -2,6 +2,11 @@
 
 Reverse Chronological Order:
 
+## `0.6.3` (2018-01-26)
+
+* Add ability to use own proxy for `ProxyFetcher::Client`
+* Improve specs
+
 ## `0.6.2` (2017-12-27)
 
 * Fix ProxyDocker provider.

README.md

@@ -245,6 +245,17 @@ require 'proxy-fetcher'
 ProxyFetcher::Client.get 'https://example.com/resource', options: { max_retries: 10_000 }
 ```
 
+You can also use your own proxy object when using ProxyFetcher client:
+
+```ruby
+require 'proxy-fetcher'
+
+manager = ProxyFetcher::Manager.new # will immediately load proxy list from the server
+
+#random will return random proxy object from the list
+ProxyFetcher::Client.get 'https://example.com/resource', options: { proxy: manager.random }
+```
+
 Btw, if you need support of JavaScript or some other features, you need to implement your own client using, for example,
 `selenium-webdriver`.
 

lib/proxy_fetcher/client/client.rb

@@ -127,20 +127,20 @@ def patch(url, payload, headers: {}, options: {})
       # Executes HTTP request with user payload.
       #
       def request_with_payload(method, url, payload, headers, options)
-        safe_request_to(url, options.fetch(:max_retries, 1000)) do |proxy|
-          opts = options.merge(url: url, payload: payload, proxy: proxy, headers: default_headers.merge(headers))
+        with_proxy_for(url, options.fetch(:max_retries, 1000)) do |proxy|
+          opts = options.merge(payload: payload, proxy: options.fetch(:proxy, proxy), headers: default_headers.merge(headers))
 
-          Request.execute(method: method, **opts)
+          Request.execute(url: url, method: method, **opts)
         end
       end
 
       # Executes HTTP request without user payload.
       #
       def request_without_payload(method, url, headers, options)
-        safe_request_to(url, options.fetch(:max_retries, 1000)) do |proxy|
-          opts = options.merge(url: url, proxy: proxy, headers: default_headers.merge(headers))
+        with_proxy_for(url, options.fetch(:max_retries, 1000)) do |proxy|
+          opts = options.merge(proxy: options.fetch(:proxy, proxy), headers: default_headers.merge(headers))
 
-          Request.execute(method: method, **opts)
+          Request.execute(url: url, method: method, **opts)
         end
       end
 
@@ -156,15 +156,16 @@ def default_headers
         }
       end
 
-      # Sends request to the URL using proxy with specific number
-      # of retries (default is 1000) if request failed (404, 500, timeout, etc).
+      # Searches for valid proxy (suitable for URL type) using <code>ProxyFetcher::Manager</code>
+      # instance and executes the block with found proxy with retries (N times, default is 1000) if
+      # something goes wrong.
       #
       # @param url [String] request URL
       # @param max_retries [Integer] maximum number of retries
       #
-      # @raise [ProxyFetcher::Error] internal error
+      # @raise [ProxyFetcher::Error] internal error happened during block execution
       #
-      def safe_request_to(url, max_retries = 1000)
+      def with_proxy_for(url, max_retries = 1000)
         tries = 0
 
         begin

lib/proxy_fetcher/client/proxies_registry.rb

@@ -41,7 +41,7 @@ def find_proxy_for(url)
           find_proxy_for(url)
         end
 
-        # Instantiate or returns <code>ProxyFetcher::Manager</code> instance
+        # Instantiates or returns <code>ProxyFetcher::Manager</code> instance
         # for current <code>Thread</code>.
         #
         # @return [ProxyFetcher::Manager]

lib/proxy_fetcher/providers/base.rb

@@ -29,7 +29,7 @@ def self.fetch_proxies!(*args)
       #   ProxyFetcher document object
       #
       def load_document(url, filters = {})
-        raise ArgumentError, 'filters must be a Hash' unless filters.is_a?(Hash)
+        raise ArgumentError, 'filters must be a Hash' if filters && !filters.is_a?(Hash)
 
         uri = URI.parse(url)
         uri.query = URI.encode_www_form(filters) if filters && filters.any?

lib/proxy_fetcher/version.rb

@@ -15,7 +15,7 @@ module VERSION
     # Minor version number
     MINOR = 6
     # Smallest version number
-    TINY  = 2
+    TINY  = 3
 
     # Full version number
     STRING = [MAJOR, MINOR, TINY].compact.join('.')

proxy_fetcher.gemspec

@@ -5,7 +5,7 @@ require 'proxy_fetcher/version'
 Gem::Specification.new do |gem|
   gem.name = 'proxy_fetcher'
   gem.version = ProxyFetcher.gem_version
-  gem.date = '2017-12-27'
+  gem.date = '2018-01-26'
   gem.summary = 'Ruby gem for dealing with proxy lists from different providers'
   gem.description = 'This gem can help your Ruby application to make HTTP(S) requests ' \
                     'using proxies by fetching and validating proxy lists from the different providers.'

spec/proxy_fetcher/client/proxies_registry_spec.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe ProxyFetcher::Client::ProxiesRegistry do
+  context '#manager' do
+    it 'instantiates ProxyFetcher::Manager instance' do
+      expect(described_class.manager).not_to be_nil
+      expect(described_class.manager).to be_an_instance_of(ProxyFetcher::Manager)
+    end
+
+    it 'caches manager instance' do
+      expect(described_class.manager).to eq(described_class.manager)
+    end
+  end
+
+  context '#invalidate_proxy!' do
+    it 'removes proxy from the list' do
+      proxy = described_class.manager.proxies.first
+      described_class.invalidate_proxy!(proxy)
+
+      expect(described_class.manager.proxies).not_to include(proxy)
+    end
+
+    it 'refreshes the list if it is empty' do
+      proxy = described_class.manager.proxies.first
+      described_class.manager.instance_variable_set(:'@proxies', [proxy])
+
+      described_class.invalidate_proxy!(proxy)
+
+      expect(described_class.manager.proxies.count).to be > 1
+    end
+  end
+
+  context '#find_proxy_for' do
+    it 'searches for specific proxy based on URL schema type' do
+      expect(described_class.find_proxy_for('http://google.com')).not_to be_nil
+
+      proxy = described_class.find_proxy_for('https://google.com')
+      expect(proxy.ssl?).to be_truthy
+    end
+  end
+end

spec/proxy_fetcher/version_spec.rb

@@ -0,0 +1,3 @@
+RSpec.describe ProxyFetcher::VERSION do
+  it { expect(ProxyFetcher::VERSION::STRING).to eq '0.6.3' }
+end