diff --git a/CHANGELOG.md b/CHANGELOG.md index 30d58bf..c4e00f3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,11 @@ Reverse Chronological Order: * Add your description here +## `0.10.1` (2019-03-07) + +* Fix broken ProxyDocker provider. +* Refactor gem internals. + ## `0.9.0` (2019-01-22) * Fix a problem with stuck of proxies list loading. diff --git a/bin/console b/bin/console new file mode 100755 index 0000000..237a43d --- /dev/null +++ b/bin/console @@ -0,0 +1,14 @@ +#!/usr/bin/env ruby + +require 'bundler/setup' +require 'proxy_fetcher' + +# You can add fixtures and/or initialization code here to make experimenting +# with your gem easier. You can also use a different console, if you like. + +# (If you use this, don't forget to add pry to your Gemfile!) +# require "pry" +# Pry.start + +require 'irb' +IRB.start(__FILE__) diff --git a/lib/proxy_fetcher/configuration.rb b/lib/proxy_fetcher/configuration.rb index d1f4917..1d90dc5 100644 --- a/lib/proxy_fetcher/configuration.rb +++ b/lib/proxy_fetcher/configuration.rb @@ -35,7 +35,11 @@ class Configuration # @!attribute [r] adapter # @return [Object] HTML parser adapter - attr_reader :adapter + attr_accessor :adapter + + # @!attribute [r] adapter_class + # @return [Object] HTML adapter class + attr_reader :adapter_class # @!attribute [r] http_client # @return [Object] HTTP client class @@ -120,14 +124,11 @@ def reset! self.providers = self.class.registered_providers end - # Setups HTML parser adapter for all the proxy providers. - # - # @param name_or_class [String, Symbol, Class] - # name of the adapter or it's class - # - def adapter=(name_or_class) - @adapter = ProxyFetcher::Document::Adapters.lookup(name_or_class) - @adapter.setup! + def adapter_class + return @adapter_class if defined?(@adapter_class) + + @adapter_class = ProxyFetcher::Document::Adapters.lookup(adapter) + @adapter_class.setup! end # Setups collection of providers that will be used to fetch proxies. diff --git a/lib/proxy_fetcher/document.rb b/lib/proxy_fetcher/document.rb index 55abd53..f957f50 100644 --- a/lib/proxy_fetcher/document.rb +++ b/lib/proxy_fetcher/document.rb @@ -17,7 +17,7 @@ class Document # ProxyFetcher document model # def self.parse(data) - new(ProxyFetcher.config.adapter.parse(data)) + new(ProxyFetcher.config.adapter_class.parse(data)) end # Initialize abstract ProxyFetcher HTML Document diff --git a/lib/proxy_fetcher/document/adapters.rb b/lib/proxy_fetcher/document/adapters.rb index 6910d20..32a91f9 100644 --- a/lib/proxy_fetcher/document/adapters.rb +++ b/lib/proxy_fetcher/document/adapters.rb @@ -30,7 +30,7 @@ def lookup(name_or_class) case name_or_class when Symbol, String - adapter_name = name_or_class.to_s.capitalize << ADAPTER + adapter_name = "#{name_or_class.to_s.capitalize}#{ADAPTER}" ProxyFetcher::Document.const_get(adapter_name) else name_or_class diff --git a/lib/proxy_fetcher/document/adapters/abstract_adapter.rb b/lib/proxy_fetcher/document/adapters/abstract_adapter.rb index 6ed093e..8684709 100644 --- a/lib/proxy_fetcher/document/adapters/abstract_adapter.rb +++ b/lib/proxy_fetcher/document/adapters/abstract_adapter.rb @@ -52,6 +52,7 @@ def proxy_node # def self.setup!(*args) install_requirements!(*args) + self rescue LoadError => error raise Exceptions::AdapterSetupError.new(name, error.message) end diff --git a/lib/proxy_fetcher/providers/base.rb b/lib/proxy_fetcher/providers/base.rb index 2aef445..216ca5e 100644 --- a/lib/proxy_fetcher/providers/base.rb +++ b/lib/proxy_fetcher/providers/base.rb @@ -74,7 +74,10 @@ def load_document(url, filters = {}) def build_proxy(*args) to_proxy(*args) rescue StandardError => error - ProxyFetcher.logger.warn("Failed to build Proxy object due to error: #{error.message}") + ProxyFetcher.logger.warn( + "Failed to build Proxy object for #{self.class.name} due to error: #{error.message}" + ) + nil end diff --git a/lib/proxy_fetcher/providers/proxy_docker.rb b/lib/proxy_fetcher/providers/proxy_docker.rb index 2325e38..1f494b1 100644 --- a/lib/proxy_fetcher/providers/proxy_docker.rb +++ b/lib/proxy_fetcher/providers/proxy_docker.rb @@ -6,6 +6,8 @@ module ProxyFetcher module Providers # ProxyDocker provider class. class ProxyDocker < Base + attr_reader :cookie, :token + # Provider URL to fetch proxy list def provider_url 'https://www.proxydocker.com/en/api/proxylist/' @@ -17,7 +19,7 @@ def provider_method def provider_params { - token: 'GmZyl0OJmmgrWakdzO7AFf6AWfkdledR6xmKvGmwmJg', + token: @token, country: 'all', city: 'all', state: 'all', @@ -31,7 +33,7 @@ def provider_params def provider_headers { - cookie: 'PHPSESSID=7f59558ee58b1e4352c4ab4c2f1a3c11' + cookie: @cookie } end @@ -44,6 +46,8 @@ def provider_headers # # [NOTE] Doesn't support direct filters def load_proxy_list(*) + load_dependencies + json = JSON.parse(load_html(provider_url, {})) json.fetch('proxies', []) rescue JSON::ParserError @@ -70,6 +74,8 @@ def to_proxy(node) end end + private + def types_mapping { '16' => ProxyFetcher::Proxy::HTTP, @@ -80,6 +86,24 @@ def types_mapping '6' => ProxyFetcher::Proxy::HTTP # CON80 } end + + def load_dependencies + client = ProxyFetcher.config.http_client.new('https://www.proxydocker.com') + response = client.fetch_with_headers + + @cookie = load_cookie_from(response) + @token = load_token_from(response) + end + + def load_cookie_from(response) + cookie_headers = (response.headers['Set-Cookie'] || []) + cookie_headers.join('; ') + end + + def load_token_from(response) + html = response.body.to_s + html[/meta\s+name\s*=["']_token["']\s+content.+["'](.+?)["']\s*>/i, 1] + end end ProxyFetcher::Configuration.register_provider(:proxy_docker, ProxyDocker) diff --git a/lib/proxy_fetcher/utils/http_client.rb b/lib/proxy_fetcher/utils/http_client.rb index ea8deea..712e98a 100644 --- a/lib/proxy_fetcher/utils/http_client.rb +++ b/lib/proxy_fetcher/utils/http_client.rb @@ -68,21 +68,32 @@ def initialize(url, method: :get, params: {}, headers: {}) # response body # def fetch - # TODO: must be more generic - response = if method == :post - http.post(url, form: params, ssl_context: ssl_ctx) - else - http.get(url, ssl_context: ssl_ctx) - end - + response = process_http_request response.body.to_s rescue StandardError => error - ProxyFetcher.logger.warn("Failed to load proxy list for #{url} (#{error.message})") + ProxyFetcher.logger.warn("Failed to process request to #{url} (#{error.message})") '' end + def fetch_with_headers + process_http_request + rescue StandardError => error + ProxyFetcher.logger.warn("Failed to process request to #{url} (#{error.message})") + HTTP::Response.new(version: '1.1', status: 500, body: '') + end + protected + def process_http_request(http_method: method, http_params: params) + raise ArgumentError, 'wrong http method name!' unless HTTP::Request::METHODS.include?(http_method) + + http.public_send( + http_method.to_sym, url, + form: http_params, + ssl_context: ssl_ctx + ) + end + # Default HTTP client headers # # @return [Hash] diff --git a/lib/proxy_fetcher/version.rb b/lib/proxy_fetcher/version.rb index 326a1ac..3e330dd 100644 --- a/lib/proxy_fetcher/version.rb +++ b/lib/proxy_fetcher/version.rb @@ -15,7 +15,7 @@ module VERSION # Minor version number MINOR = 10 # Smallest version number - TINY = 0 + TINY = 1 # Full version number STRING = [MAJOR, MINOR, TINY].compact.join('.') diff --git a/spec/proxy_fetcher/configuration_spec.rb b/spec/proxy_fetcher/configuration_spec.rb index b55f401..f775293 100644 --- a/spec/proxy_fetcher/configuration_spec.rb +++ b/spec/proxy_fetcher/configuration_spec.rb @@ -68,7 +68,9 @@ def self.install_requirements! end end - expect { ProxyFetcher.config.adapter = CustomAdapter } + ProxyFetcher.config.adapter = CustomAdapter + + expect { ProxyFetcher::Manager.new } .to raise_error(ProxyFetcher::Exceptions::AdapterSetupError) ProxyFetcher.instance_variable_set('@config', old_config) diff --git a/spec/proxy_fetcher/providers/base_spec.rb b/spec/proxy_fetcher/providers/base_spec.rb index b9b83b9..6d7554b 100644 --- a/spec/proxy_fetcher/providers/base_spec.rb +++ b/spec/proxy_fetcher/providers/base_spec.rb @@ -44,7 +44,7 @@ def load_proxy_list(*) allow_any_instance_of(HTTP::Client).to receive(:get).and_raise(StandardError) - expect(logger).to receive(:warn).with(/Failed to load proxy list for http[s:\/]/) + expect(logger).to receive(:warn).with(/Failed to process request to http[s:\/]/) ProxyFetcher::Manager.new end