From 66db49e1f58250045c5eab8d222553081430ff7b Mon Sep 17 00:00:00 2001 From: Nikita Bulai Date: Wed, 17 Feb 2021 11:05:31 +0300 Subject: [PATCH] Add Ruby 3 to Travis --- .rubocop.yml | 8 ++- .travis.yml | 3 +- Gemfile | 4 +- README.md | 2 +- gemfiles/nokogiri.gemfile | 2 +- gemfiles/oga.gemfile | 2 +- lib/proxy_fetcher.rb | 64 ++++++++++++------------ lib/proxy_fetcher/client/request.rb | 6 +-- lib/proxy_fetcher/configuration.rb | 2 +- lib/proxy_fetcher/document/node.rb | 2 +- lib/proxy_fetcher/utils/http_client.rb | 38 ++++++++------ proxy_fetcher.gemspec | 4 +- spec/proxy_fetcher/client/client_spec.rb | 5 ++ 13 files changed, 82 insertions(+), 60 deletions(-) diff --git a/.rubocop.yml b/.rubocop.yml index 6501874..f268ac7 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -1,5 +1,5 @@ AllCops: - TargetRubyVersion: 2.3 + TargetRubyVersion: 2.4 Exclude: - 'bin/*' DisplayCopNames: true @@ -16,11 +16,15 @@ Style/StringLiteralsInInterpolation: Layout/MultilineMethodCallIndentation: EnforcedStyle: indented -Layout/TrailingBlankLines: +Layout/TrailingEmptyLines: Enabled: true Layout/DotPosition: EnforcedStyle: leading +Lint/ConstantDefinitionInBlock: + Exclude: + - spec/**/* + Metrics/LineLength: Exclude: - spec/**/* diff --git a/.travis.yml b/.travis.yml index a211606..ed79fa7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,7 @@ cache: bundler before_install: - "find /home/travis/.rvm/rubies -wholename '*default/bundler-*.gemspec' -delete" - rvm @global do gem uninstall bundler -a -x -I || true - - gem install bundler -v '~> 1.10' + - gem install bundler bundler_args: --without yard guard benchmarks env: JRUBY_OPTS="$JRUBY_OPTS --debug" @@ -18,6 +18,7 @@ rvm: - 2.5 - 2.6 - 2.7 + - 3.0 - ruby-head - jruby-9.2.8.0 - truffleruby-head diff --git a/Gemfile b/Gemfile index 23c98dd..673ccfc 100644 --- a/Gemfile +++ b/Gemfile @@ -6,9 +6,11 @@ gemspec gem "nokogiri", "~> 1.8" gem "oga", "~> 3.2" -gem "rubocop", "~> 0.80" +gem "rubocop", "~> 1.0" group :test do gem "coveralls", require: false + # Until I find a way to introduce other MITM proxy + gem "webrick", "1.4.2" gem "evil-proxy", "~> 0.2" end diff --git a/README.md b/README.md index e751f0d..1031240 100644 --- a/README.md +++ b/README.md @@ -295,7 +295,7 @@ Default configuration looks as follows: ```ruby ProxyFetcher.configure do |config| - config.logger = Logger.new(STDOUT) + config.logger = Logger.new($stdout) config.user_agent = ProxyFetcher::Configuration::DEFAULT_USER_AGENT config.pool_size = 10 config.client_timeout = 3 diff --git a/gemfiles/nokogiri.gemfile b/gemfiles/nokogiri.gemfile index 5ea1a48..fa4e682 100644 --- a/gemfiles/nokogiri.gemfile +++ b/gemfiles/nokogiri.gemfile @@ -8,6 +8,6 @@ gem "nokogiri", "~> 1.8" group :test do gem "coveralls", require: false + gem "webrick" gem "evil-proxy", "~> 0.2" - gem "rspec", "~> 3.9" end diff --git a/gemfiles/oga.gemfile b/gemfiles/oga.gemfile index 801f12e..8a9f3d2 100644 --- a/gemfiles/oga.gemfile +++ b/gemfiles/oga.gemfile @@ -8,6 +8,6 @@ gem "oga", "~> 3.0" group :test do gem "coveralls", require: false + gem "webrick" gem "evil-proxy", "~> 0.2" - gem "rspec", "~> 3.9" end diff --git a/lib/proxy_fetcher.rb b/lib/proxy_fetcher.rb index 3639fa8..177cc93 100644 --- a/lib/proxy_fetcher.rb +++ b/lib/proxy_fetcher.rb @@ -4,47 +4,47 @@ require "http" require "logger" -require File.dirname(__FILE__) + "/proxy_fetcher/version" +require "#{File.dirname(__FILE__)}/proxy_fetcher/version" -require File.dirname(__FILE__) + "/proxy_fetcher/exceptions" -require File.dirname(__FILE__) + "/proxy_fetcher/configuration" -require File.dirname(__FILE__) + "/proxy_fetcher/configuration/providers_registry" -require File.dirname(__FILE__) + "/proxy_fetcher/proxy" -require File.dirname(__FILE__) + "/proxy_fetcher/manager" -require File.dirname(__FILE__) + "/proxy_fetcher/null_logger" +require "#{File.dirname(__FILE__)}/proxy_fetcher/exceptions" +require "#{File.dirname(__FILE__)}/proxy_fetcher/configuration" +require "#{File.dirname(__FILE__)}/proxy_fetcher/configuration/providers_registry" +require "#{File.dirname(__FILE__)}/proxy_fetcher/proxy" +require "#{File.dirname(__FILE__)}/proxy_fetcher/manager" +require "#{File.dirname(__FILE__)}/proxy_fetcher/null_logger" -require File.dirname(__FILE__) + "/proxy_fetcher/utils/http_client" -require File.dirname(__FILE__) + "/proxy_fetcher/utils/proxy_validator" -require File.dirname(__FILE__) + "/proxy_fetcher/utils/proxy_list_validator" -require File.dirname(__FILE__) + "/proxy_fetcher/client/client" -require File.dirname(__FILE__) + "/proxy_fetcher/client/request" -require File.dirname(__FILE__) + "/proxy_fetcher/client/proxies_registry" +require "#{File.dirname(__FILE__)}/proxy_fetcher/utils/http_client" +require "#{File.dirname(__FILE__)}/proxy_fetcher/utils/proxy_validator" +require "#{File.dirname(__FILE__)}/proxy_fetcher/utils/proxy_list_validator" +require "#{File.dirname(__FILE__)}/proxy_fetcher/client/client" +require "#{File.dirname(__FILE__)}/proxy_fetcher/client/request" +require "#{File.dirname(__FILE__)}/proxy_fetcher/client/proxies_registry" -require File.dirname(__FILE__) + "/proxy_fetcher/document" -require File.dirname(__FILE__) + "/proxy_fetcher/document/adapters" -require File.dirname(__FILE__) + "/proxy_fetcher/document/node" -require File.dirname(__FILE__) + "/proxy_fetcher/document/adapters/abstract_adapter" -require File.dirname(__FILE__) + "/proxy_fetcher/document/adapters/nokogiri_adapter" -require File.dirname(__FILE__) + "/proxy_fetcher/document/adapters/oga_adapter" +require "#{File.dirname(__FILE__)}/proxy_fetcher/document" +require "#{File.dirname(__FILE__)}/proxy_fetcher/document/adapters" +require "#{File.dirname(__FILE__)}/proxy_fetcher/document/node" +require "#{File.dirname(__FILE__)}/proxy_fetcher/document/adapters/abstract_adapter" +require "#{File.dirname(__FILE__)}/proxy_fetcher/document/adapters/nokogiri_adapter" +require "#{File.dirname(__FILE__)}/proxy_fetcher/document/adapters/oga_adapter" ## # Ruby / JRuby lib for managing proxies module ProxyFetcher # ProxyFetcher providers namespace module Providers - require File.dirname(__FILE__) + "/proxy_fetcher/providers/base" - require File.dirname(__FILE__) + "/proxy_fetcher/providers/free_proxy_list" - require File.dirname(__FILE__) + "/proxy_fetcher/providers/free_proxy_list_socks" - require File.dirname(__FILE__) + "/proxy_fetcher/providers/free_proxy_list_ssl" - require File.dirname(__FILE__) + "/proxy_fetcher/providers/free_proxy_list_us" - require File.dirname(__FILE__) + "/proxy_fetcher/providers/http_tunnel" - require File.dirname(__FILE__) + "/proxy_fetcher/providers/mtpro" - require File.dirname(__FILE__) + "/proxy_fetcher/providers/proxy_list" - require File.dirname(__FILE__) + "/proxy_fetcher/providers/proxypedia" - require File.dirname(__FILE__) + "/proxy_fetcher/providers/proxyscrape_http" - require File.dirname(__FILE__) + "/proxy_fetcher/providers/proxyscrape_socks4" - require File.dirname(__FILE__) + "/proxy_fetcher/providers/proxyscrape_socks5" - require File.dirname(__FILE__) + "/proxy_fetcher/providers/xroxy" + require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/base" + require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/free_proxy_list" + require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/free_proxy_list_socks" + require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/free_proxy_list_ssl" + require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/free_proxy_list_us" + require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/http_tunnel" + require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/mtpro" + require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/proxy_list" + require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/proxypedia" + require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/proxyscrape_http" + require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/proxyscrape_socks4" + require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/proxyscrape_socks5" + require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/xroxy" end @__config_access_lock__ = Mutex.new diff --git a/lib/proxy_fetcher/client/request.rb b/lib/proxy_fetcher/client/request.rb index e373bb7..c1bffce 100644 --- a/lib/proxy_fetcher/client/request.rb +++ b/lib/proxy_fetcher/client/request.rb @@ -41,15 +41,15 @@ class Request # @return [String] # response body (requested resource content) # - def self.execute(args) - new(args).execute + def self.execute(**args) + new(**args).execute end # Initialize new HTTP request # # @return [Request] # - def initialize(args) + def initialize(**args) raise ArgumentError, "args must be a Hash!" unless args.is_a?(Hash) @url = args.fetch(:url) diff --git a/lib/proxy_fetcher/configuration.rb b/lib/proxy_fetcher/configuration.rb index 1281481..468d783 100644 --- a/lib/proxy_fetcher/configuration.rb +++ b/lib/proxy_fetcher/configuration.rb @@ -114,7 +114,7 @@ def initialize # Sets default configuration options def reset! - @logger = Logger.new(STDOUT) + @logger = Logger.new($stdout) @user_agent = DEFAULT_USER_AGENT @pool_size = 10 @client_timeout = 3 diff --git a/lib/proxy_fetcher/document/node.rb b/lib/proxy_fetcher/document/node.rb index e02e963..5c184d5 100644 --- a/lib/proxy_fetcher/document/node.rb +++ b/lib/proxy_fetcher/document/node.rb @@ -83,7 +83,7 @@ def html def clear(text) return "" if text.nil? || text.empty? - text.strip.gsub(/[\t]/i, "") + text.strip.gsub(/\t/i, "") end end end diff --git a/lib/proxy_fetcher/utils/http_client.rb b/lib/proxy_fetcher/utils/http_client.rb index 08a14c6..89aa4a3 100644 --- a/lib/proxy_fetcher/utils/http_client.rb +++ b/lib/proxy_fetcher/utils/http_client.rb @@ -41,9 +41,8 @@ class HTTPClient # @return [String] # resource content # - def self.fetch(*args) - url = args.shift - new(url, **args.first).fetch + def self.fetch(*args, **kwargs, &block) + new(*args, **kwargs, &block).fetch end # Initialize HTTP client instance @@ -52,15 +51,17 @@ def self.fetch(*args) # def initialize(url, method: :get, params: {}, headers: {}) @url = url.to_s - @method = method + @method = method.to_sym @params = params @headers = headers - @http = HTTP.headers(default_headers.merge(headers)).timeout(connect: timeout, read: timeout) - @timeout = ProxyFetcher.config.provider_proxies_load_timeout + unless HTTP::Request::METHODS.include?(@method) + raise ArgumentError, "'#{@method}' is a wrong HTTP method name" + end - @ssl_ctx = OpenSSL::SSL::SSLContext.new - @ssl_ctx.verify_mode = OpenSSL::SSL::VERIFY_NONE + @timeout = ProxyFetcher.config.provider_proxies_load_timeout + @http = build_http_engine + @ssl_ctx = build_ssl_context end # Fetches resource content by sending HTTP request to it. @@ -68,8 +69,10 @@ def initialize(url, method: :get, params: {}, headers: {}) # @return [String] # response body # - def fetch - response = process_http_request + def fetch(**options) + response = perform_http_request + return response if options.fetch(:raw, false) + response.body.to_s rescue StandardError => e ProxyFetcher.config.logger.warn("Failed to process request to #{url} (#{e.message})") @@ -78,13 +81,20 @@ def fetch protected - def process_http_request(http_method: method, http_params: params) - unless HTTP::Request::METHODS.include?(http_method) - raise ArgumentError, "'#{http_method}' is a wrong HTTP method name!" + def build_ssl_context + OpenSSL::SSL::SSLContext.new.tap do |context| + context.verify_mode = OpenSSL::SSL::VERIFY_NONE end + end + + def build_http_engine + HTTP.headers(default_headers.merge(headers)).timeout(connect: timeout, read: timeout) + end + def perform_http_request(http_method: method, http_params: params) http.public_send( - http_method.to_sym, url, + http_method, + url, form: http_params, ssl_context: ssl_ctx ) diff --git a/proxy_fetcher.gemspec b/proxy_fetcher.gemspec index 8552100..4daa91e 100644 --- a/proxy_fetcher.gemspec +++ b/proxy_fetcher.gemspec @@ -9,8 +9,8 @@ Gem::Specification.new do |gem| gem.version = ProxyFetcher.gem_version gem.summary = "Ruby gem for dealing with proxy lists from different providers" gem.description = <<-TEXT.strip.gsub(/[\s\n]+/, " ") - This gem can help your Ruby application to make HTTP(S) requests - using proxies by fetching and validating proxy lists from + This gem can help your Ruby application to make HTTP(S) requests + using proxies by fetching and validating proxy lists from the different providers. TEXT gem.authors = ["Nikita Bulai"] diff --git a/spec/proxy_fetcher/client/client_spec.rb b/spec/proxy_fetcher/client/client_spec.rb index d55681b..27d6c9c 100644 --- a/spec/proxy_fetcher/client/client_spec.rb +++ b/spec/proxy_fetcher/client/client_spec.rb @@ -3,6 +3,11 @@ require "spec_helper" require "json" +begin + require "webrick" +rescue LoadError + # nop +end require "evil-proxy" require "evil-proxy/async"