From 29d104e6b6b26b713571c2db4bb740f01dce3b2f Mon Sep 17 00:00:00 2001 From: Daniel Vandersluis Date: Mon, 4 Oct 2021 15:43:43 -0400 Subject: [PATCH] [Fix #8101] Reformat `rake spec` output to amplify signal and reduce noise. --- lib/rubocop/rspec/parallel_formatter.rb | 90 +++++++++++++++++++++++++ lib/rubocop/rspec/support.rb | 1 + spec/spec_helper.rb | 12 ++++ tasks/spec_runner.rake | 58 ++++++++++++++-- 4 files changed, 157 insertions(+), 4 deletions(-) create mode 100644 lib/rubocop/rspec/parallel_formatter.rb diff --git a/lib/rubocop/rspec/parallel_formatter.rb b/lib/rubocop/rspec/parallel_formatter.rb new file mode 100644 index 000000000000..01b7991ebca1 --- /dev/null +++ b/lib/rubocop/rspec/parallel_formatter.rb @@ -0,0 +1,90 @@ +# frozen_string_literal: true + +RSpec::Support.require_rspec_core 'formatters/base_text_formatter' +RSpec::Support.require_rspec_core 'formatters/console_codes' + +module RuboCop + module RSpec + # RSpec formatter for use with running `rake spec` in parallel. This formatter + # removes much of the noise from RSpec so that only the important information + # will be surfaced by test-queue. + # It also adds metadata to the output in order to more easily find the text + # needed for outputting after the parallel run completes. + class ParallelFormatter < ::RSpec::Core::Formatters::BaseTextFormatter + ::RSpec::Core::Formatters.register self, :dump_pending, :dump_failures, :dump_summary + + # Don't show pending tests + def dump_pending(*); end + + # The BEGIN/END comments are used by `spec_runner.rake` to determine what + # output goes where in the final parallelized output, and should not be + # removed! + def dump_failures(notification) + return if notification.failure_notifications.empty? + + output.puts '# FAILURES BEGIN' + notification.failure_notifications.each do |failure| + output.puts failure.fully_formatted('*', colorizer) + end + output.puts + output.puts '# FAILURES END' + end + + def dump_summary(summary) + output_summary(summary) + output_rerun_commands(summary) + end + + private + + def colorizer + @colorizer ||= ::RSpec::Core::Formatters::ConsoleCodes + end + + # The BEGIN/END comments are used by `spec_runner.rake` to determine what + # output goes where in the final parallelized output, and should not be + # removed! + def output_summary(summary) + output.puts '# SUMMARY BEGIN' + output.puts colorize_summary(summary) + output.puts '# SUMMARY END' + end + + def colorize_summary(summary) + totals = totals(summary) + + if summary.failure_count.positive? || summary.errors_outside_of_examples_count.positive? + colorizer.wrap(totals, ::RSpec.configuration.failure_color) + else + colorizer.wrap(totals, ::RSpec.configuration.success_color) + end + end + + # The BEGIN/END comments are used by `spec_runner.rake` to determine what + # output goes where in the final parallelized output, and should not be + # removed! + def output_rerun_commands(summary) + output.puts '# RERUN BEGIN' + output.puts summary.colorized_rerun_commands.lines[3..-1].join + output.puts '# RERUN END' + end + + def totals(summary) + output = pluralize(summary.example_count, 'example') + output += ", #{summary.pending_count} pending" if summary.pending_count.positive? + output += ", #{pluralize(summary.failure_count, 'failure')}" + + if summary.errors_outside_of_examples_count.positive? + error_count = pluralize(summary.errors_outside_of_examples_count, 'error') + output += ", #{error_count} occurred outside of examples" + end + + output + end + + def pluralize(*args) + ::RSpec::Core::Formatters::Helpers.pluralize(*args) + end + end + end +end diff --git a/lib/rubocop/rspec/support.rb b/lib/rubocop/rspec/support.rb index d21ef506a239..8e2dd89019d7 100644 --- a/lib/rubocop/rspec/support.rb +++ b/lib/rubocop/rspec/support.rb @@ -6,6 +6,7 @@ require_relative 'host_environment_simulation_helper' require_relative 'shared_contexts' require_relative 'expect_offense' +require_relative 'parallel_formatter' RSpec.configure do |config| config.include CopHelper diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 224389fa10b5..5a1a04a24ae1 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -78,3 +78,15 @@ config.filter_run_excluding broken_on: :jruby end end + +module ::RSpec + module Core + class ExampleGroup + # Override `failure_count` from test-queue to prevent RSpec deprecation notice + # Treating `metadata[:execution_result]` as a hash is deprecated. + def self.failure_count + examples.map { |e| e.execution_result.status == 'failed' }.length + end + end + end +end diff --git a/tasks/spec_runner.rake b/tasks/spec_runner.rake index 5e30fed7b3a3..c1b69d4994ff 100644 --- a/tasks/spec_runner.rake +++ b/tasks/spec_runner.rake @@ -4,6 +4,14 @@ require 'rspec/core' require 'test_queue' require 'test_queue/runner/rspec' +# Add `failed_examples` into `TestQueue::Worker` so we can keep +# track of the output for re-running failed examples from RSpec. +module TestQueue + class Worker + attr_accessor :failed_examples + end +end + module RuboCop # Helper for running specs with a temporary external encoding. # This is a bit risky, since strings defined before the block may have a @@ -13,7 +21,7 @@ module RuboCop class SpecRunner attr_reader :rspec_args - def initialize(rspec_args = %w[spec], parallel: true, + def initialize(rspec_args = %w[spec --force-color], parallel: true, external_encoding: 'UTF-8', internal_encoding: nil) @rspec_args = rspec_args @previous_external_encoding = Encoding.default_external @@ -59,10 +67,15 @@ module RuboCop # `TestQueue::Runner::RSpec`, but modified so that it takes an argument # (an array of paths of specs to run) instead of relying on ARGV. class ParallelRunner < ::TestQueue::Runner + SUMMARY_REGEXP = /(?<=# SUMMARY BEGIN\n).*(?=\n# SUMMARY END)/m.freeze + FAILURE_OUTPUT_REGEXP = /(?<=# FAILURES BEGIN\n\n).*(?=# FAILURES END)/m.freeze + RERUN_REGEXP = /(?<=# RERUN BEGIN\n).+(?=\n# RERUN END)/m.freeze + def initialize(rspec_args) super(Framework.new(rspec_args)) @exit_when_done = false + @failure_count = 0 end def run_worker(iterator) @@ -70,9 +83,42 @@ module RuboCop rspec.run_each(iterator).to_i end + # Override `TestQueue::Runner#worker_completed` to not output anything + # as it adds a lot of noise by default + def worker_completed(worker) + return if @aborting + + @completed << worker + end + def summarize_worker(worker) - worker.summary = worker.lines.grep(/\A\d+ examples?, /).first - worker.failure_output = worker.output[/^Failures:\n\n(.*)\n^Finished/m, 1] + worker.summary = worker.output[SUMMARY_REGEXP] + worker.failure_output = update_count(worker.output[FAILURE_OUTPUT_REGEXP]) + worker.failed_examples = worker.output[RERUN_REGEXP] + end + + def summarize_internal + ret = super + + unless @failures.blank? + puts "==> Failed Examples\n\n" + puts @completed.map(&:failed_examples).compact.sort.join("\n") + puts + end + + ret + end + + private + + def update_count(failures) + # The ParallelFormatter formatter doesn't try to count failures, but + # prefixes each with `*)`, so that they can be updated to count failures + # globally once all workers have completed. + + return unless failures + + failures.gsub('*)') { "#{@failure_count += 1})" } end end @@ -96,7 +142,11 @@ module RuboCop class Framework < ::TestQueue::TestFramework::RSpec def initialize(rspec_args) super() - @rspec_args = rspec_args + formatter_args = %w[ + --require ./lib/rubocop/rspec/parallel_formatter.rb + --format RuboCop::RSpec::ParallelFormatter + ] + @rspec_args = rspec_args.concat(formatter_args) end def all_suite_files