Omnibus test change branch (#58)

* fail on missing snapshots (#57) * include fork of rspec-snapshot that fails the build when the snapshot is written * confirm missing snapshots fail the build * Revert "confirm missing snapshots fail the build" This reverts commit b4e3dd2. * test exit more carefully (#56) * graceful exit within abq was masking rspec tests on older versions of rspec * add an additional exit safeguard * check num tests vs expected tests * sanitize backtrace tweaks * regenerate snapshots * break test to ensure bin/check_num_tests.rb works * Revert "break test to ensure bin/check_num_tests.rb works" This reverts commit 1da32a2. * set subprocess ENV outside of the global context (#54) * use captain cli (#55) * use captain cli * unset RWX_ACCESS_TOKEN to be compatible with captain-cli
rwx-research · Dec 9, 2022 · 00e5559 · 00e5559
1 parent f91022e
commit 00e5559
Show file tree

Hide file tree

Showing 70 changed files with 253 additions and 188 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -55,27 +55,27 @@ jobs:
         uses: rwx-research/setup-abq@v1
         with:
           access-token: ${{ secrets.RWX_ACCESS_TOKEN }}
-      - run: |
+      - uses: rwx-research/setup-captain@v1
+      - name: run rspec & upload to captain
+        run: |
+          captain run \
+            --suite-id rspec-abq \
+            --test-results tmp/rspec.json \
+            -- \
+            bundle exec rspec \
+              --format json --out tmp/rspec.json \
+              --format documentation && bin/check_num_tests.rb
+        if: ${{ matrix.ruby == 3.1 && matrix.gemfile == 3.12 }}
+        env:
+          RWX_ACCESS_TOKEN: ${{ secrets.RWX_ACCESS_TOKEN }}
+        timeout-minutes: 1
+      - name: run rspec without uploading to captain
+        run: |
           bundle exec rspec \
             --format json --out tmp/rspec.json \
-            --format documentation
+            --format documentation && bin/check_num_tests.rb
+        if: ${{ matrix.ruby != 3.1 || matrix.gemfile != 3.12 }}
         timeout-minutes: 1
-      - name: Upload test results to Captain
-        uses: rwx-research/upload-captain-artifact@v1
-        if: ${{ matrix.ruby == 3.1 && matrix.gemfile == 3.12 && always() }}
-        continue-on-error: true
-        with:
-          artifacts: |
-            [
-              {
-                "name": "RSpec",
-                "path": "tmp/rspec.json",
-                "kind": "test_results",
-                "parser": "rspec_json"
-              }
-            ]
-          job-matrix: ${{ toJSON(matrix) }}
-          captain-token: ${{ secrets.RWX_ACCESS_TOKEN }}
 
   # strategy borrowed from https://brunoscheufler.com/blog/2022-04-09-the-required-github-status-check-that-wasnt
   # this is used to summarize the matrix results into a single job (abq-work) that can be used as a required check

diff --git a/Gemfile b/Gemfile
@@ -19,4 +19,5 @@ group :development do
   gem "standard", require: false
   gem "yard", require: false
   gem "gem-release", require: false
+  gem "rspec-snapshot", github: "rwx-research/rspec-snapshot"
 end
diff --git a/Gemfile.lock b/Gemfile.lock
@@ -1,3 +1,11 @@
+GIT
+  remote: https://github.com/rwx-research/rspec-snapshot.git
+  revision: 77b3098b46190a291b32b52686d07c96cced18d5
+  specs:
+    rspec-snapshot (2.0.1)
+      awesome_print (> 1.0.0)
+      rspec (> 3.0.0)
+
 PATH
   remote: .
   specs:
@@ -12,19 +20,19 @@ GEM
     coderay (1.1.3)
     diff-lcs (1.5.0)
     gem-release (2.2.2)
-    json (2.6.2)
-    language_server-protocol (3.17.0.1)
+    json (2.6.3)
+    language_server-protocol (3.17.0.2)
     method_source (1.0.0)
     parallel (1.22.1)
-    parser (3.1.2.1)
+    parser (3.1.3.0)
       ast (~> 2.4.1)
     prettier_print (1.1.0)
     pry (0.14.1)
       coderay (~> 1.1)
       method_source (~> 1.0)
     rainbow (3.1.1)
     rake (13.0.6)
-    regexp_parser (2.6.0)
+    regexp_parser (2.6.1)
     rexml (3.2.5)
     rspec (3.12.0)
       rspec-core (~> 3.12.0)
@@ -38,38 +46,36 @@ GEM
     rspec-mocks (3.12.0)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.12.0)
-    rspec-snapshot (2.0.1)
-      awesome_print (> 1.0.0)
-      rspec (> 3.0.0)
     rspec-support (3.12.0)
-    rubocop (1.35.1)
+    rubocop (1.39.0)
       json (~> 2.3)
       parallel (~> 1.10)
       parser (>= 3.1.2.1)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.8, < 3.0)
       rexml (>= 3.2.5, < 4.0)
-      rubocop-ast (>= 1.20.1, < 2.0)
+      rubocop-ast (>= 1.23.0, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 1.4.0, < 3.0)
-    rubocop-ast (1.23.0)
+    rubocop-ast (1.24.0)
       parser (>= 3.1.1.0)
-    rubocop-performance (1.14.3)
+    rubocop-performance (1.15.1)
       rubocop (>= 1.7.0, < 2.0)
       rubocop-ast (>= 0.4.0)
     rubocop-rake (0.6.0)
       rubocop (~> 1.0)
-    rubocop-rspec (2.14.2)
+    rubocop-rspec (2.15.0)
       rubocop (~> 1.33)
     ruby-lsp (0.3.6)
       language_server-protocol (~> 3.17.0)
       sorbet-runtime
       syntax_tree (>= 4.0.2, < 5.0.0)
     ruby-progressbar (1.11.0)
-    sorbet-runtime (0.5.10564)
-    standard (1.16.1)
-      rubocop (= 1.35.1)
-      rubocop-performance (= 1.14.3)
+    sorbet-runtime (0.5.10577)
+    standard (1.19.1)
+      language_server-protocol (~> 3.17.0.2)
+      rubocop (= 1.39.0)
+      rubocop-performance (= 1.15.1)
     syntax_tree (4.3.0)
       prettier_print (>= 1.0.2)
     unicode-display_width (2.3.0)
@@ -88,7 +94,7 @@ DEPENDENCIES
   rake (~> 13.0)
   rspec (~> 3.12)
   rspec-abq!
-  rspec-snapshot (~> 2.0.1)
+  rspec-snapshot!
   rubocop
   rubocop-performance
   rubocop-rake

diff --git a/bin/check_num_tests.rb b/bin/check_num_tests.rb
@@ -0,0 +1,28 @@
+#!/usr/bin/env ruby
+
+# rspec-abq calls exit(0). Loading the code path that calls exit(0) returns from rspec immediately with a exit status of 0
+# which shows as "passing" in CI even if tests failed.
+#
+# Here, we compare the number of tests run to the expected number.
+
+DEFAULT_TEST_JSON_PATH = 'tmp/rspec.json'
+EXPECTED_TEST_NUMBER_SNAPSHOT_PATH = 'spec/NUM_TESTS'
+
+json_path = ARGV[0] || DEFAULT_TEST_JSON_PATH
+unless File.exists?(json_path)
+  warn "no test file found at #{json_path}"
+  warn "usage: bin/snapshot_num_tests.rb (path to json file) (default: #{DEFAULT_TEST_JSON_PATH}"
+  exit 1
+end
+
+require 'json'
+
+actual_num_tests = JSON.parse(File.read(json_path))['examples'].length
+expected_num_tests = File.read(EXPECTED_TEST_NUMBER_SNAPSHOT_PATH).strip.to_i
+
+if actual_num_tests != expected_num_tests
+  warn "We ran #{actual_num_tests} tests, but expected to run #{expected_num_tests}."
+  exit 1
+end
+
+puts "We ran #{actual_num_tests} tests, as expected."
diff --git a/bin/generate-abq-test-output.rb b/bin/generate-abq-test-output.rb
@@ -6,27 +6,27 @@
   FileUtils.rm file
 end
 
+ENV['UPDATE_SNAPSHOTS'] = 'true'
 threads = Dir['gemfiles/*.gemfile'].map do |gemfile|
   Thread.new do
-    ENV['BUNDLE_GEMFILE'] = gemfile
-    ENV['UPDATE_SNAPSHOTS'] = 'true'
-    puts(gemfile + ":" + `bundle exec rspec spec/features/integration_spec.rb`)
+    system({"BUNDLE_GEMFILE" => gemfile}, "bundle exec rspec spec/features/integration_spec.rb")
   end
 end
 
 threads << Thread.new do
-  ENV['UPDATE_SNAPSHOTS'] = 'true'
-  puts(`bundle exec rspec spec/features/manifest_spec.rb`)
+  system("bundle exec rspec spec/features/manifest_spec.rb")
 end
 
 threads.map(&:join)
 
-
-ENV['UPDATE_SNAPSHOTS'] = 'true'
-`bundle exec rspec spec/features/manifest_spec.rb`
+# rspec-abq calls exit(0). Loading the code path that calls exit(0) returns from rspec immediately with a exit status of 0
+# which shows as "passing" in CI even if tests failed.
+#
+# Here, we count the tests without running them and save them in spec/NUM_TESTS.
+# In CI, we fail the build if the number of tests in the results JSON isn't equal to this number.
+`bundle exec rspec --dry-run --format json | jq '.examples | length' > spec/NUM_TESTS`
 
 # symlink the results for the latest gemfile to the results for the default gemfile
-
 Dir['spec/**/*-rspec-3.12.gemfile.snap'].each do |path|
   Dir.chdir(File.dirname(path)) do
     source = File.basename(path)

diff --git a/bin/update_gemfiles.rb b/bin/update_gemfiles.rb
@@ -4,7 +4,6 @@
 
 GEMFILES.map do |gemfile|
   Thread.new do
-    ENV['BUNDLE_GEMFILE'] = gemfile
-    puts `bundle install`
+    system({"BUNDLE_GEMFILE" => gemfile}, "bundle update")
   end
 end.map(&:join)
diff --git a/gemfiles/rspec-3.10.gemfile b/gemfiles/rspec-3.10.gemfile
@@ -7,3 +7,4 @@ gemspec(path: "..")
 
 # and the specific patch version of rspec
 gem "rspec", "~> 3.10", "< 3.11"
+gem "rspec-snapshot", github: "rwx-research/rspec-snapshot"
diff --git a/gemfiles/rspec-3.10.gemfile.lock b/gemfiles/rspec-3.10.gemfile.lock
@@ -1,3 +1,11 @@
+GIT
+  remote: https://github.com/rwx-research/rspec-snapshot.git
+  revision: 77b3098b46190a291b32b52686d07c96cced18d5
+  specs:
+    rspec-snapshot (2.0.1)
+      awesome_print (> 1.0.0)
+      rspec (> 3.0.0)
+
 PATH
   remote: ..
   specs:
@@ -26,9 +34,6 @@ GEM
     rspec-mocks (3.10.3)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.10.0)
-    rspec-snapshot (2.0.1)
-      awesome_print (> 1.0.0)
-      rspec (> 3.0.0)
     rspec-support (3.10.3)
 
 PLATFORMS
@@ -40,7 +45,7 @@ DEPENDENCIES
   pry (~> 0.14.1)
   rspec (~> 3.10, < 3.11)
   rspec-abq!
-  rspec-snapshot (~> 2.0.1)
+  rspec-snapshot!
 
 BUNDLED WITH
    2.3.7
diff --git a/gemfiles/rspec-3.11.gemfile b/gemfiles/rspec-3.11.gemfile
@@ -7,3 +7,4 @@ gemspec(path: "..")
 
 # and the specific patch version of rspec
 gem "rspec", "~> 3.11", "< 3.12"
+gem "rspec-snapshot", github: "rwx-research/rspec-snapshot"
diff --git a/gemfiles/rspec-3.11.gemfile.lock b/gemfiles/rspec-3.11.gemfile.lock
@@ -1,3 +1,11 @@
+GIT
+  remote: https://github.com/rwx-research/rspec-snapshot.git
+  revision: 77b3098b46190a291b32b52686d07c96cced18d5
+  specs:
+    rspec-snapshot (2.0.1)
+      awesome_print (> 1.0.0)
+      rspec (> 3.0.0)
+
 PATH
   remote: ..
   specs:
@@ -26,9 +34,6 @@ GEM
     rspec-mocks (3.11.2)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.11.0)
-    rspec-snapshot (2.0.1)
-      awesome_print (> 1.0.0)
-      rspec (> 3.0.0)
     rspec-support (3.11.1)
 
 PLATFORMS
@@ -40,7 +45,7 @@ DEPENDENCIES
   pry (~> 0.14.1)
   rspec (~> 3.11, < 3.12)
   rspec-abq!
-  rspec-snapshot (~> 2.0.1)
+  rspec-snapshot!
 
 BUNDLED WITH
    2.3.7
diff --git a/gemfiles/rspec-3.12.gemfile b/gemfiles/rspec-3.12.gemfile
@@ -7,3 +7,4 @@ gemspec(path: "..")
 
 # and the specific patch version of rspec
 gem "rspec", "~> 3.12", "< 3.13"
+gem "rspec-snapshot", github: "rwx-research/rspec-snapshot"
diff --git a/gemfiles/rspec-3.12.gemfile.lock b/gemfiles/rspec-3.12.gemfile.lock
@@ -1,3 +1,11 @@
+GIT
+  remote: https://github.com/rwx-research/rspec-snapshot.git
+  revision: 77b3098b46190a291b32b52686d07c96cced18d5
+  specs:
+    rspec-snapshot (2.0.1)
+      awesome_print (> 1.0.0)
+      rspec (> 3.0.0)
+
 PATH
   remote: ..
   specs:
@@ -26,9 +34,6 @@ GEM
     rspec-mocks (3.12.0)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.12.0)
-    rspec-snapshot (2.0.1)
-      awesome_print (> 1.0.0)
-      rspec (> 3.0.0)
     rspec-support (3.12.0)
 
 PLATFORMS
@@ -40,7 +45,7 @@ DEPENDENCIES
   pry (~> 0.14.1)
   rspec (~> 3.12, < 3.13)
   rspec-abq!
-  rspec-snapshot (~> 2.0.1)
+  rspec-snapshot!
 
 BUNDLED WITH
    2.3.7
diff --git a/gemfiles/rspec-3.5.gemfile b/gemfiles/rspec-3.5.gemfile
@@ -7,3 +7,4 @@ gemspec(path: "..")
 
 # and the specific patch version of rspec
 gem "rspec", "~> 3.5", "< 3.6"
+gem "rspec-snapshot", github: "rwx-research/rspec-snapshot"
diff --git a/gemfiles/rspec-3.5.gemfile.lock b/gemfiles/rspec-3.5.gemfile.lock
@@ -1,3 +1,11 @@
+GIT
+  remote: https://github.com/rwx-research/rspec-snapshot.git
+  revision: 77b3098b46190a291b32b52686d07c96cced18d5
+  specs:
+    rspec-snapshot (2.0.1)
+      awesome_print (> 1.0.0)
+      rspec (> 3.0.0)
+
 PATH
   remote: ..
   specs:
@@ -26,9 +34,6 @@ GEM
     rspec-mocks (3.5.0)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.5.0)
-    rspec-snapshot (2.0.1)
-      awesome_print (> 1.0.0)
-      rspec (> 3.0.0)
     rspec-support (3.5.0)
 
 PLATFORMS
@@ -40,7 +45,7 @@ DEPENDENCIES
   pry (~> 0.14.1)
   rspec (~> 3.5, < 3.6)
   rspec-abq!
-  rspec-snapshot (~> 2.0.1)
+  rspec-snapshot!
 
 BUNDLED WITH
    2.3.7
diff --git a/gemfiles/rspec-3.6.gemfile b/gemfiles/rspec-3.6.gemfile
@@ -7,3 +7,4 @@ gemspec(path: "..")
 
 # and the specific patch version of rspec
 gem "rspec", "~> 3.6", "< 3.7"
+gem "rspec-snapshot", github: "rwx-research/rspec-snapshot"
Original file line number	Diff line number	Diff line change
Expand Up		@@ -7,3 +7,4 @@ gemspec(path: "..")

		# and the specific patch version of rspec
		gem "rspec", "~> 3.10", "< 3.11"
		gem "rspec-snapshot", github: "rwx-research/rspec-snapshot"