Retry spinach tests

af434016 · Kamil Trzcinski · 640bf8ab · af434016 · af434016 · af434016
--- a/Gemfile
+++ b/Gemfile
@@ -265,6 +265,7 @@ group :development, :test do
  gem 'rspec-rails',        '~> 3.3.0'
  gem 'rspec-retry'
  gem 'spinach-rails',      '~> 0.2.1'
+  gem 'spinach-rerun-reporter', '~> 0.0.2'

  # Prevent occasions where minitest is not bundled in packaged versions of ruby (see #3826)
  gem 'minitest', '~> 5.7.0'

--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -766,6 +766,8 @@ GEM
      capybara (>= 2.0.0)
      railties (>= 3)
      spinach (>= 0.4)
+    spinach-rerun-reporter (0.0.2)
+      spinach (~> 0.8)
    spring (1.6.4)
    spring-commands-rspec (1.0.4)
      spring (>= 0.9.1)
@@ -1020,6 +1022,7 @@ DEPENDENCIES
  six (~> 0.2.0)
  slack-notifier (~> 1.2.0)
  spinach-rails (~> 0.2.1)
+  spinach-rerun-reporter (~> 0.0.2)
  spring (~> 1.6.4)
  spring-commands-rspec (~> 1.0.4)
  spring-commands-spinach (~> 1.0.0)

--- a/features/support/env.rb
+++ b/features/support/env.rb
@@ -14,6 +14,7 @@

 require_relative 'capybara'
 require_relative 'db_cleaner'
+require_relative 'rerun'

 %w(select2_helper test_env repo_helpers).each do |f|
  require Rails.root.join('spec', 'support', f)

--- a/features/support/rerun.rb
+++ b/features/support/rerun.rb
+# The spinach-rerun-reporter doesn't define the on_undefined_step
+# See it here: https://github.com/javierav/spinach-rerun-reporter/blob/master/lib/spinach/reporter/rerun.rb
+module Spinach
+  class Reporter
+    class Rerun
+      def on_undefined_step(step_data, failure, step_definitions = nil)
+        super step_data, failure, step_definitions
+
+        # save feature file and scenario line
+        @rerun << "#{current_feature.filename}:#{current_scenario.line}"
+      end
+    end
+  end
+end
--- a/lib/tasks/spinach.rake
+++ b/lib/tasks/spinach.rake
@@ -4,53 +4,59 @@ namespace :spinach do
  namespace :project do
    desc "GitLab | Spinach | Run project commits, issues and merge requests spinach features"
    task :half do
-      cmds = [
-        %W(rake gitlab:setup),
-        %W(spinach --tags @project_commits,@project_issues,@project_merge_requests),
-      ]
-      run_commands(cmds)
+      run_spinach_tests('@project_commits,@project_issues,@project_merge_requests')
    end

    desc "GitLab | Spinach | Run remaining project spinach features"
    task :rest do
-      cmds = [
-        %W(rake gitlab:setup),
-        %W(spinach --tags ~@admin,~@dashboard,~@profile,~@public,~@snippets,~@project_commits,~@project_issues,~@project_merge_requests),
-      ]
-      run_commands(cmds)
+      run_spinach_tests('~@admin,~@dashboard,~@profile,~@public,~@snippets,~@project_commits,~@project_issues,~@project_merge_requests')
    end
  end

  desc "GitLab | Spinach | Run project spinach features"
  task :project do
-    cmds = [
-      %W(rake gitlab:setup),
-      %W(spinach --tags ~@admin,~@dashboard,~@profile,~@public,~@snippets),
-    ]
-    run_commands(cmds)
+    run_spinach_tests('~@admin,~@dashboard,~@profile,~@public,~@snippets')
  end

  desc "GitLab | Spinach | Run other spinach features"
  task :other do
-    cmds = [
-      %W(rake gitlab:setup),
-      %W(spinach --tags @admin,@dashboard,@profile,@public,@snippets),
-    ]
-    run_commands(cmds)
+    run_spinach_tests('@admin,@dashboard,@profile,@public,@snippets')
+  end
+
+  desc "GitLab | Spinach | Run other spinach features"
+  task :builds do
+    run_spinach_tests('@builds')
  end
 end

 desc "GitLab | Run spinach"
 task :spinach do
-  cmds = [
-    %W(rake gitlab:setup),
-    %W(spinach),
-  ]
-  run_commands(cmds)
+  run_spinach_tests(nil)
+end
+
+def run_command(cmd)
+  system({'RAILS_ENV' => 'test', 'force' => 'yes'}, *cmd)
 end

-def run_commands(cmds)
-  cmds.each do |cmd|
-    system({'RAILS_ENV' => 'test', 'force' => 'yes'}, *cmd) or raise("#{cmd} failed!")
+def run_spinach_command(args)
+  run_command(%w(spinach -r rerun) + args)
+end
+
+def run_spinach_tests(tags)
+  #run_command(%w(rake gitlab:setup)) or raise('gitlab:setup failed!')
+
+  success = run_spinach_command(%W(--tags #{tags}))
+  3.times do |_|
+    break if success
+    break unless File.exists?('tmp/spinach-rerun.txt')
+
+    tests = File.foreach('tmp/spinach-rerun.txt').map(&:chomp)
+    puts ''
+    puts "Spinach tests for #{tags}: Retrying tests... #{tests}".red
+    puts ''
+    sleep(3)
+    success = run_spinach_command(tests)
  end
+
+  raise("spinach tests for #{tags} failed!") unless success
 end