Cucumber: add support for Split by Test Examples (split slow feature files by scenario)

CHANGELOG.md

@@ -2,6 +2,9 @@
 
 ### Unreleased
 
+* Cucumber: Add support for [Split by Test Examples](https://docs.knapsackpro.com/ruby/split-by-test-examples/) so slow feature files can be split by scenario (e.g., `features/a.feature:12`) across parallel CI nodes. Opt in with `KNAPSACK_PRO_CUCUMBER_SPLIT_BY_TEST_EXAMPLES=true` (requires Cucumber >= 4.0).
+* Cucumber & Queue Mode: Add an experimental preload mode (`KNAPSACK_PRO_CUCUMBER_QUEUE_PRELOAD=true`) that boots Cucumber (and the app loaded by the support code) once, then runs each batch of tests from the Queue in a forked child process instead of shelling out a brand new `cucumber` process per batch. This removes the per-batch application boot cost, which is significant for Rails apps and grows with the number of batches (e.g., when Split by Test Examples is enabled). POSIX only (requires `Process.fork`).
+
 ### 10.0.1
 
 * Add support for [File Paths Encryption](https://docs.knapsackpro.com/ruby/encryption/) to [Retry only Failures](https://docs.knapsackpro.com/ruby/retry-only-failures/).

lib/knapsack_pro.rb

@@ -58,6 +58,7 @@
 require_relative 'knapsack_pro/mask_string'
 require_relative 'knapsack_pro/test_suite'
 require_relative 'knapsack_pro/test_case_mergers/rspec_merger'
+require_relative 'knapsack_pro/test_case_mergers/cucumber_merger'
 require_relative 'knapsack_pro/build_distribution_fetcher'
 require_relative 'knapsack_pro/slow_test_file_determiner'
 require_relative 'knapsack_pro/base_allocator_builder'
@@ -76,6 +77,8 @@
 require_relative 'knapsack_pro/runners/queue/cucumber_runner'
 require_relative 'knapsack_pro/runners/queue/minitest_runner'
 require_relative 'knapsack_pro/test_case_detectors/rspec_test_example_detector'
+require_relative 'knapsack_pro/test_case_detectors/cucumber_test_example_detector'
+require_relative 'knapsack_pro/cucumber/runtime_preloader'
 require_relative 'knapsack_pro/crypto/encryptor'
 require_relative 'knapsack_pro/crypto/branch_encryptor'
 require_relative 'knapsack_pro/crypto/decryptor'

lib/knapsack_pro/adapters/cucumber_adapter.rb

@@ -4,6 +4,75 @@ module KnapsackPro
   module Adapters
     class CucumberAdapter < BaseAdapter
       TEST_DIR_PATTERN = 'features/**{,/*/**}/*.feature'
+      # Matches a test example path like features/a.feature:12
+      REGEX = /\A(.*?)(?::(\d+))?\z/.freeze
+
+      def self.split_by_test_cases_enabled?
+        return false unless KnapsackPro::Config::Env.cucumber_split_by_test_examples?
+
+        require 'cucumber/platform'
+        unless ::Cucumber::VERSION.to_i >= 4
+          raise "Cucumber >= 4.0 is required to split test files by test examples. Learn more: #{KnapsackPro::Urls::SPLIT_BY_TEST_EXAMPLES}"
+        end
+
+        true
+      end
+
+      def self.calculate_slow_id_paths
+        # Shell out not to pollute the Cucumber state
+        cmd = [
+          'RACK_ENV=test',
+          'RAILS_ENV=test',
+          KnapsackPro::Config::Env.cucumber_test_example_detector_prefix,
+          'rake knapsack_pro:cucumber_test_example_detector',
+        ].join(' ')
+        raise "Failed to calculate Split by Test Examples: #{cmd}" unless Kernel.system(cmd)
+
+        KnapsackPro::TestCaseDetectors::CucumberTestExampleDetector.new.slow_id_paths!
+      end
+
+      def self.parse_file_path(path)
+        file, _id = path.match(REGEX).captures
+        file
+      end
+
+      def self.id_path?(path)
+        _file, id = path.match(REGEX).captures
+        !id.nil?
+      end
+
+      def self.concat_test_files(test_files, id_paths)
+        paths = concat_paths(test_files, id_paths)
+        KnapsackPro::TestFilePresenter.test_files(paths)
+      end
+
+      def self.concat_paths(test_files, id_paths)
+        paths = KnapsackPro::TestFilePresenter.paths(test_files)
+        file_paths = id_paths.map { |id_path| parse_file_path(id_path) }
+        paths + id_paths - file_paths
+      end
+
+      def self.remove_formatters(cli_args)
+        formatter_options = ['-f', '--format', '-o', '--out']
+        cli_args.dup.each_with_index do |arg, index|
+          if formatter_options.include?(arg)
+            cli_args[index] = nil
+            cli_args[index + 1] = nil
+          end
+        end
+        cli_args.compact
+      end
+
+      # When the test file was split by test examples, track the time execution
+      # per test example path (e.g., features/a.feature:12).
+      # Otherwise, track the time execution per test file path.
+      def self.tracked_test_path(object)
+        file = test_path(object)
+        return file unless split_by_test_cases_enabled?
+
+        id_path = "#{file}:#{object.location.line}"
+        KnapsackPro.tracker.scheduled_test_path?(id_path) ? id_path : file
+      end
 
       def self.test_path(object)
         if ::Cucumber::VERSION.to_i >= 2
@@ -32,7 +101,7 @@ def self.test_path(object)
 
       def bind_time_tracker
         Around do |object, block|
-          KnapsackPro.tracker.current_test_path = KnapsackPro::Adapters::CucumberAdapter.test_path(object)
+          KnapsackPro.tracker.current_test_path = KnapsackPro::Adapters::CucumberAdapter.tracked_test_path(object)
           KnapsackPro.tracker.start_timer
           block.call
           KnapsackPro.tracker.stop_timer
@@ -67,11 +136,21 @@ def bind_before_queue_hook
 
       def bind_after_queue_hook
         ::Kernel.at_exit do
-          KnapsackPro::Hooks::Queue.call_after_subset_queue
-          KnapsackPro::Report.save_subset_queue_to_file
+          # In the preload mode ( KNAPSACK_PRO_CUCUMBER_QUEUE_PRELOAD ) the parent
+          # process loads the support code but never executes tests; only the forked
+          # child processes that ran a batch of tests should save a report.
+          unless KnapsackPro::Adapters::CucumberAdapter.preload_parent_process?
+            KnapsackPro::Hooks::Queue.call_after_subset_queue
+            KnapsackPro::Report.save_subset_queue_to_file
+          end
         end
       end
 
+      def self.preload_parent_process?
+        !!ENV['KNAPSACK_PRO_CUCUMBER_PRELOAD_PARENT_PID'] &&
+          ENV['KNAPSACK_PRO_CUCUMBER_PRELOAD_PARENT_PID'] == Process.pid.to_s
+      end
+
       private
 
       def Around(*tag_expressions, &proc)

lib/knapsack_pro/config/env.rb

@@ -202,6 +202,32 @@ def rspec_test_example_detector_prefix
           ENV.fetch('KNAPSACK_PRO_RSPEC_TEST_EXAMPLE_DETECTOR_PREFIX', 'bundle exec')
         end
 
+        def cucumber_split_by_test_examples?
+          return @cucumber_split_by_test_examples if defined?(@cucumber_split_by_test_examples)
+
+          split = ENV['KNAPSACK_PRO_CUCUMBER_SPLIT_BY_TEST_EXAMPLES'].to_s == 'true'
+
+          if split && test_files_encrypted?
+            KnapsackPro.logger.warn("Skipping split by test examples because test file names encryption is enabled:\n#{KnapsackPro::Urls::ENCRYPTION}\n#{KnapsackPro::Urls::SPLIT_BY_TEST_EXAMPLES}")
+            return (@cucumber_split_by_test_examples = false)
+          end
+
+          if split && ci_node_total < 2
+            KnapsackPro.logger.debug('Skipping split by test examples because tests are running on a single CI node (no parallelism)')
+            @cucumber_split_by_test_examples = false
+          else
+            @cucumber_split_by_test_examples = split
+          end
+        end
+
+        def cucumber_test_example_detector_prefix
+          ENV.fetch('KNAPSACK_PRO_CUCUMBER_TEST_EXAMPLE_DETECTOR_PREFIX', 'bundle exec')
+        end
+
+        def cucumber_queue_preload_enabled?
+          ENV['KNAPSACK_PRO_CUCUMBER_QUEUE_PRELOAD'] == 'true'
+        end
+
         def test_suite_token
           env_name = 'KNAPSACK_PRO_TEST_SUITE_TOKEN'
           ENV[env_name] || raise("Missing environment variable #{env_name}. You should set environment variable like #{env_name}_RSPEC (note there is suffix _RSPEC at the end). knapsack_pro gem will set #{env_name} based on #{env_name}_RSPEC value. If you use other test runner than RSpec then use proper suffix.")

lib/knapsack_pro/cucumber/runtime_preloader.rb

@@ -0,0 +1,163 @@
+# frozen_string_literal: true
+
+require 'shellwords'
+
+module KnapsackPro
+  module Cucumber
+    class RuntimePreloader
+      # Cucumber::Runtime memoizes these based on the configuration it was
+      # first run with. When the runtime is reused for another run via
+      # Cucumber::Cli::Main#execute!(existing_runtime), they must be reset so
+      # that reports, formatters, and parsed feature files are rebuilt against
+      # the new configuration and its event bus. Without this, test results
+      # are published to the previous run's event bus: formatters print
+      # nothing and the process exits 0 even when scenarios fail.
+      RUNTIME_MEMOIZED_IVARS = %i[
+        @features
+        @filespecs
+        @report
+        @summary_report
+        @fail_fast_report
+        @publish_banner_printer
+        @formatters
+      ].freeze
+
+      # Boots Cucumber once (a dry run loads the support code, e.g.
+      # features/support/env.rb which usually boots a Rails app) and returns
+      # the runtime so batches can be executed in forked child processes
+      # without paying the boot cost again.
+      #
+      # sample_test_file_path limits the dry run to a single test file: the
+      # dry run only exists to load the support code, and dry-running a whole
+      # test suite can take minutes (e.g., step matching on every step).
+      def self.preload(test_dir, args, sample_test_file_path)
+        unless Process.respond_to?(:fork)
+          raise "KNAPSACK_PRO_CUCUMBER_QUEUE_PRELOAD requires an operating system that supports Process.fork (POSIX). Please disable the preload mode."
+        end
+
+        require 'cucumber'
+
+        # The parent process must not run the adapter's at_exit hooks
+        # (saving a subset queue report to disk / calling queue hooks);
+        # only the forked children that actually execute tests should.
+        # See lib/knapsack_pro/adapters/cucumber_adapter.rb
+        ENV['KNAPSACK_PRO_CUCUMBER_PRELOAD_PARENT_PID'] = Process.pid.to_s
+
+        KnapsackPro.logger.info('Preloading Cucumber (loading support code once; batches will run in forked processes).')
+
+        cli_args_without_formatters = KnapsackPro::Adapters::CucumberAdapter.remove_formatters(Shellwords.split(args || ''))
+        # The preload dry run only exists to load the support code. Profiles from
+        # cucumber.yml can register formatters, and a formatter that touches
+        # Capybara would boot the Capybara server (and a browser) in this parent
+        # process; forked children would then reuse the parent's server via
+        # Capybara's inherited session/port registries and run their requests in
+        # the wrong process.
+        cli_args = remove_profiles(cli_args_without_formatters) + [
+          '--no-profile',
+          '--dry-run',
+          # A unique --out path: pointing two formatters at the same stream is an
+          # error in Cucumber, and a profile from cucumber.yml may already use
+          # --out /dev/null (profiles are expanded before the stream conflict check).
+          '--format', 'progress',
+          '--out', dry_run_progress_report_path,
+          '--require', test_dir,
+          sample_test_file_path,
+        ]
+
+        # Cucumber's option parser mutates the args array, so build the debug command upfront.
+        command = (['bundle exec cucumber'] + cli_args).join(' ')
+
+        runtime = ::Cucumber::Runtime.new
+        exit_code =
+          begin
+            ::Cucumber::Cli::Main.new(cli_args).execute!(runtime)
+            0
+          rescue SystemExit => e
+            e.status
+          end
+
+        unless exit_code.zero?
+          raise "Failed to preload Cucumber (exit code #{exit_code}). To reproduce the problem, run: #{command}"
+        end
+
+        disconnect_active_record_in_parent!
+
+        runtime
+      end
+
+      # Database connections opened while booting the app must not be shared
+      # with the forked children (concurrent use of an inherited socket
+      # corrupts the connection protocol and hangs or fails queries). Clearing
+      # the pools here means both the parent and each forked child lazily
+      # check out fresh connections when they first need one.
+      def self.disconnect_active_record_in_parent!
+        return unless defined?(::ActiveRecord::Base)
+
+        handler = ::ActiveRecord::Base.connection_handler
+
+        # Connection leases are keyed by thread identity, which survives a fork.
+        # Without releasing the lease acquired while booting the app, a forked
+        # child's first `pool.checkout`/`pin_connection!` returns the parent's
+        # connection object and reconnects it in place (undefined behavior on a
+        # forked libpq handle; observed as a segfault in the pg gem).
+        if handler.respond_to?(:each_connection_pool)
+          handler.each_connection_pool do |pool|
+            pool.release_connection if pool.respond_to?(:release_connection)
+          end
+        end
+
+        handler.clear_all_connections!
+      rescue StandardError => e
+        KnapsackPro.logger.warn("Could not clear ActiveRecord connections after preloading Cucumber: #{e.class}: #{e.message}")
+      end
+
+      def self.dry_run_progress_report_path
+        KnapsackPro::Config::TempFiles.ensure_temp_directory_exists!
+        dir = "#{KnapsackPro::Config::TempFiles::TEMP_DIRECTORY_PATH}/cucumber_queue_preload"
+        FileUtils.mkdir_p(dir)
+        "#{dir}/dry_run_progress_node_#{KnapsackPro::Config::Env.ci_node_index}.txt"
+      end
+
+      def self.remove_profiles(cli_args)
+        profile_options = ['-p', '--profile']
+        cli_args.dup.each_with_index do |arg, index|
+          if profile_options.include?(arg)
+            cli_args[index] = nil
+            cli_args[index + 1] = nil
+          elsif arg == '--no-profile' || arg == '-P'
+            cli_args[index] = nil
+          end
+        end
+        cli_args.compact
+      end
+
+      # Called in the forked child process before running a batch of tests.
+      def self.reset_forked_child_state(runtime)
+        reset_runtime_memoization(runtime)
+        reset_capybara!
+      end
+
+      def self.reset_runtime_memoization(runtime)
+        RUNTIME_MEMOIZED_IVARS.each do |ivar|
+          runtime.remove_instance_variable(ivar) if runtime.instance_variable_defined?(ivar)
+        end
+      end
+
+      # Capybara's session registry and app->port registry are inherited by the
+      # fork. If a Capybara server was booted in the parent process, a child
+      # reusing these registries would find the parent's server responsive and
+      # send its requests to the wrong process (which cannot see, e.g., the
+      # child's open database transactions). Clearing them makes each child
+      # boot its own server on first use.
+      def self.reset_capybara!
+        return unless defined?(::Capybara)
+
+        ::Capybara.send(:session_pool).clear if ::Capybara.respond_to?(:session_pool, true)
+
+        if defined?(::Capybara::Server) && ::Capybara::Server.respond_to?(:ports)
+          ::Capybara::Server.ports.clear
+        end
+      end
+    end
+  end
+end

lib/knapsack_pro/hooks/queue.rb

@@ -7,7 +7,8 @@ class << self
         attr_reader :before_queue_store,
           :before_subset_queue_store,
           :after_subset_queue_store,
-          :after_queue_store
+          :after_queue_store,
+          :after_preload_fork_store
 
         def reset_before_queue
           @before_queue_store = nil
@@ -25,6 +26,10 @@ def reset_after_queue
           @after_queue_store = nil
         end
 
+        def reset_after_preload_fork
+          @after_preload_fork_store = nil
+        end
+
         def before_queue(&block)
           @before_queue_store ||= []
           @before_queue_store << block
@@ -45,6 +50,20 @@ def after_queue(&block)
           @after_queue_store << block
         end
 
+        # Called in a forked child process right after the fork when the Cucumber
+        # preload mode ( KNAPSACK_PRO_CUCUMBER_QUEUE_PRELOAD ) is enabled.
+        # Use it to re-establish resources that do not survive a fork,
+        # e.g., reopen log appenders or reconnect clients holding sockets.
+        def after_preload_fork(&block)
+          @after_preload_fork_store ||= []
+          @after_preload_fork_store << block
+        end
+
+        def call_after_preload_fork
+          return unless after_preload_fork_store
+          after_preload_fork_store.each(&:call)
+        end
+
         def call_before_queue
           return unless before_queue_store
           before_queue_store.each do |block|

lib/knapsack_pro/runners/cucumber_runner.rb

@@ -6,6 +6,7 @@ class CucumberRunner < BaseRunner
       def self.run(args)
         ENV['KNAPSACK_PRO_TEST_SUITE_TOKEN'] = KnapsackPro::Config::Env.test_suite_token_cucumber
         ENV['KNAPSACK_PRO_REGULAR_MODE_ENABLED'] = 'true'
+        ENV['KNAPSACK_PRO_CUCUMBER_OPTIONS'] = args.to_s
 
         adapter_class = KnapsackPro::Adapters::CucumberAdapter
         KnapsackPro::Config::Env.set_test_runner_adapter(adapter_class)