class Tryouts::TestBatch
Modern TestBatch using Ruby 3.4+ patterns and formatter system
def aggregate_evaluation_results(evaluation_results)
def aggregate_evaluation_results(evaluation_results) { passed: evaluation_results.all? { |r| r[:passed] }, actual_results: evaluation_results.map { |r| r[:actual] }, expected_results: evaluation_results.map { |r| r[:expected] }, } end
def build_error_result(test_case, exception)
def build_error_result(test_case, exception) TestCaseResultPacket.from_error(test_case, exception) end
def build_test_result(test_case, result_value, expectations_result)
def build_test_result(test_case, result_value, expectations_result) if expectations_result[:passed] TestCaseResultPacket.from_success( test_case, result_value, expectations_result[:actual_results], expectations_result[:expected_results], ) else TestCaseResultPacket.from_failure( test_case, result_value, expectations_result[:actual_results], expectations_result[:expected_results], ) end end
def capture_output
def capture_output old_stdout = $stdout old_stderr = $stderr $stdout = StringIO.new $stderr = StringIO.new yield captured = $stdout.string + $stderr.string captured.empty? ? nil : captured ensure $stdout = old_stdout $stderr = old_stderr end
def completed?
def completed? @status == :completed end
def empty?
def empty? @testrun.empty? end
def evaluate_expectations(test_case, actual_result, context, execution_time_ns = nil, stdout_content = nil, stderr_content = nil)
def evaluate_expectations(test_case, actual_result, context, execution_time_ns = nil, stdout_content = nil, stderr_content = nil) return { passed: true, actual_results: [], expected_results: [] } if test_case.expectations.empty? evaluation_results = test_case.expectations.map do |expectation| evaluator = ExpectationEvaluators::Registry.evaluator_for(expectation, test_case, context) # Pass appropriate data to different evaluator types if expectation.performance_time? && execution_time_ns evaluator.evaluate(actual_result, execution_time_ns) elsif expectation.output? && (stdout_content || stderr_content) evaluator.evaluate(actual_result, stdout_content, stderr_content) else evaluator.evaluate(actual_result) end end aggregate_evaluation_results(evaluation_results) end
def execute_fresh_context_setup
def execute_fresh_context_setup setup = @testrun.setup if setup && !setup.code.empty? && !@options[:shared_context] @output_manager&.setup_start(setup.line_range) # Create setup container to hold @instance_variables @setup_container = Object.new # Capture setup output instead of letting it print directly captured_output = capture_output do @setup_container.instance_eval(setup.code, setup.path, setup.line_range.first + 1) end @output_manager&.setup_output(captured_output) if captured_output && !captured_output.empty? end rescue StandardError => ex @setup_failed = true @global_tally[:total_errors] += 1 if @global_tally # Classify error and handle appropriately error_type = Tryouts.classify_error(ex) Tryouts.debug "Setup failed with #{error_type} error: (#{ex.class}): #{ex.message}" Tryouts.trace ex.backtrace # For non-catastrophic errors, we still stop batch execution unless Tryouts.batch_stopping_error?(ex) @output_manager&.error("Fresh context setup failed: #{ex.message}") return end # For catastrophic errors, still raise to stop execution raise "Fresh context setup failed (#{ex.class}): #{ex.message}" end
def execute_global_setup
def execute_global_setup setup = @testrun.setup if setup && !setup.code.empty? && @options[:shared_context] @output_manager&.setup_start(setup.line_range) # Capture setup output instead of letting it print directly captured_output = capture_output do @container.instance_eval(setup.code, setup.path, setup.line_range.first + 1) end @output_manager&.setup_output(captured_output) if captured_output && !captured_output.empty? end rescue StandardError => ex @setup_failed = true @global_tally[:total_errors] += 1 if @global_tally # Classify error and handle appropriately error_type = Tryouts.classify_error(ex) Tryouts.debug "Setup failed with #{error_type} error: (#{ex.class}): #{ex.message}" Tryouts.trace ex.backtrace # For non-catastrophic errors, we still stop batch execution unless Tryouts.batch_stopping_error?(ex) @output_manager&.error("Global setup failed: #{ex.message}") return end # For catastrophic errors, still raise to stop execution raise "Global setup failed (#{ex.class}): #{ex.message}" end
def execute_global_teardown
def execute_global_teardown teardown = @testrun.teardown if teardown && !teardown.code.empty? @output_manager&.teardown_start(teardown.line_range) # Capture teardown output instead of letting it print directly captured_output = capture_output do @container.instance_eval(teardown.code, teardown.path, teardown.line_range.first + 1) end @output_manager&.teardown_output(captured_output) if captured_output && !captured_output.empty? end rescue StandardError => ex @global_tally[:total_errors] += 1 if @global_tally # Classify error and handle appropriately error_type = Tryouts.classify_error(ex) Tryouts.debug "Teardown failed with #{error_type} error: (#{ex.class}): #{ex.message}" Tryouts.trace ex.backtrace @output_manager&.error("Teardown failed: #{ex.message}") # Teardown failures are generally non-fatal - log and continue if Tryouts.batch_stopping_error?(ex) # Only catastrophic errors should potentially affect batch completion @output_manager&.error('Teardown failure may affect subsequent operations') else @output_manager&.error('Continuing despite teardown failure') end end
def execute_single_test(test_case, before_test_hook = nil)
def execute_single_test(test_case, before_test_hook = nil) before_test_hook&.call(test_case) # Capture output during test execution result = nil captured_output = capture_output do result = case @options[:shared_context] when true execute_with_shared_context(test_case) when false, nil execute_with_fresh_context(test_case) else raise 'Invalid execution context configuration' end end # Add captured output to the result if any exists if captured_output && !captured_output.empty? # Create new result packet with captured output result = result.class.new( test_case: result.test_case, status: result.status, result_value: result.result_value, actual_results: result.actual_results, expected_results: result.expected_results, error: result.error, captured_output: captured_output, elapsed_time: result.elapsed_time, metadata: result.metadata, ) end process_test_result(result) yield(test_case) if block_given? result end
def execute_test_case_with_container(test_case, container)
def execute_test_case_with_container(test_case, container) # Individual test timeout protection test_timeout = @options[:test_timeout] || 30 # 30 second default if test_case.exception_expectations? # For exception tests, don't execute code here - let evaluate_expectations handle it expectations_result = execute_with_timeout(test_timeout, test_case) do evaluate_expectations(test_case, nil, container) end build_test_result(test_case, nil, expectations_result) else # Regular execution for non-exception tests with timing and output capture code = test_case.code path = test_case.path range = test_case.line_range # Check if we need output capture for any expectations needs_output_capture = test_case.expectations.any?(&:output?) result_value, _, _, _, expectations_result = execute_with_timeout(test_timeout, test_case) do if needs_output_capture # Execute with output capture using Fiber-local isolation result_value, execution_time_ns, stdout_content, stderr_content = execute_with_output_capture(container, code, path, range) expectations_result = evaluate_expectations( test_case, result_value, container, execution_time_ns, stdout_content, stderr_content ) [result_value, execution_time_ns, stdout_content, stderr_content, expectations_result] else # Regular execution with timing capture only execution_start_ns = Process.clock_gettime(Process::CLOCK_MONOTONIC, :nanosecond) result_value = container.instance_eval(code, path, range.first + 1) execution_end_ns = Process.clock_gettime(Process::CLOCK_MONOTONIC, :nanosecond) execution_time_ns = execution_end_ns - execution_start_ns expectations_result = evaluate_expectations(test_case, result_value, container, execution_time_ns) [result_value, execution_time_ns, nil, nil, expectations_result] end end build_test_result(test_case, result_value, expectations_result) end rescue StandardError => ex build_error_result(test_case, ex) rescue SystemExit, SignalException => ex # Handle process control exceptions gracefully Tryouts.debug "Test received #{ex.class}: #{ex.message}" build_error_result(test_case, StandardError.new("Test terminated by #{ex.class}: #{ex.message}")) end
def execute_with_fresh_context(test_case)
def execute_with_fresh_context(test_case) fresh_container = if @shared_context.is_a?(FreshContextFactory) @shared_context.create_container else Object.new # Fallback for backwards compatibility end # Copy @instance_variables from setup container to fresh container if @setup_container @setup_container.instance_variables.each do |var| value = @setup_container.instance_variable_get(var) fresh_container.instance_variable_set(var, value) end end execute_test_case_with_container(test_case, fresh_container) end
def execute_with_output_capture(container, code, path, range)
def execute_with_output_capture(container, code, path, range) # Fiber-local storage for output redirection original_stdout = $stdout original_stderr = $stderr # Create StringIO objects for capturing output captured_stdout = StringIO.new captured_stderr = StringIO.new begin # Redirect output streams using Fiber-local variables Fiber.new do $stdout = captured_stdout $stderr = captured_stderr # Execute with timing capture execution_start_ns = Process.clock_gettime(Process::CLOCK_MONOTONIC, :nanosecond) result_value = container.instance_eval(code, path, range.first + 1) execution_end_ns = Process.clock_gettime(Process::CLOCK_MONOTONIC, :nanosecond) execution_time_ns = execution_end_ns - execution_start_ns [result_value, execution_time_ns] end.resume.tap do |result_value, execution_time_ns| # Return captured content along with result return [result_value, execution_time_ns, captured_stdout.string, captured_stderr.string] end ensure # Always restore original streams $stdout = original_stdout $stderr = original_stderr end end
def execute_with_shared_context(test_case)
def execute_with_shared_context(test_case) execute_test_case_with_container(test_case, @container) end
def execute_with_timeout(timeout_seconds, test_case, &)
def execute_with_timeout(timeout_seconds, test_case, &) Timeout.timeout(timeout_seconds, &) rescue Timeout::Error Tryouts.debug "Test timeout after #{timeout_seconds}s: #{test_case.description}" raise StandardError.new("Test execution timeout (#{timeout_seconds}s)") end
def failed?
def failed? @failed_count > 0 end
def finalize_results(_execution_results)
def finalize_results(_execution_results) @status = :completed elapsed_time = Time.now - @start_time show_summary(elapsed_time) end
def handle_batch_error(exception)
def handle_batch_error(exception) @status = :error @failed_count = 1 error_message = "Batch execution failed: #{exception.message}" backtrace = exception.respond_to?(:backtrace) ? exception.backtrace : nil @output_manager&.error(error_message, backtrace) end
def initialize(testrun, **options)
def initialize(testrun, **options) @testrun = testrun @container = Object.new @options = options @formatter = Tryouts::CLI::FormatterFactory.create_formatter(options) @output_manager = options[:output_manager] @global_tally = options[:global_tally] @failed_count = 0 @status = :pending @results = [] @start_time = nil @test_case_count = 0 @setup_failed = false # Setup container for fresh context mode - preserves @instance_variables from setup @setup_container = nil # Circuit breaker for batch-level failure protection @consecutive_failures = 0 @max_consecutive_failures = options[:max_consecutive_failures] || 10 @circuit_breaker_active = false # Expose context objects for testing - different strategies for each mode @shared_context = if options[:shared_context] @container # Shared mode: single container reused across tests else FreshContextFactory.new # Fresh mode: factory that creates new containers end end
def path
def path @testrun.source_file end
def process_test_result(result)
def process_test_result(result) @results << result if result.failed? || result.error? @failed_count += 1 # Collect failure details for end-of-run summary if @global_tally && @global_tally[:failure_collector] @global_tally[:failure_collector].add_failure(@testrun.source_file, result) end end show_test_result(result) # Show captured output if any exists if result.has_output? @output_manager&.test_output(result.test_case, result.captured_output, result) end end
def run(before_test_hook = nil, &)
def run(before_test_hook = nil, &) return false if empty? @start_time = Time.now @test_case_count = test_cases.size @output_manager&.execution_phase(@test_case_count) @output_manager&.info("Context: #{@options[:shared_context] ? 'shared' : 'fresh'}", 1) @output_manager&.file_start(path, context: @options[:shared_context] ? :shared : :fresh) if shared_context? @output_manager&.info('Running global setup...', 2) execute_global_setup # Stop execution if setup failed if @setup_failed @output_manager&.error('Stopping batch execution due to setup failure') @status = :failed finalize_results([]) return false end else # Fresh context mode: execute setup once to establish shared @instance_variables @output_manager&.info('Running setup for fresh context...', 2) execute_fresh_context_setup # Stop execution if setup failed if @setup_failed @output_manager&.error('Stopping batch execution due to setup failure') @status = :failed finalize_results([]) return false end end idx = 0 execution_results = test_cases.map do |test_case| @output_manager&.trace("Test #{idx + 1}/#{@test_case_count}: #{test_case.description}", 2) idx += 1 # Check circuit breaker before executing test if @circuit_breaker_active @output_manager&.error("Circuit breaker active - skipping remaining tests after #{@consecutive_failures} consecutive failures") break end @output_manager&.test_start(test_case, idx, @test_case_count) result = execute_single_test(test_case, before_test_hook, &) # runs the test code @output_manager&.test_end(test_case, idx, @test_case_count) # Update circuit breaker state based on result update_circuit_breaker(result) result rescue StandardError => ex @output_manager&.test_end(test_case, idx, @test_case_count) # Create error result packet to maintain consistent data flow error_result = build_error_result(test_case, ex) process_test_result(error_result) # Update circuit breaker for exception cases update_circuit_breaker(error_result) error_result end # Used for a separate purpose then execution_phase. # e.g. the quiet formatter prints a newline after all test dots @output_manager&.file_end(path, context: @options[:shared_context] ? :shared : :fresh) @output_manager&.execution_phase(test_cases.size) execute_global_teardown finalize_results(execution_results) @status = :completed !failed? end
def shared_context?
def shared_context? @options[:shared_context] == true end
def show_summary(elapsed_time)
def show_summary(elapsed_time) # Summary is now handled by TestRunner with failure details # This method kept for compatibility but no longer calls batch_summary end
def show_test_result(result)
def show_test_result(result) @output_manager&.test_result(result) end
def size
def size @testrun.total_tests end
def test_cases
def test_cases @testrun.test_cases end
def update_circuit_breaker(result)
def update_circuit_breaker(result) if result.failed? || result.error? @consecutive_failures += 1 if @consecutive_failures >= @max_consecutive_failures @circuit_breaker_active = true Tryouts.debug "Circuit breaker activated after #{@consecutive_failures} consecutive failures" end else # Reset on success @consecutive_failures = 0 @circuit_breaker_active = false end end