From b40ba5528f7d2d141a1ffe2bc4884c23491343b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20Cie=C5=9Blak?= Date: Sat, 19 Sep 2015 00:29:33 +0000 Subject: [PATCH] Allow testing for exitcode and stderr output If files "error" and "status" are existing in the test directory, do expect errors and test against them. Also create error and status files on --nuke. An error will be reported in this case anyway. Fixes: https://github.com/sass/sass-spec/issues/136 --- lib/sass_spec/runner.rb | 20 +++++++++++++++---- lib/sass_spec/test.rb | 39 +++++++++++++++++++++++++++++--------- lib/sass_spec/test_case.rb | 34 ++++++++++++++++++++++++++++++++- 3 files changed, 79 insertions(+), 14 deletions(-) diff --git a/lib/sass_spec/runner.rb b/lib/sass_spec/runner.rb index 13792028dd..b6b9d53196 100644 --- a/lib/sass_spec/runner.rb +++ b/lib/sass_spec/runner.rb @@ -38,14 +38,26 @@ def _get_cases glob = File.join(@options[:spec_directory], "**", "#{@options[:input_file]}") Dir.glob(glob) do |filename| input = Pathname.new(filename) + folder = File.dirname(filename) + expected_stderr_file_path = File.join(folder, "error") + expected_status_file_path = File.join(folder, "status") @options[:output_styles].each do |output_style| - folder = File.dirname(filename) output_file_name = @options["#{output_style}_output_file".to_sym] - expected_file_path = File.join(folder, output_file_name + ".css") + expected_stdout_file_path = File.join(folder, output_file_name + ".css") clean_file_name = File.join(folder, output_file_name + ".clean") - if File.file?(expected_file_path) && !File.file?(expected_file_path.sub(/\.css$/, ".skip")) && filename.include?(@options[:filter]) + if ( File.file?(expected_stdout_file_path) || + File.file?(expected_stderr_file_path) || + File.file?(expected_status_file_path) || + @options[:nuke] + ) && + !File.file?(expected_stdout_file_path.sub(/\.css$/, ".skip")) && + filename.include?(@options[:filter]) clean = File.file?(clean_file_name) - cases.push SassSpec::TestCase.new(input.realpath(), expected_file_path, output_style, clean, @options) + cases.push SassSpec::TestCase.new(input.realpath(), + expected_stdout_file_path, + expected_stderr_file_path, + expected_status_file_path, + output_style, clean, @options) end end end diff --git a/lib/sass_spec/test.rb b/lib/sass_spec/test.rb index 9dab45f7aa..bf0a1fc18b 100644 --- a/lib/sass_spec/test.rb +++ b/lib/sass_spec/test.rb @@ -6,11 +6,33 @@ def run_spec_test(test_case, options = {}) end assert File.exists?(test_case.input_path), "Input #{test_case.input_path} file does not exist" - assert File.exists?(test_case.expected_path), "Expected #{test_case.expected_path} file does not exist" output, clean_output, error, status = test_case.output - if status != 0 && !options[:unexpected_pass] + if options[:nuke] + if status != 0 + File.open(test_case.status_path, "w+") do |f| + f.write(status) + f.close + end + end + + if error.length > 0 + File.open(test_case.error_path, "w+") do |f| + f.write(error) + f.close + end + end + + File.open(test_case.expected_path, "w+") do |f| + f.write(output) + f.close + end + end + + assert File.exists?(test_case.expected_path), "Expected #{test_case.expected_path} file does not exist" + + if status != 0 && !options[:unexpected_pass] && (options[:nuke] || !test_case.should_fail?) msg = "Command `#{options[:engine_adapter]}` did not complete:\n\n#{error}" if options[:skip] @@ -26,17 +48,16 @@ def run_spec_test(test_case, options = {}) raise "#{test_case.input_path} passed a test we expected it to fail" end - if options[:nuke] - File.open(test_case.expected_path, "w+") do |f| - f.write(output) - f.close - end - end - if test_case.todo? && options[:unexpected_pass] assert test_case.expected != clean_output, "Marked as todo and passed" elsif !test_case.todo? || !options[:skip_todo] + if test_case.should_fail? + assert_equal test_case.expected_status, status, "Expected did not match status" + end assert_equal test_case.expected, clean_output, "Expected did not match output" + if test_case.verify_stderr? + assert_equal test_case.expected_error, error, "Expected did not match error" + end end end diff --git a/lib/sass_spec/test_case.rb b/lib/sass_spec/test_case.rb index d265a44b74..daf9b5a6b0 100644 --- a/lib/sass_spec/test_case.rb +++ b/lib/sass_spec/test_case.rb @@ -1,8 +1,10 @@ # This represents a specific test case. class SassSpec::TestCase - def initialize(input_scss, expected_css, style, clean, options = {}) + def initialize(input_scss, expected_css, error_file, status_file, style, clean, options = {}) @input_path = input_scss @expected_path = expected_css + @error_path = error_file + @status_path = status_file @output_style = style @clean_test = clean @options = options @@ -28,6 +30,22 @@ def expected_path @expected_path end + def error_path + @error_path + end + + def verify_stderr? + File.file?(@error_path) + end + + def status_path + @status_path + end + + def should_fail? + File.file?(@status_path) + end + def todo? @input_path.to_s.include? "todo" end @@ -36,7 +54,9 @@ def output if @output return @output end + stdout, stderr, status = engine.compile(@input_path, @output_style) + if @clean_test cleaned = _clean_output(stdout) else @@ -54,6 +74,18 @@ def expected end end + def expected_error + @expected_error = File.read(@error_path, :encoding => "utf-8") + end + + def expected_status + if should_fail? + @expected_status = File.read(@status_path, :encoding => "utf-8").to_i + else + @expected_status = 0 + end + end + def engine @options[:engine_adapter] end