Allow testing for exitcode and stderr output

If files "error" and "status" are existing
in the test directory, do expect errors
and test against them.

Also create error and status files on --nuke.
An error will be reported in this case anyway.

Fixes: sass#136

lib/sass_spec/runner.rb

@@ -38,14 +38,26 @@ def _get_cases
     glob = File.join(@options[:spec_directory], "**", "#{@options[:input_file]}")
     Dir.glob(glob) do |filename|
       input = Pathname.new(filename)
+      folder = File.dirname(filename)
+      expected_stderr_file_path = File.join(folder, "error")
+      expected_status_file_path = File.join(folder, "status")
       @options[:output_styles].each do |output_style|
-        folder = File.dirname(filename)
         output_file_name = @options["#{output_style}_output_file".to_sym]
-        expected_file_path = File.join(folder, output_file_name + ".css")
+        expected_stdout_file_path = File.join(folder, output_file_name + ".css")
         clean_file_name = File.join(folder, output_file_name + ".clean")
-        if File.file?(expected_file_path) && !File.file?(expected_file_path.sub(/\.css$/, ".skip")) && filename.include?(@options[:filter])
+        if ( File.file?(expected_stdout_file_path) || 
+             File.file?(expected_stderr_file_path) ||
+             File.file?(expected_status_file_path) ||
+             @options[:nuke]
+           ) &&
+           !File.file?(expected_stdout_file_path.sub(/\.css$/, ".skip")) &&
+           filename.include?(@options[:filter]) 
           clean = File.file?(clean_file_name)
-          cases.push SassSpec::TestCase.new(input.realpath(), expected_file_path, output_style, clean, @options)
+          cases.push SassSpec::TestCase.new(input.realpath(),
+            expected_stdout_file_path,
+            expected_stderr_file_path,
+            expected_status_file_path,
+            output_style, clean, @options)
         end
       end
     end

lib/sass_spec/test.rb

@@ -6,11 +6,33 @@ def run_spec_test(test_case, options = {})
   end
 
   assert File.exists?(test_case.input_path), "Input #{test_case.input_path} file does not exist"
-  assert File.exists?(test_case.expected_path), "Expected #{test_case.expected_path} file does not exist"
 
   output, clean_output, error, status = test_case.output
 
-  if status != 0 && !options[:unexpected_pass]
+  if options[:nuke]
+    if status != 0
+      File.open(test_case.status_path, "w+") do |f|
+        f.write(status)
+        f.close
+      end
+    end
+
+    if error.length > 0
+      File.open(test_case.error_path, "w+") do |f|
+        f.write(error)
+        f.close
+      end
+    end
+
+    File.open(test_case.expected_path, "w+") do |f|
+      f.write(output)
+      f.close
+    end
+  end
+
+  assert File.exists?(test_case.expected_path), "Expected #{test_case.expected_path} file does not exist"
+
+  if status != 0 && !options[:unexpected_pass] && (options[:nuke] || !test_case.should_fail?)
     msg = "Command `#{options[:engine_adapter]}` did not complete:\n\n#{error}"
 
     if options[:skip]
@@ -26,17 +48,16 @@ def run_spec_test(test_case, options = {})
     raise "#{test_case.input_path} passed a test we expected it to fail"
   end
 
-  if options[:nuke]
-    File.open(test_case.expected_path, "w+") do |f|
-      f.write(output)
-      f.close
-    end
-  end
-
   if test_case.todo? && options[:unexpected_pass]
     assert test_case.expected != clean_output, "Marked as todo and passed"
   elsif !test_case.todo? || !options[:skip_todo]
+    if test_case.should_fail?
+       assert_equal test_case.expected_status, status, "Expected did not match status"
+    end
     assert_equal test_case.expected, clean_output, "Expected did not match output"
+    if test_case.verify_stderr?
+      assert_equal test_case.expected_error, error, "Expected did not match error"
+    end
   end
 end
 

lib/sass_spec/test_case.rb

@@ -1,8 +1,10 @@
 # This represents a specific test case.
 class SassSpec::TestCase
-  def initialize(input_scss, expected_css, style, clean, options = {})
+  def initialize(input_scss, expected_css, error_file, status_file, style, clean, options = {})
     @input_path = input_scss
     @expected_path = expected_css
+    @error_path = error_file
+    @status_path = status_file
     @output_style = style
     @clean_test = clean
     @options = options
@@ -28,6 +30,22 @@ def expected_path
     @expected_path
   end
 
+  def error_path
+    @error_path
+  end
+
+  def verify_stderr?
+    File.file?(@error_path)
+  end
+
+  def status_path
+    @status_path
+  end
+
+  def should_fail?
+    File.file?(@status_path)
+  end
+
   def todo?
     @input_path.to_s.include? "todo"
   end
@@ -36,7 +54,9 @@ def output
     if @output
       return @output
     end
+
     stdout, stderr, status = engine.compile(@input_path, @output_style)
+
     if @clean_test
       cleaned = _clean_output(stdout)
     else
@@ -54,6 +74,18 @@ def expected
     end
   end
 
+  def expected_error
+    @expected_error = File.read(@error_path, :encoding => "utf-8")
+  end
+
+  def expected_status
+    if should_fail?
+      @expected_status = File.read(@status_path, :encoding => "utf-8").to_i
+    else
+      @expected_status = 0
+    end
+  end
+
   def engine
     @options[:engine_adapter]
   end