exiting with proper status

Makefile

@@ -4,7 +4,7 @@ filename=steadymark-`python -c 'import steadymark;print steadymark.version'`.tar
 
 export PYTHONPATH:=  ${PWD}
 
-test: clean unit functional
+test: clean unit functional integration
 
 unit:
 	@echo "Running unit tests"
@@ -13,6 +13,9 @@ unit:
 functional:
 	@python steadymark/__init__.py
 
+integration:
+	@nosetests --verbosity=2 -s tests/integration
+
 clean:
 	@printf "Cleaning up files that are already in .gitignore... "
 	@for pattern in `cat .gitignore`; do rm -rf $$pattern; find . -name "$$pattern" -exec rm -rf {} \;; done

steadymark/__init__.py

@@ -248,14 +248,20 @@ class Runner(object):
         if self.filename:
             print "Running tests from {0}".format(self.filename)
 
+        exit_status = 0
         for test in self.steadymark.tests:
             title = "{0} ".format(test.title)
             title_length = len(title)
             print "." * title_length
             sys.stdout.write(title)
             result, failure, before, after = test.run()
+            if failure:
+                exit_status = 1
             self.report_test_result(test, failure, before, after)
 
+        if exit_status is not 0:
+            sys.exit(exit_status)
+
         return self.steadymark
 
 

tests/integration/__init__.py

@@ -0,0 +1,25 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# <steadymark - markdown-based test runner for python>
+# Copyright (C) <2012>  Gabriel Falcão <gabriel@nacaolivre.org>
+#
+# Permission is hereby granted, free of charge, to any person
+# obtaining a copy of this software and associated documentation
+# files (the "Software"), to deal in the Software without
+# restriction, including without limitation the rights to use,
+# copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following
+# conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.

tests/integration/fails.md

@@ -0,0 +1,5 @@
+# This one fails
+
+```python
+assert False, "LOL"
+```

tests/integration/passes.md

@@ -0,0 +1,5 @@
+# This one passes
+
+```python
+assert True
+```

tests/integration/test_exit_status.py

@@ -0,0 +1,56 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# <steadymark - markdown-based test runner for python>
+# Copyright (C) <2012>  Gabriel Falcão <gabriel@nacaolivre.org>
+#
+# Permission is hereby granted, free of charge, to any person
+# obtaining a copy of this software and associated documentation
+# files (the "Software"), to deal in the Software without
+# restriction, including without limitation the rights to use,
+# copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following
+# conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+
+import sys
+from os.path import dirname, abspath, join
+from subprocess import check_call, CalledProcessError
+
+
+CURDIR = abspath(join(dirname(__file__)))
+MAINDIR = abspath(join(CURDIR, '..', '..'))
+
+main_file = join(MAINDIR, 'steadymark', '__init__.py')
+
+
+def run(path):
+    out = open('/dev/null')
+    return check_call([sys.executable, main_file, path],
+                        stdout=out, stderr=out)
+
+
+def test_failure_exits_with_1():
+    (u"SteadyMark should exit with status 1 in case of failure")
+
+    path = join(CURDIR, 'fails.md')
+    run.when.called_with(path).should.throw(
+                CalledProcessError, 'exit status 1')
+
+
+def test_success_exits_with_0():
+    (u"SteadyMark should exit with status 0 in case of success")
+
+    status = run(join(CURDIR, 'passes.md'))
+    status.should.equal(0)