diff --git a/octane/tests/conftest.py b/octane/tests/conftest.py
index 655886be..97d375a4 100644
--- a/octane/tests/conftest.py
+++ b/octane/tests/conftest.py
@@ -30,9 +30,10 @@ class SafeOctaneApp(app.OctaneApp):
 
     def run(self, argv):
         try:
-            super(SafeOctaneApp, self).run(argv)
+            exit_code = super(SafeOctaneApp, self).run(argv)
         except SystemExit as e:
-            assert e.code == 0
+            exit_code = e.code
+        assert exit_code == 0
 
 
 @pytest.fixture
diff --git a/octane/tests/test_test.py b/octane/tests/test_test.py
index f2c6e2df..2a413b1a 100644
--- a/octane/tests/test_test.py
+++ b/octane/tests/test_test.py
@@ -13,6 +13,12 @@
 import pytest
 
 
+def test_octane_app_not_exception_return_value(mocker, octane_app):
+    mocker.patch('octane.app.OctaneApp.run', return_value=2)
+    with pytest.raises(AssertionError):
+        octane_app.run([])
+
+
 def test_octane_app_pass_exceptions_through(mocker, octane_app):
     mocker.patch.object(octane_app.command_manager, 'find_command',
                         side_effect=ValueError)