From f0e5175420ee22ccb1d3fa7757b9a5d0915a2d45 Mon Sep 17 00:00:00 2001 From: Matthew Treinish Date: Wed, 18 Mar 2015 22:47:31 -0400 Subject: [PATCH] Add basic unit tests to check ostestr return codes This commit adds some basic unit tests which runs ostestr in some basic configurations against a fake test suite to ensure that the ostestr always exits with a 0 on success and 1 on errors when running tests. This is invaluable for using ostestr in ci systems. --- os_testr/tests/files/__init__.py | 0 os_testr/tests/files/failing-tests | 23 ++++++ os_testr/tests/files/passing-tests | 23 ++++++ os_testr/tests/files/setup.cfg | 20 ++++++ os_testr/tests/files/testr-conf | 5 ++ os_testr/tests/test_return_codes.py | 104 ++++++++++++++++++++++++++++ requirements.txt | 1 + test-requirements.txt | 3 - 8 files changed, 176 insertions(+), 3 deletions(-) create mode 100644 os_testr/tests/files/__init__.py create mode 100644 os_testr/tests/files/failing-tests create mode 100644 os_testr/tests/files/passing-tests create mode 100644 os_testr/tests/files/setup.cfg create mode 100644 os_testr/tests/files/testr-conf create mode 100644 os_testr/tests/test_return_codes.py diff --git a/os_testr/tests/files/__init__.py b/os_testr/tests/files/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/os_testr/tests/files/failing-tests b/os_testr/tests/files/failing-tests new file mode 100644 index 0000000..78efc93 --- /dev/null +++ b/os_testr/tests/files/failing-tests @@ -0,0 +1,23 @@ +# Copyright 2013 IBM Corp. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import testtools + +class FakeTestClass(testtools.TestCase): + def test_pass(self): + self.assertTrue(False) + + def test_pass_list(self): + test_list = ['test', 'a', 'b'] + self.assertIn('fail', test_list) diff --git a/os_testr/tests/files/passing-tests b/os_testr/tests/files/passing-tests new file mode 100644 index 0000000..a55cb1b --- /dev/null +++ b/os_testr/tests/files/passing-tests @@ -0,0 +1,23 @@ +# Copyright 2013 IBM Corp. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import testtools + +class FakeTestClass(testtools.TestCase): + def test_pass(self): + self.assertTrue(True) + + def test_pass_list(self): + test_list = ['test', 'a', 'b'] + self.assertIn('test', test_list) diff --git a/os_testr/tests/files/setup.cfg b/os_testr/tests/files/setup.cfg new file mode 100644 index 0000000..f6f9f73 --- /dev/null +++ b/os_testr/tests/files/setup.cfg @@ -0,0 +1,20 @@ +[metadata] +name = tempest_unit_tests +version = 1 +summary = Fake Project for testing wrapper scripts +author = OpenStack +author-email = openstack-dev@lists.openstack.org +home-page = http://www.openstack.org/ +classifier = + Intended Audience :: Information Technology + Intended Audience :: System Administrators + Intended Audience :: Developers + License :: OSI Approved :: Apache Software License + Operating System :: POSIX :: Linux + Programming Language :: Python + Programming Language :: Python :: 2 + Programming Language :: Python :: 2.7 + +[global] +setup-hooks = + pbr.hooks.setup_hook diff --git a/os_testr/tests/files/testr-conf b/os_testr/tests/files/testr-conf new file mode 100644 index 0000000..d5ad083 --- /dev/null +++ b/os_testr/tests/files/testr-conf @@ -0,0 +1,5 @@ +[DEFAULT] +test_command=${PYTHON:-python} -m subunit.run discover -t ./ ./tests $LISTOPT $IDOPTION +test_id_option=--load-list $IDFILE +test_list_option=--list +group_regex=([^\.]*\.)* diff --git a/os_testr/tests/test_return_codes.py b/os_testr/tests/test_return_codes.py new file mode 100644 index 0000000..591e4dd --- /dev/null +++ b/os_testr/tests/test_return_codes.py @@ -0,0 +1,104 @@ +# Copyright 2015 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +import shutil +import StringIO +import subprocess +import tempfile + +import testtools + +from os_testr.tests import base + +DEVNULL = open(os.devnull, 'wb') + + +class TestReturnCodes(base.TestCase): + def setUp(self): + super(TestReturnCodes, self).setUp() + # Setup test dirs + self.directory = tempfile.mkdtemp(prefix='ostestr-unit') + self.addCleanup(shutil.rmtree, self.directory) + self.test_dir = os.path.join(self.directory, 'tests') + os.mkdir(self.test_dir) + # Setup Test files + self.testr_conf_file = os.path.join(self.directory, '.testr.conf') + self.setup_cfg_file = os.path.join(self.directory, 'setup.cfg') + self.passing_file = os.path.join(self.test_dir, 'test_passing.py') + self.failing_file = os.path.join(self.test_dir, 'test_failing.py') + self.init_file = os.path.join(self.test_dir, '__init__.py') + self.setup_py = os.path.join(self.directory, 'setup.py') + shutil.copy('os_testr/tests/files/testr-conf', self.testr_conf_file) + shutil.copy('os_testr/tests/files/passing-tests', self.passing_file) + shutil.copy('os_testr/tests/files/failing-tests', self.failing_file) + shutil.copy('setup.py', self.setup_py) + shutil.copy('os_testr/tests/files/setup.cfg', self.setup_cfg_file) + shutil.copy('os_testr/tests/files/__init__.py', self.init_file) + + self.stdout = StringIO.StringIO() + self.stderr = StringIO.StringIO() + # Change directory, run wrapper and check result + self.addCleanup(os.chdir, os.path.abspath(os.curdir)) + os.chdir(self.directory) + + def assertRunExit(self, cmd, expected, subunit=False): + p = subprocess.Popen( + "%s" % cmd, shell=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = p.communicate() + + if not subunit: + self.assertEqual( + p.returncode, expected, + "Stdout: %s; Stderr: %s" % (out, err)) + else: + self.assertEqual(p.returncode, expected, + "Expected return code: %s doesn't match actual " + "return code of: %s" % (expected, p.returncode)) + + def test_default_passing(self): + self.assertRunExit('ostestr --regex passing', 0) + + def test_default_fails(self): + self.assertRunExit('ostestr', 1) + + def test_default_passing_no_slowest(self): + self.assertRunExit('ostestr --no-slowest --regex passing', 0) + + def test_default_fails_no_slowest(self): + self.assertRunExit('ostestr --no-slowest', 1) + + def test_default_serial_passing(self): + self.assertRunExit('ostestr --serial --regex passing', 0) + + def test_default_serial_fails(self): + self.assertRunExit('ostestr --serial', 1) + + def test_testr_subunit_passing(self): + self.assertRunExit('ostestr --no-pretty --subunit --regex passing', 0, + subunit=True) + + @testtools.skip('Skipped because of testrepository lp bug #1411804') + def test_testr_subunit_fails(self): + self.assertRunExit('ostestr --no-pretty --subunit', 1, subunit=True) + + def test_testr_no_pretty_passing(self): + self.assertRunExit('ostestr --no-pretty --regex passing', 0) + + def test_testr_no_pretty_fails(self): + self.assertRunExit('ostestr --no-pretty', 1) + + def test_list(self): + self.assertRunExit('ostestr --list', 0) diff --git a/requirements.txt b/requirements.txt index 9b50a0b..6b439b7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,3 +6,4 @@ pbr>=0.6,!=0.7,<1.0 Babel>=1.3 testrepository>=0.0.18 python-subunit>=0.0.18 +testtools>=0.9.36,!=1.2.0 diff --git a/test-requirements.txt b/test-requirements.txt index 8592bde..c7208f0 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -6,10 +6,7 @@ hacking<0.11,>=0.10.0 coverage>=3.6 discover -python-subunit>=0.0.18 sphinx>=1.1.2,!=1.2.0,!=1.3b1,<1.3 oslosphinx>=2.2.0 # Apache-2.0 oslotest>=1.2.0 # Apache-2.0 -testrepository>=0.0.18 testscenarios>=0.4 -testtools>=0.9.36,!=1.2.0