Add basic unit tests to check ostestr return codes

This commit adds some basic unit tests which runs ostestr in some
basic configurations against a fake test suite to ensure that the
ostestr always exits with a 0 on success and 1 on errors when running
tests. This is invaluable for using ostestr in ci systems.
This commit is contained in:
Matthew Treinish 2015-03-18 22:47:31 -04:00
parent bbc8b8f541
commit f0e5175420
8 changed files with 176 additions and 3 deletions

View File

View File

@ -0,0 +1,23 @@
# Copyright 2013 IBM Corp.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import testtools
class FakeTestClass(testtools.TestCase):
def test_pass(self):
self.assertTrue(False)
def test_pass_list(self):
test_list = ['test', 'a', 'b']
self.assertIn('fail', test_list)

View File

@ -0,0 +1,23 @@
# Copyright 2013 IBM Corp.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import testtools
class FakeTestClass(testtools.TestCase):
def test_pass(self):
self.assertTrue(True)
def test_pass_list(self):
test_list = ['test', 'a', 'b']
self.assertIn('test', test_list)

View File

@ -0,0 +1,20 @@
[metadata]
name = tempest_unit_tests
version = 1
summary = Fake Project for testing wrapper scripts
author = OpenStack
author-email = openstack-dev@lists.openstack.org
home-page = http://www.openstack.org/
classifier =
Intended Audience :: Information Technology
Intended Audience :: System Administrators
Intended Audience :: Developers
License :: OSI Approved :: Apache Software License
Operating System :: POSIX :: Linux
Programming Language :: Python
Programming Language :: Python :: 2
Programming Language :: Python :: 2.7
[global]
setup-hooks =
pbr.hooks.setup_hook

View File

@ -0,0 +1,5 @@
[DEFAULT]
test_command=${PYTHON:-python} -m subunit.run discover -t ./ ./tests $LISTOPT $IDOPTION
test_id_option=--load-list $IDFILE
test_list_option=--list
group_regex=([^\.]*\.)*

View File

@ -0,0 +1,104 @@
# Copyright 2015 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
import shutil
import StringIO
import subprocess
import tempfile
import testtools
from os_testr.tests import base
DEVNULL = open(os.devnull, 'wb')
class TestReturnCodes(base.TestCase):
def setUp(self):
super(TestReturnCodes, self).setUp()
# Setup test dirs
self.directory = tempfile.mkdtemp(prefix='ostestr-unit')
self.addCleanup(shutil.rmtree, self.directory)
self.test_dir = os.path.join(self.directory, 'tests')
os.mkdir(self.test_dir)
# Setup Test files
self.testr_conf_file = os.path.join(self.directory, '.testr.conf')
self.setup_cfg_file = os.path.join(self.directory, 'setup.cfg')
self.passing_file = os.path.join(self.test_dir, 'test_passing.py')
self.failing_file = os.path.join(self.test_dir, 'test_failing.py')
self.init_file = os.path.join(self.test_dir, '__init__.py')
self.setup_py = os.path.join(self.directory, 'setup.py')
shutil.copy('os_testr/tests/files/testr-conf', self.testr_conf_file)
shutil.copy('os_testr/tests/files/passing-tests', self.passing_file)
shutil.copy('os_testr/tests/files/failing-tests', self.failing_file)
shutil.copy('setup.py', self.setup_py)
shutil.copy('os_testr/tests/files/setup.cfg', self.setup_cfg_file)
shutil.copy('os_testr/tests/files/__init__.py', self.init_file)
self.stdout = StringIO.StringIO()
self.stderr = StringIO.StringIO()
# Change directory, run wrapper and check result
self.addCleanup(os.chdir, os.path.abspath(os.curdir))
os.chdir(self.directory)
def assertRunExit(self, cmd, expected, subunit=False):
p = subprocess.Popen(
"%s" % cmd, shell=True,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, err = p.communicate()
if not subunit:
self.assertEqual(
p.returncode, expected,
"Stdout: %s; Stderr: %s" % (out, err))
else:
self.assertEqual(p.returncode, expected,
"Expected return code: %s doesn't match actual "
"return code of: %s" % (expected, p.returncode))
def test_default_passing(self):
self.assertRunExit('ostestr --regex passing', 0)
def test_default_fails(self):
self.assertRunExit('ostestr', 1)
def test_default_passing_no_slowest(self):
self.assertRunExit('ostestr --no-slowest --regex passing', 0)
def test_default_fails_no_slowest(self):
self.assertRunExit('ostestr --no-slowest', 1)
def test_default_serial_passing(self):
self.assertRunExit('ostestr --serial --regex passing', 0)
def test_default_serial_fails(self):
self.assertRunExit('ostestr --serial', 1)
def test_testr_subunit_passing(self):
self.assertRunExit('ostestr --no-pretty --subunit --regex passing', 0,
subunit=True)
@testtools.skip('Skipped because of testrepository lp bug #1411804')
def test_testr_subunit_fails(self):
self.assertRunExit('ostestr --no-pretty --subunit', 1, subunit=True)
def test_testr_no_pretty_passing(self):
self.assertRunExit('ostestr --no-pretty --regex passing', 0)
def test_testr_no_pretty_fails(self):
self.assertRunExit('ostestr --no-pretty', 1)
def test_list(self):
self.assertRunExit('ostestr --list', 0)

View File

@ -6,3 +6,4 @@ pbr>=0.6,!=0.7,<1.0
Babel>=1.3 Babel>=1.3
testrepository>=0.0.18 testrepository>=0.0.18
python-subunit>=0.0.18 python-subunit>=0.0.18
testtools>=0.9.36,!=1.2.0

View File

@ -6,10 +6,7 @@ hacking<0.11,>=0.10.0
coverage>=3.6 coverage>=3.6
discover discover
python-subunit>=0.0.18
sphinx>=1.1.2,!=1.2.0,!=1.3b1,<1.3 sphinx>=1.1.2,!=1.2.0,!=1.3b1,<1.3
oslosphinx>=2.2.0 # Apache-2.0 oslosphinx>=2.2.0 # Apache-2.0
oslotest>=1.2.0 # Apache-2.0 oslotest>=1.2.0 # Apache-2.0
testrepository>=0.0.18
testscenarios>=0.4 testscenarios>=0.4
testtools>=0.9.36,!=1.2.0