Unittests refactoring for ci

Logic that supports automated execution of unit tests for ci purpose
were added via setUp/tearDownPackage functions in
testing/tests/__init__.py file. Now when tests started via nose in
package setUp function nailgunmimic server is setting up in separate
process and terminating in tearDown. This functionality provides unified
way to execute unit tests.

Change-Id: I4a033757c8d902c90b0a6c7eb54c1511ab721f99
This commit is contained in:
Artem Roma 2013-11-14 16:17:16 +02:00
parent 633fd89cae
commit fdf8007eab
41 changed files with 191 additions and 76 deletions

1
.gitignore vendored
View File

@ -11,3 +11,4 @@ dist
.coverage
.ropeproject
*.swp
etc/*

26
fabfile.py vendored
View File

@ -37,11 +37,11 @@ def startserver():
def startdebugserver():
local(('ostf-server '
'--nailgun-port=8888 '
'--debug_tests=fuel_plugin/tests/functional/dummy_tests'))
'--debug_tests=fuel_plugin/testing/fixture/dummy_tests'))
def startnailgunmimic():
path = 'fuel_plugin/tests/test_utils/nailgun_mimic.py'
path = 'fuel_plugin/testing/test_utils/nailgun_mimic.py'
local('python {0}'.format(path))
@ -66,7 +66,7 @@ def migrate(database='ostf'):
def auth(method='trust', os='ubuntu'):
"""By default postgres doesnot allow auth withour password
"""By default postgres doesn't allow auth without password
development without password is more fun
"""
if os == 'centos':
@ -98,8 +98,24 @@ def testall():
def integration():
local('nosetests fuel_plugin/tests/functional/tests.py:AdapterTests -v')
local(
('nosetests fuel_plugin/testing/'
'tests/functional/tests.py:AdapterTests -v')
)
def unit():
local('nosetests fuel_plugin/tests/unit -v')
local('nosetests fuel_plugin/testing/tests/unit -v')
def masstest():
for i in range(10):
testall()
def testissue():
for i in range(10):
local(
('nosetests -sv fuel_plugin/testing/tests/functional/'
'tests.py:AdapterTests.test_run_single_test')
)

View File

@ -1,5 +1,3 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2013 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may

View File

@ -66,7 +66,17 @@ class NoseDriver(object):
session = engine.get_session()
if test_run_id in self._named_threads:
self._named_threads[test_run_id].terminate()
try:
self._named_threads[test_run_id].terminate()
except OSError as e:
if e.errno != os.errno.ESRCH:
raise
LOG.warning(
'There is no process for test_run with following id - %s',
test_run_id
)
self._named_threads.pop(test_run_id, None)
if cleanup:

View File

@ -10,4 +10,4 @@
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# under the License.

View File

@ -10,4 +10,4 @@
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# under the License.

View File

@ -0,0 +1,43 @@
# Copyright 2013 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import signal
import subprocess
import time
def setup():
global processes_pool
with open('fuel_plugin/testing/tests/etc/server.log', 'w') as serverlogs:
with open('/dev/null', 'w') as devnull:
processes_pool = tuple(
[
subprocess.Popen(
[
'python',
'fuel_plugin/testing/test_utils/nailgun_mimic.py'
],
stdout=devnull,
stderr=devnull
)
]
)
time.sleep(5)
def teardown():
for process in processes_pool:
process.send_signal(signal.SIGINT)
process.wait()

View File

@ -0,0 +1,2 @@
*
!.gitignore

View File

@ -15,7 +15,8 @@
import time
from sqlalchemy import create_engine
from fuel_plugin.tests.functional.base import BaseAdapterTest, Response
from fuel_plugin.testing.tests.functional.base import \
BaseAdapterTest, Response
from fuel_plugin.ostf_client.client import TestingAdapterClient as adapter
@ -27,25 +28,25 @@ class AdapterTests(BaseAdapterTest):
url = 'http://0.0.0.0:8989/v1'
cls.mapping = {
('fuel_plugin.tests.functional.dummy_tests.'
('fuel_plugin.testing.fixture.dummy_tests.'
'general_test.Dummy_test.test_fast_pass'): 'fast_pass',
('fuel_plugin.tests.functional.dummy_tests.'
('fuel_plugin.testing.fixture.dummy_tests.'
'general_test.Dummy_test.test_fast_error'): 'fast_error',
('fuel_plugin.tests.functional.dummy_tests.'
('fuel_plugin.testing.fixture.dummy_tests.'
'general_test.Dummy_test.test_fast_fail'): 'fast_fail',
('fuel_plugin.tests.functional.dummy_tests.'
('fuel_plugin.testing.fixture.dummy_tests.'
'general_test.Dummy_test.test_long_pass'): 'long_pass',
('fuel_plugin.tests.functional.dummy_tests.'
('fuel_plugin.testing.fixture.dummy_tests.'
'general_test.Dummy_test.test_fail_with_step'): 'fail_step',
('fuel_plugin.tests.functional.dummy_tests.stopped_test.'
('fuel_plugin.testing.fixture.dummy_tests.stopped_test.'
'dummy_tests_stopped.test_really_long'): 'really_long',
('fuel_plugin.tests.functional.dummy_tests.stopped_test.'
('fuel_plugin.testing.fixture.dummy_tests.stopped_test.'
'dummy_tests_stopped.test_not_long_at_all'): 'not_long',
('fuel_plugin.tests.functional.dummy_tests.stopped_test.'
('fuel_plugin.testing.fixture.dummy_tests.stopped_test.'
'dummy_tests_stopped.test_one_no_so_long'): 'so_long',
('fuel_plugin.tests.functional.dummy_tests.deployment_types_tests.'
('fuel_plugin.testing.fixture.dummy_tests.deployment_types_tests.'
'ha_deployment_test.HATest.test_ha_depl'): 'ha_depl',
('fuel_plugin.tests.functional.dummy_tests.deployment_types_tests.'
('fuel_plugin.testing.fixture.dummy_tests.deployment_types_tests.'
'ha_deployment_test.HATest.test_ha_rhel_depl'): 'ha_rhel_depl'
}
cls.testsets = {
@ -218,6 +219,9 @@ class AdapterTests(BaseAdapterTest):
for cluster_id in range(1, 2):
r = self.client.start_testrun(testset, cluster_id)
msg = '{0} was empty'.format(r.request)
if r.is_empty:
print r
self.assertFalse(r.is_empty, msg)
'''TODO: Rewrite assertions to verity that all
@ -228,10 +232,10 @@ class AdapterTests(BaseAdapterTest):
"""Verify that you can run individual tests from given testset"""
testset = "general_test"
tests = [
('fuel_plugin.tests.functional.dummy_tests.'
('fuel_plugin.testing.fixture.dummy_tests.'
'general_test.Dummy_test.test_fast_pass'),
('fuel_plugin.tests.functional.dummy_tests.'
'general_test.Dummy_test.test_fast_fail')
('fuel_plugin.testing.fixture.dummy_tests.'
'general_test.Dummy_test.test_fast_error')
]
cluster_id = 1
@ -248,31 +252,31 @@ class AdapterTests(BaseAdapterTest):
{
'status': 'disabled',
'name': 'Fast fail with step',
'id': ('fuel_plugin.tests.functional.dummy_tests.'
'id': ('fuel_plugin.testing.fixture.dummy_tests.'
'general_test.Dummy_test.test_fail_with_step'),
},
{
'status': 'disabled',
'status': 'wait_running',
'name': 'And fast error',
'id': ('fuel_plugin.tests.functional.dummy_tests.'
'id': ('fuel_plugin.testing.fixture.dummy_tests.'
'general_test.Dummy_test.test_fast_error'),
},
{
'status': 'wait_running',
'status': 'disabled',
'name': 'Fast fail',
'id': ('fuel_plugin.tests.functional.dummy_tests.'
'id': ('fuel_plugin.testing.fixture.dummy_tests.'
'general_test.Dummy_test.test_fast_fail'),
},
{
'status': 'wait_running',
'name': 'fast pass test',
'id': ('fuel_plugin.tests.functional.dummy_tests.'
'id': ('fuel_plugin.testing.fixture.dummy_tests.'
'general_test.Dummy_test.test_fast_pass'),
},
{
'status': 'disabled',
'name': 'Will sleep 5 sec',
'id': ('fuel_plugin.tests.functional.dummy_tests.'
'id': ('fuel_plugin.testing.fixture.dummy_tests.'
'general_test.Dummy_test.test_long_pass'),
}
],
@ -281,25 +285,40 @@ class AdapterTests(BaseAdapterTest):
])
self.compare(r, assertions)
time.sleep(2)
time.sleep(5)
r = self.client.testruns_last(cluster_id)
assertions.general_test['status'] = 'finished'
for test in assertions.general_test['tests']:
if test['name'] == 'Fast fail':
test['status'] = 'failure'
if test['name'] == 'And fast error':
test['status'] = 'error'
elif test['name'] == 'fast pass test':
test['status'] = 'success'
self.compare(r, assertions)
try:
self.compare(r, assertions)
except AssertionError:
from fuel_plugin.ostf_adapter.storage import models
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
session = sessionmaker()(bind=create_engine('postgresql+psycopg2://ostf:ostf@localhost/ostf'))
test_run_id = [r[k]['id'] for k in r.keys() if k == 'general_test'].pop()
fast_error = session.query(models.Test).filter_by(test_run_id=test_run_id)\
.filter_by(test_set_id='general_test')\
.filter_by(name='fuel_plugin.testing.fixture.dummy_tests.general_test.Dummy_test.test_fast_error')\
.one()
raise AssertionError('Status of fast_error_test from database -- {0}'.format(fast_error.status))
def test_single_test_restart(self):
"""Verify that you restart individual tests for given testrun"""
testset = "general_test"
tests = [
('fuel_plugin.tests.functional.dummy_tests.'
('fuel_plugin.testing.fixture.dummy_tests.'
'general_test.Dummy_test.test_fast_pass'),
('fuel_plugin.tests.functional.dummy_tests.general_test.'
('fuel_plugin.testing.fixture.dummy_tests.general_test.'
'Dummy_test.test_fast_fail')
]
cluster_id = 1
@ -319,31 +338,31 @@ class AdapterTests(BaseAdapterTest):
{
'status': 'failure',
'name': 'Fast fail with step',
'id': ('fuel_plugin.tests.functional.dummy_tests.'
'id': ('fuel_plugin.testing.fixture.dummy_tests.'
'general_test.Dummy_test.test_fail_with_step'),
},
{
'status': 'error',
'name': 'And fast error',
'id': ('fuel_plugin.tests.functional.dummy_tests.'
'id': ('fuel_plugin.testing.fixture.dummy_tests.'
'general_test.Dummy_test.test_fast_error'),
},
{
'status': 'wait_running',
'name': 'Fast fail',
'id': ('fuel_plugin.tests.functional.dummy_tests.'
'id': ('fuel_plugin.testing.fixture.dummy_tests.'
'general_test.Dummy_test.test_fast_fail'),
},
{
'status': 'wait_running',
'name': 'fast pass test',
'id': ('fuel_plugin.tests.functional.dummy_tests.'
'id': ('fuel_plugin.testing.fixture.dummy_tests.'
'general_test.Dummy_test.test_fast_pass'),
},
{
'status': 'success',
'name': 'Will sleep 5 sec',
'id': ('fuel_plugin.tests.functional.dummy_tests.'
'id': ('fuel_plugin.testing.fixture.dummy_tests.'
'general_test.Dummy_test.test_long_pass'),
}
],
@ -362,20 +381,36 @@ class AdapterTests(BaseAdapterTest):
test['status'] = 'failure'
elif test['name'] == 'fast pass test':
test['status'] = 'success'
self.compare(r, assertions)
#DEBUG
try:
self.compare(r, assertions)
except AssertionError:
from fuel_plugin.ostf_adapter.storage import models
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
session = sessionmaker()(bind=create_engine('postgresql+psycopg2://ostf:ostf@localhost/ostf'))
test_run_id = [r[k]['id'] for k in r.keys() if k == 'general_test'].pop()
fast_error = session.query(models.Test).filter_by(test_run_id=test_run_id)\
.filter_by(test_set_id='general_test')\
.filter_by(name='fuel_plugin.testing.fixture.dummy_tests.general_test.Dummy_test.test_fast_fail')\
.one()
raise AssertionError('Status of test_fast_fail from database -- {0}'.format(fast_error.status))
def test_restart_combinations(self):
"""Verify that you can restart both tests that
ran and did not run during single test start"""
testset = "general_test"
tests = [
('fuel_plugin.tests.functional.dummy_tests.'
('fuel_plugin.testing.fixture.dummy_tests.'
'general_test.Dummy_test.test_fast_pass'),
('fuel_plugin.tests.functional.dummy_tests.'
('fuel_plugin.testing.fixture.dummy_tests.'
'general_test.Dummy_test.test_fast_fail')
]
disabled_test = [
('fuel_plugin.tests.functional.dummy_tests.'
('fuel_plugin.testing.fixture.dummy_tests.'
'general_test.Dummy_test.test_fast_error')
]
cluster_id = 1
@ -383,7 +418,7 @@ class AdapterTests(BaseAdapterTest):
#make sure we have all needed data in db
self.adapter.testsets(cluster_id)
self.client.run_with_timeout(testset, tests, cluster_id, 70)
self.client.run_with_timeout(testset, tests, cluster_id, 80)
self.client.restart_with_timeout(testset, tests, cluster_id, 10)
r = self.client.restart_tests_last(testset, disabled_test, cluster_id)
@ -396,31 +431,31 @@ class AdapterTests(BaseAdapterTest):
{
'status': 'disabled',
'name': 'Fast fail with step',
'id': ('fuel_plugin.tests.functional.dummy_tests.'
'id': ('fuel_plugin.testing.fixture.dummy_tests.'
'general_test.Dummy_test.test_fail_with_step'),
},
{
'status': 'wait_running',
'name': 'And fast error',
'id': ('fuel_plugin.tests.functional.dummy_tests.'
'id': ('fuel_plugin.testing.fixture.dummy_tests.'
'general_test.Dummy_test.test_fast_error'),
},
{
'status': 'failure',
'name': 'Fast fail',
'id': ('fuel_plugin.tests.functional.dummy_tests.'
'id': ('fuel_plugin.testing.fixture.dummy_tests.'
'general_test.Dummy_test.test_fast_fail'),
},
{
'status': 'success',
'name': 'fast pass test',
'id': ('fuel_plugin.tests.functional.dummy_tests.'
'id': ('fuel_plugin.testing.fixture.dummy_tests.'
'general_test.Dummy_test.test_fast_pass'),
},
{
'status': 'disabled',
'name': 'Will sleep 5 sec',
'id': ('fuel_plugin.tests.functional.dummy_tests.'
'id': ('fuel_plugin.testing.fixture.dummy_tests.'
'general_test.Dummy_test.test_long_pass'),
}
],
@ -441,11 +476,11 @@ class AdapterTests(BaseAdapterTest):
def test_cant_restart_during_run(self):
testset = 'general_test'
tests = [
('fuel_plugin.tests.functional.dummy_tests.'
('fuel_plugin.testing.fixture.dummy_tests.'
'general_test.Dummy_test.test_fast_pass'),
('fuel_plugin.tests.functional.dummy_tests.'
('fuel_plugin.testing.fixture.dummy_tests.'
'general_test.Dummy_test.test_fast_fail'),
('fuel_plugin.tests.functional.dummy_tests.'
('fuel_plugin.testing.fixture.dummy_tests.'
'general_test.Dummy_test.test_fast_pass')
]
cluster_id = 1
@ -481,7 +516,7 @@ class AdapterTests(BaseAdapterTest):
'tests': [
{
'id': (
'fuel_plugin.tests.functional.'
'fuel_plugin.testing.fixture.'
'dummy_tests.test_with_error.WithErrorTest.'
'test_supposed_to_be_fail'
),
@ -489,7 +524,7 @@ class AdapterTests(BaseAdapterTest):
},
{
'id': (
'fuel_plugin.tests.functional.'
'fuel_plugin.testing.fixture.'
'dummy_tests.test_with_error.WithErrorTest.'
'test_supposed_to_be_success'
),

View File

@ -0,0 +1,13 @@
# Copyright 2013 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

View File

@ -23,7 +23,7 @@ from fuel_plugin.ostf_adapter.nose_plugin.nose_discovery import discovery
from fuel_plugin.ostf_adapter.storage import models
from fuel_plugin.ostf_adapter.storage import simple_cache
TEST_PATH = 'fuel_plugin/tests/functional/dummy_tests'
TEST_PATH = 'fuel_plugin/testing/fixture/dummy_tests'
class BaseWSGITest(unittest2.TestCase):
@ -35,7 +35,7 @@ class BaseWSGITest(unittest2.TestCase):
'postgresql+psycopg2://ostf:ostf@localhost/ostf'
)
cls.ext_id = 'fuel_plugin.tests.functional.dummy_tests.'
cls.ext_id = 'fuel_plugin.testing.fixture.dummy_tests.'
cls.expected = {
'cluster': {
'id': 1,

View File

@ -20,7 +20,7 @@ from sqlalchemy.orm import sessionmaker
from fuel_plugin.ostf_adapter.nose_plugin import nose_discovery
from fuel_plugin.ostf_adapter.storage import models
TEST_PATH = 'fuel_plugin/tests/functional/dummy_tests'
TEST_PATH = 'fuel_plugin/testing/fixture/dummy_tests'
class TransactionBeginMock:
@ -84,7 +84,7 @@ class TestNoseDiscovery(unittest2.TestCase):
def test_get_proper_description(self):
expected = {
'title': 'fake empty test',
'name': ('fuel_plugin.tests.functional.'
'name': ('fuel_plugin.testing.fixture.'
'dummy_tests.deployment_types_tests.'
'ha_deployment_test.HATest.test_ha_rhel_depl'),
'duration': '0sec',
@ -111,7 +111,7 @@ class TestNoseDiscovery(unittest2.TestCase):
'deployment_tags': ['alternative | alternative_test']
},
'test': {
'name': ('fuel_plugin.tests.functional.dummy_tests.'
'name': ('fuel_plugin.testing.fixture.dummy_tests.'
'deployment_types_tests.alternative_depl_tags_test.'
'AlternativeDeplTagsTests.test_simple_fake_test'),
'deployment_tags': ['one_tag| another_tag', 'other_tag']

View File

@ -18,11 +18,10 @@ from mock import patch, Mock
from fuel_plugin.ostf_adapter.wsgi import controllers
from fuel_plugin.ostf_adapter.storage import models
from fuel_plugin.tests.unit.base \
import BaseWSGITest
from fuel_plugin.testing.tests.unit import base
class TestTestsController(BaseWSGITest):
class TestTestsController(base.BaseWSGITest):
def setUp(self):
super(TestTestsController, self).setUp()
@ -40,7 +39,7 @@ class TestTestsController(BaseWSGITest):
)
class TestTestSetsController(BaseWSGITest):
class TestTestSetsController(base.BaseWSGITest):
def setUp(self):
super(TestTestSetsController, self).setUp()
@ -66,7 +65,7 @@ class TestTestSetsController(BaseWSGITest):
)
class TestTestRunsController(BaseWSGITest):
class TestTestRunsController(base.BaseWSGITest):
def setUp(self):
super(TestTestRunsController, self).setUp()
@ -106,10 +105,10 @@ class TestTestRunsPostController(TestTestRunsController):
'cluster_id': 1,
'tests': {
'names': [
('fuel_plugin.tests.functional.dummy_tests.'
('fuel_plugin.testing.fixture.dummy_tests.'
'deployment_types_tests.ha_deployment_test.'
'HATest.test_ha_depl'),
('fuel_plugin.tests.functional.dummy_tests.'
('fuel_plugin.testing.fixture.dummy_tests.'
'deployment_types_tests.ha_deployment_test.'
'HATest.test_ha_rhel_depl')
]
@ -147,7 +146,6 @@ class TestTestRunsPostController(TestTestRunsController):
)
#@unittest2.skip('Is broken, fixing is appreciated')
class TestTestRunsPutController(TestTestRunsController):
def setUp(self):
@ -173,10 +171,10 @@ class TestTestRunsPutController(TestTestRunsController):
'cluster_id': 1,
'tests': {
'names': [
('fuel_plugin.tests.functional.dummy_tests.'
('fuel_plugin.testing.fixture.dummy_tests.'
'deployment_types_tests.ha_deployment_test.'
'HATest.test_ha_depl'),
('fuel_plugin.tests.functional.dummy_tests.'
('fuel_plugin.testing.fixture.dummy_tests.'
'deployment_types_tests.ha_deployment_test.'
'HATest.test_ha_rhel_depl')
]
@ -215,7 +213,7 @@ class TestTestRunsPutController(TestTestRunsController):
)
class TestClusterRedeployment(BaseWSGITest):
class TestClusterRedeployment(base.BaseWSGITest):
def setUp(self):
super(TestClusterRedeployment, self).setUp()

View File

@ -18,11 +18,10 @@ from webtest import TestApp
from fuel_plugin.ostf_adapter.wsgi import app
from fuel_plugin.ostf_adapter.wsgi.app import PECAN_DEFAULT
from fuel_plugin.tests.unit.base \
import BaseWSGITest
from fuel_plugin.testing.tests.unit import base
class WsgiInterfaceTests(BaseWSGITest):
class WsgiInterfaceTests(base.BaseWSGITest):
@classmethod
def setUpClass(cls):