Add some documentation on functional testing tools
Change-Id: I4c694ba8da0ece7d8e94921edc8ff7b46242f705
This commit is contained in:
parent
48d9a22ded
commit
e7b99a0baa
|
@ -85,3 +85,34 @@ Other Global Objects
|
||||||
.. autoclass:: zuul.model.RepoFiles
|
.. autoclass:: zuul.model.RepoFiles
|
||||||
.. autoclass:: zuul.model.Worker
|
.. autoclass:: zuul.model.Worker
|
||||||
.. autoclass:: zuul.model.TriggerEvent
|
.. autoclass:: zuul.model.TriggerEvent
|
||||||
|
|
||||||
|
|
||||||
|
Testing
|
||||||
|
-------
|
||||||
|
|
||||||
|
Zuul provides an extensive framework for performing functional testing
|
||||||
|
on the system from end-to-end with major external components replaced
|
||||||
|
by fakes for ease of use and speed.
|
||||||
|
|
||||||
|
Test classes that subclass :py:class:`~tests.base.ZuulTestCase` have
|
||||||
|
access to a number of attributes useful for manipulating or inspecting
|
||||||
|
the environment being simulated in the test:
|
||||||
|
|
||||||
|
.. autoclass:: tests.base.ZuulTestCase
|
||||||
|
:members:
|
||||||
|
|
||||||
|
.. autoclass:: tests.base.FakeGerritConnection
|
||||||
|
:members:
|
||||||
|
:inherited-members:
|
||||||
|
|
||||||
|
.. autoclass:: tests.base.FakeGearmanServer
|
||||||
|
:members:
|
||||||
|
|
||||||
|
.. autoclass:: tests.base.RecordingLaunchServer
|
||||||
|
:members:
|
||||||
|
|
||||||
|
.. autoclass:: tests.base.FakeBuild
|
||||||
|
:members:
|
||||||
|
|
||||||
|
.. autoclass:: tests.base.BuildHistory
|
||||||
|
:members:
|
||||||
|
|
125
tests/base.py
125
tests/base.py
|
@ -386,6 +386,13 @@ class FakeChange(object):
|
||||||
|
|
||||||
|
|
||||||
class FakeGerritConnection(zuul.connection.gerrit.GerritConnection):
|
class FakeGerritConnection(zuul.connection.gerrit.GerritConnection):
|
||||||
|
"""A Fake Gerrit connection for use in tests.
|
||||||
|
|
||||||
|
This subclasses
|
||||||
|
:py:class:`~zuul.connection.gerrit.GerritConnection` to add the
|
||||||
|
ability for tests to add changes to the fake Gerrit it represents.
|
||||||
|
"""
|
||||||
|
|
||||||
log = logging.getLogger("zuul.test.FakeGerritConnection")
|
log = logging.getLogger("zuul.test.FakeGerritConnection")
|
||||||
|
|
||||||
def __init__(self, connection_name, connection_config,
|
def __init__(self, connection_name, connection_config,
|
||||||
|
@ -402,6 +409,7 @@ class FakeGerritConnection(zuul.connection.gerrit.GerritConnection):
|
||||||
|
|
||||||
def addFakeChange(self, project, branch, subject, status='NEW',
|
def addFakeChange(self, project, branch, subject, status='NEW',
|
||||||
files=None):
|
files=None):
|
||||||
|
"""Add a change to the fake Gerrit."""
|
||||||
self.change_number += 1
|
self.change_number += 1
|
||||||
c = FakeChange(self, self.change_number, project, branch, subject,
|
c = FakeChange(self, self.change_number, project, branch, subject,
|
||||||
upstream_root=self.upstream_root,
|
upstream_root=self.upstream_root,
|
||||||
|
@ -552,6 +560,7 @@ class FakeBuild(object):
|
||||||
self.changes = self.parameters['ZUUL_CHANGE_IDS']
|
self.changes = self.parameters['ZUUL_CHANGE_IDS']
|
||||||
|
|
||||||
def release(self):
|
def release(self):
|
||||||
|
"""Release this build."""
|
||||||
self.wait_condition.acquire()
|
self.wait_condition.acquire()
|
||||||
self.wait_condition.notify()
|
self.wait_condition.notify()
|
||||||
self.waiting = False
|
self.waiting = False
|
||||||
|
@ -559,6 +568,12 @@ class FakeBuild(object):
|
||||||
self.wait_condition.release()
|
self.wait_condition.release()
|
||||||
|
|
||||||
def isWaiting(self):
|
def isWaiting(self):
|
||||||
|
"""Return whether this build is being held.
|
||||||
|
|
||||||
|
:returns: Whether the build is being held.
|
||||||
|
:rtype: bool
|
||||||
|
"""
|
||||||
|
|
||||||
self.wait_condition.acquire()
|
self.wait_condition.acquire()
|
||||||
if self.waiting:
|
if self.waiting:
|
||||||
ret = True
|
ret = True
|
||||||
|
@ -613,13 +628,23 @@ class FakeBuild(object):
|
||||||
|
|
||||||
return result
|
return result
|
||||||
|
|
||||||
def hasChanges(self, *commits):
|
def hasChanges(self, *changes):
|
||||||
|
"""Return whether this build has certain changes in its git repos.
|
||||||
|
|
||||||
|
:arg FakeChange changes: One or more changes (varargs) that
|
||||||
|
are expected to be present (in order) in the git repository of
|
||||||
|
the active project.
|
||||||
|
|
||||||
|
:returns: Whether the build has the indicated changes.
|
||||||
|
:rtype: bool
|
||||||
|
|
||||||
|
"""
|
||||||
project = self.parameters['ZUUL_PROJECT']
|
project = self.parameters['ZUUL_PROJECT']
|
||||||
path = os.path.join(self.jobdir.git_root, project)
|
path = os.path.join(self.jobdir.git_root, project)
|
||||||
repo = git.Repo(path)
|
repo = git.Repo(path)
|
||||||
ref = self.parameters['ZUUL_REF']
|
ref = self.parameters['ZUUL_REF']
|
||||||
repo_messages = [c.message.strip() for c in repo.iter_commits(ref)]
|
repo_messages = [c.message.strip() for c in repo.iter_commits(ref)]
|
||||||
commit_messages = ['%s-1' % commit.subject for commit in commits]
|
commit_messages = ['%s-1' % change.subject for change in changes]
|
||||||
self.log.debug("Checking if build %s has changes; commit_messages %s;"
|
self.log.debug("Checking if build %s has changes; commit_messages %s;"
|
||||||
" repo_messages %s" % (self, commit_messages,
|
" repo_messages %s" % (self, commit_messages,
|
||||||
repo_messages))
|
repo_messages))
|
||||||
|
@ -632,6 +657,16 @@ class FakeBuild(object):
|
||||||
|
|
||||||
|
|
||||||
class RecordingLaunchServer(zuul.launcher.server.LaunchServer):
|
class RecordingLaunchServer(zuul.launcher.server.LaunchServer):
|
||||||
|
"""An Ansible launcher to be used in tests.
|
||||||
|
|
||||||
|
:ivar bool hold_jobs_in_build: If true, when jobs are launched
|
||||||
|
they will report that they have started but then pause until
|
||||||
|
released before reporting completion. This attribute may be
|
||||||
|
changed at any time and will take effect for subsequently
|
||||||
|
launched builds, but previously held builds will still need to
|
||||||
|
be explicitly released.
|
||||||
|
|
||||||
|
"""
|
||||||
def __init__(self, *args, **kw):
|
def __init__(self, *args, **kw):
|
||||||
self._run_ansible = kw.pop('_run_ansible', False)
|
self._run_ansible = kw.pop('_run_ansible', False)
|
||||||
super(RecordingLaunchServer, self).__init__(*args, **kw)
|
super(RecordingLaunchServer, self).__init__(*args, **kw)
|
||||||
|
@ -645,6 +680,12 @@ class RecordingLaunchServer(zuul.launcher.server.LaunchServer):
|
||||||
self.job_builds = {}
|
self.job_builds = {}
|
||||||
|
|
||||||
def addFailTest(self, name, change):
|
def addFailTest(self, name, change):
|
||||||
|
"""Instruct the launcher to report matching builds as failures.
|
||||||
|
|
||||||
|
:arg str name: The name of the job to fail.
|
||||||
|
:arg change: TODO: document
|
||||||
|
|
||||||
|
"""
|
||||||
l = self.fail_tests.get(name, [])
|
l = self.fail_tests.get(name, [])
|
||||||
l.append(change)
|
l.append(change)
|
||||||
self.fail_tests[name] = l
|
self.fail_tests[name] = l
|
||||||
|
@ -657,6 +698,13 @@ class RecordingLaunchServer(zuul.launcher.server.LaunchServer):
|
||||||
return False
|
return False
|
||||||
|
|
||||||
def release(self, regex=None):
|
def release(self, regex=None):
|
||||||
|
"""Release a held build.
|
||||||
|
|
||||||
|
:arg str regex: A regular expression which, if supplied, will
|
||||||
|
cause only builds with matching names to be released. If
|
||||||
|
not supplied, all builds will be released.
|
||||||
|
|
||||||
|
"""
|
||||||
builds = self.running_builds[:]
|
builds = self.running_builds[:]
|
||||||
self.log.debug("Releasing build %s (%s)" % (regex,
|
self.log.debug("Releasing build %s (%s)" % (regex,
|
||||||
len(self.running_builds)))
|
len(self.running_builds)))
|
||||||
|
@ -705,6 +753,17 @@ class RecordingLaunchServer(zuul.launcher.server.LaunchServer):
|
||||||
|
|
||||||
|
|
||||||
class FakeGearmanServer(gear.Server):
|
class FakeGearmanServer(gear.Server):
|
||||||
|
"""A Gearman server for use in tests.
|
||||||
|
|
||||||
|
:ivar bool hold_jobs_in_queue: If true, submitted jobs will be
|
||||||
|
added to the queue but will not be distributed to workers
|
||||||
|
until released. This attribute may be changed at any time and
|
||||||
|
will take effect for subsequently enqueued jobs, but
|
||||||
|
previously held jobs will still need to be explicitly
|
||||||
|
released.
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
self.hold_jobs_in_queue = False
|
self.hold_jobs_in_queue = False
|
||||||
super(FakeGearmanServer, self).__init__(0)
|
super(FakeGearmanServer, self).__init__(0)
|
||||||
|
@ -729,6 +788,12 @@ class FakeGearmanServer(gear.Server):
|
||||||
return None
|
return None
|
||||||
|
|
||||||
def release(self, regex=None):
|
def release(self, regex=None):
|
||||||
|
"""Release a held job.
|
||||||
|
|
||||||
|
:arg str regex: A regular expression which, if supplied, will
|
||||||
|
cause only jobs with matching names to be released. If
|
||||||
|
not supplied, all jobs will be released.
|
||||||
|
"""
|
||||||
released = False
|
released = False
|
||||||
qlen = (len(self.high_queue) + len(self.normal_queue) +
|
qlen = (len(self.high_queue) + len(self.normal_queue) +
|
||||||
len(self.low_queue))
|
len(self.low_queue))
|
||||||
|
@ -846,6 +911,52 @@ class BaseTestCase(testtools.TestCase):
|
||||||
|
|
||||||
|
|
||||||
class ZuulTestCase(BaseTestCase):
|
class ZuulTestCase(BaseTestCase):
|
||||||
|
"""A test case with a functioning Zuul.
|
||||||
|
|
||||||
|
The following class variables are used during test setup and can
|
||||||
|
be overidden by subclasses but are effectively read-only once a
|
||||||
|
test method starts running:
|
||||||
|
|
||||||
|
:cvar str config_file: This points to the main zuul config file
|
||||||
|
within the fixtures directory. Subclasses may override this
|
||||||
|
to obtain a different behavior.
|
||||||
|
|
||||||
|
:cvar str tenant_config_file: This is the tenant config file
|
||||||
|
(which specifies from what git repos the configuration should
|
||||||
|
be loaded). It defaults to the value specified in
|
||||||
|
`config_file` but can be overidden by subclasses to obtain a
|
||||||
|
different tenant/project layout while using the standard main
|
||||||
|
configuration.
|
||||||
|
|
||||||
|
The following are instance variables that are useful within test
|
||||||
|
methods:
|
||||||
|
|
||||||
|
:ivar FakeGerritConnection fake_<connection>:
|
||||||
|
A :py:class:`~tests.base.FakeGerritConnection` will be
|
||||||
|
instantiated for each connection present in the config file
|
||||||
|
and stored here. For instance, `fake_gerrit` will hold the
|
||||||
|
FakeGerritConnection object for a connection named `gerrit`.
|
||||||
|
|
||||||
|
:ivar FakeGearmanServer gearman_server: An instance of
|
||||||
|
:py:class:`~tests.base.FakeGearmanServer` which is the Gearman
|
||||||
|
server that all of the Zuul components in this test use to
|
||||||
|
communicate with each other.
|
||||||
|
|
||||||
|
:ivar RecordingLaunchServer launch_server: An instance of
|
||||||
|
:py:class:`~tests.base.RecordingLaunchServer` which is the
|
||||||
|
Ansible launch server used to run jobs for this test.
|
||||||
|
|
||||||
|
:ivar list builds: A list of :py:class:`~tests.base.FakeBuild` objects
|
||||||
|
representing currently running builds. They are appended to
|
||||||
|
the list in the order they are launched, and removed from this
|
||||||
|
list upon completion.
|
||||||
|
|
||||||
|
:ivar list history: A list of :py:class:`~tests.base.BuildHistory`
|
||||||
|
objects representing completed builds. They are appended to
|
||||||
|
the list in the order they complete.
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
config_file = 'zuul.conf'
|
config_file = 'zuul.conf'
|
||||||
run_ansible = False
|
run_ansible = False
|
||||||
|
|
||||||
|
@ -898,8 +1009,6 @@ class ZuulTestCase(BaseTestCase):
|
||||||
self.init_repo("org/experimental-project")
|
self.init_repo("org/experimental-project")
|
||||||
self.init_repo("org/no-jobs-project")
|
self.init_repo("org/no-jobs-project")
|
||||||
|
|
||||||
self.setup_repos()
|
|
||||||
|
|
||||||
self.statsd = FakeStatsd()
|
self.statsd = FakeStatsd()
|
||||||
# note, use 127.0.0.1 rather than localhost to avoid getting ipv6
|
# note, use 127.0.0.1 rather than localhost to avoid getting ipv6
|
||||||
# see: https://github.com/jsocol/pystatsd/issues/61
|
# see: https://github.com/jsocol/pystatsd/issues/61
|
||||||
|
@ -1044,7 +1153,9 @@ class ZuulTestCase(BaseTestCase):
|
||||||
'_legacy_smtp', dict(self.config.items('smtp')))
|
'_legacy_smtp', dict(self.config.items('smtp')))
|
||||||
|
|
||||||
def setup_config(self):
|
def setup_config(self):
|
||||||
"""Per test config object. Override to set different config."""
|
# This creates the per-test configuration object. It can be
|
||||||
|
# overriden by subclasses, but should not need to be since it
|
||||||
|
# obeys the config_file and tenant_config_file attributes.
|
||||||
self.config = ConfigParser.ConfigParser()
|
self.config = ConfigParser.ConfigParser()
|
||||||
self.config.read(os.path.join(FIXTURE_DIR, self.config_file))
|
self.config.read(os.path.join(FIXTURE_DIR, self.config_file))
|
||||||
if hasattr(self, 'tenant_config_file'):
|
if hasattr(self, 'tenant_config_file'):
|
||||||
|
@ -1075,10 +1186,6 @@ class ZuulTestCase(BaseTestCase):
|
||||||
self.addCommitToRepo(project, 'add content from fixture',
|
self.addCommitToRepo(project, 'add content from fixture',
|
||||||
files, branch='master', tag='init')
|
files, branch='master', tag='init')
|
||||||
|
|
||||||
def setup_repos(self):
|
|
||||||
"""Subclasses can override to manipulate repos before tests"""
|
|
||||||
pass
|
|
||||||
|
|
||||||
def assertFinalState(self):
|
def assertFinalState(self):
|
||||||
# Make sure that git.Repo objects have been garbage collected.
|
# Make sure that git.Repo objects have been garbage collected.
|
||||||
repos = []
|
repos = []
|
||||||
|
|
Loading…
Reference in New Issue