openstack/rally
Code Issues Proposed changes

Rally cli functional testing

Browse Source 
Added test for:
    'rally deployment'
    'rally show'
    'rally info'
    'rally task'

  test_cli was parted

  blueprint tests-integrated

Change-Id: If9febf155d8178b5c941317d2a7b837bbce128f1
This commit is contained in:
Oleh Anufriiev 2014-09-01 17:43:01 +03:00
parent 130b355061
commit 94dd8648b8
5 changed files with 297 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

65
tests_ci/test_cli_deployment.py Normal file
View File

@ -0,0 +1,65 @@
# Copyright 2013: Mirantis Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import json
import unittest
import mock
import test_cli_utils as utils
class DeploymentTestCase(unittest.TestCase):
def setUp(self):
super(DeploymentTestCase, self).setUp()
self.rally = utils.Rally()
def test_create_fromenv_list_endpoint(self):
with mock.patch.dict("os.environ", utils.TEST_ENV):
self.rally("deployment create --name t_create_env --fromenv")
self.assertIn("t_create_env", self.rally("deployment list"))
self.assertIn(utils.TEST_ENV["OS_AUTH_URL"],
self.rally("deployment endpoint"))
def test_create_fromfile(self):
fake_d_conf = "/tmp/.tmp.deployment"
self.rally("deployment create --name t_create_file --filename %s"
% fake_d_conf)
self.assertIn("t_create_file", self.rally("deployment list"))
def test_config(self):
fake_d_conf = "/tmp/.tmp.deployment"
self.rally("deployment create --name t_create_file --filename %s"
% fake_d_conf)
with open(fake_d_conf, "r") as conf:
self.assertDictEqual(json.loads(conf.read()),
json.loads(self.rally("deployment config")))
def test_destroy(self):
with mock.patch.dict("os.environ", utils.TEST_ENV):
self.rally("deployment create --name t_create_env --fromenv")
self.assertIn("t_create_env", self.rally("deployment list"))
self.rally("deployment destroy")
self.assertNotIn("t_create_env", self.rally("deployment list"))
def test_check_success(self):
self.assertTrue(self.rally("deployment check"))
def test_check_fail(self):
with mock.patch.dict("os.environ", utils.TEST_ENV):
self.rally("deployment create --name t_create_env --fromenv")
self.assertRaises(utils.RallyCmdError, self.rally,
("deployment check"))

41
tests_ci/test_cli_info.py Normal file
View File

@ -0,0 +1,41 @@
# Copyright 2013: Mirantis Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import unittest
import test_cli_utils as utils
class InfoTestCase(unittest.TestCase):
def setUp(self):
super(InfoTestCase, self).setUp()
self.rally = utils.Rally()
def test_find_scenario_group(self):
self.assertIn("(benchmark scenario group)",
self.rally("info find Dummy"))
def test_find_scenario(self):
self.assertIn("(benchmark scenario)", self.rally("info find dummy"))
def test_find_deployment_engine(self):
marker_string = "ExistingCloud (deploy engine)."
self.assertIn(marker_string, self.rally("info find ExistingCloud"))
def test_find_server_provider(self):
marker_string = "ExistingServers (server provider)."
self.assertIn(marker_string, self.rally("info find ExistingServers"))

46
tests_ci/test_cli_show.py Normal file
View File

@ -0,0 +1,46 @@
# Copyright 2013: Mirantis Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import unittest
import test_cli_utils as utils
class ShowTestCase(unittest.TestCase):
def setUp(self):
super(ShowTestCase, self).setUp()
self.rally = utils.Rally()
def test_show_images(self):
res = self.rally("show images")
self.assertIn("cirros", res)
def test_show_flavors(self):
res = self.rally("show flavors")
self.assertIn("m1.tiny", res)
def test_show_networks(self):
res = self.rally("show networks")
self.assertIn("private", res)
def test_show_secgroups(self):
res = self.rally("show secgroups")
self.assertIn("default", res)
# TODO(oanufriev): implement after bp/add-rally-create-cli-command
def test_show_keypairs(self):
pass

131
tests_ci/test_cli_task.py Normal file
View File

@ -0,0 +1,131 @@
# Copyright 2013: Mirantis Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
import unittest
import test_cli_utils as utils
class TaskTestCase(unittest.TestCase):
def _get_sample_task_config(self):
return {
"KeystoneBasic.create_and_list_users": [
{
"args": {
"name_length": 10
},
"runner": {
"type": "constant",
"times": 5,
"concurrency": 5
}
}
]
}
def test_status(self):
rally = utils.Rally()
cfg = self._get_sample_task_config()
config = utils.TaskConfig(cfg)
rally("task start --task %s" % config.filename)
self.assertIn("finished", rally("task status"))
def test_detailed(self):
rally = utils.Rally()
cfg = self._get_sample_task_config()
config = utils.TaskConfig(cfg)
rally("task start --task %s" % config.filename)
self.assertIn("KeystoneBasic.create_and_list_users",
rally("task detailed"))
def test_results(self):
rally = utils.Rally()
cfg = self._get_sample_task_config()
config = utils.TaskConfig(cfg)
rally("task start --task %s" % config.filename)
self.assertIn("result", rally("task results"))
def test_plot2html(self):
rally = utils.Rally()
cfg = self._get_sample_task_config()
config = utils.TaskConfig(cfg)
rally("task start --task %s" % config.filename)
if os.path.exists("/tmp/test_plot.html"):
os.remove("/tmp/test_plot.html")
rally("task plot2html /tmp/test_plot")
self.assertTrue(os.path.exists("/tmp/test_plot.html"))
def test_delete(self):
rally = utils.Rally()
cfg = self._get_sample_task_config()
config = utils.TaskConfig(cfg)
rally("task start --task %s" % config.filename)
self.assertIn("finished", rally("task status"))
rally("task delete")
self.assertNotIn("finishe", rally("task list"))
# NOTE(oanufriev): Not implemented
def test_abort(self):
pass
class SLATestCase(unittest.TestCase):
def _get_sample_task_config(self, max_seconds_per_iteration=4,
max_failure_percent=0):
return {
"KeystoneBasic.create_and_list_users": [
{
"args": {
"name_length": 10
},
"runner": {
"type": "constant",
"times": 5,
"concurrency": 5
},
"sla": {
"max_seconds_per_iteration": max_seconds_per_iteration,
"max_failure_percent": max_failure_percent,
}
}
]
}
def test_sla_fail(self):
rally = utils.Rally()
cfg = self._get_sample_task_config(max_seconds_per_iteration=0.001)
config = utils.TaskConfig(cfg)
rally("task start --task %s" % config.filename)
self.assertRaises(utils.RallyCmdError, rally, "task sla_check")
def test_sla_success(self):
rally = utils.Rally()
config = utils.TaskConfig(self._get_sample_task_config())
rally("task start --task %s" % config.filename)
rally("task sla_check")
expected = [
{"benchmark": "KeystoneBasic.create_and_list_users",
"criterion": "max_seconds_per_iteration",
"pos": 0, "success": True},
{"benchmark": "KeystoneBasic.create_and_list_users",
"criterion": "max_failure_percent",
"pos": 0, "success": True},
]
data = rally("task sla_check --json", getjson=True)
self.assertEqual(expected, data)

86
tests_ci/test_cli.py → tests_ci/test_cli_utils.py
View File

@ -20,24 +20,13 @@ import pwd
import shutil
import subprocess
import tempfile
import unittest
import mock
"""Test rally command line interface.
This module is intended for running by OpenStack CI system.
To start tests manually please use
$ tox -ecli
"""
TEST_ENV = {
"OS_USERNAME": "admin",
"OS_PASSWORD": "admin",
"OS_TENANT_NAME": "admin",
"OS_AUTH_URL": "http://fake/",
"OS_USERNAME": "admin",
"OS_PASSWORD": "admin",
"OS_TENANT_NAME": "admin",
"OS_AUTH_URL": "http://fake/",
}
@ -94,6 +83,16 @@ class Rally(object):
subprocess.call(["rally-manage", "--config-file", config_filename,
"db", "recreate"])
self("deployment create --file /tmp/.rd.json --name MAIN")
with open("/tmp/.tmp.deployment", "w") as d_conf:
d_conf.write(
"""{
"type": "ExistingCloud",
"auth_url": "http://fake/",
"admin": {
"username": "admin",
"password": "admin",
"tenant_name": "admin"
}\n}""")
def __del__(self):
shutil.rmtree(self.tmp_dir)
@ -109,60 +108,3 @@ class Rally(object):
return output
except subprocess.CalledProcessError as e:
raise RallyCmdError(e.returncode, e.output)
class DeploymentTestCase(unittest.TestCase):
def test_create_fromenv_list_endpoint(self):
rally = Rally()
with mock.patch.dict("os.environ", TEST_ENV):
rally("deployment create --name t_create --fromenv")
self.assertIn("t_create", rally("deployment list"))
self.assertIn(TEST_ENV["OS_AUTH_URL"], rally("deployment endpoint"))
class SLATestCase(unittest.TestCase):
def _get_sample_task_config(self, max_seconds_per_iteration=4,
max_failure_percent=0):
return {
"KeystoneBasic.create_and_list_users": [
{
"args": {
"name_length": 10
},
"runner": {
"type": "constant",
"times": 5,
"concurrency": 5
},
"sla": {
"max_seconds_per_iteration": max_seconds_per_iteration,
"max_failure_percent": max_failure_percent,
}
}
]
}
def test_sla_fail(self):
rally = Rally()
cfg = self._get_sample_task_config(max_seconds_per_iteration=0.001)
config = TaskConfig(cfg)
rally("task start --task %s" % config.filename)
self.assertRaises(RallyCmdError, rally, "task sla_check")
def test_sla_success(self):
rally = Rally()
config = TaskConfig(self._get_sample_task_config())
rally("task start --task %s" % config.filename)
rally("task sla_check")
expected = [
{"benchmark": "KeystoneBasic.create_and_list_users",
"criterion": "max_seconds_per_iteration",
"pos": 0, "success": True},
{"benchmark": "KeystoneBasic.create_and_list_users",
"criterion": "max_failure_percent",
"pos": 0, "success": True},
]
data = rally("task sla_check --json", getjson=True)
self.assertEqual(expected, data)