Benchmark scenarios for Rally: run task in instance
This patch allows a user to define a scenario where Nova instances are created, then a script which outputs arbitrary JSON is run, and then the instance is destroyed. This allows tests/benchmarks from the instance perspective, as well as from OS APIs. blueprint benchmark-scenarios Change-Id: I4b1f77f5f8c55fe1da41b6ab0b23eb03303ba95f
This commit is contained in:
parent
22ca29d22e
commit
de35f19279
doc/samples
rally
tests/benchmark
12
doc/samples/support/instance_dd_test.sh
Normal file
12
doc/samples/support/instance_dd_test.sh
Normal file
@ -0,0 +1,12 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
time_seconds(){ (time -p $1 ) 2>&1 |awk '/real/{print $2}'; }
|
||||||
|
file=/tmp/test.img
|
||||||
|
c=1000 #1GB
|
||||||
|
write_seq_1gb=$(time_seconds "dd if=/dev/zero of=$file bs=1M count=$c")
|
||||||
|
read_seq_1gb=$(time_seconds "dd if=$file of=/dev/null bs=1M")
|
||||||
|
[ -f $file ] && rm $file
|
||||||
|
|
||||||
|
echo "{
|
||||||
|
\"write_seq_1gb\": $write_seq_1gb,
|
||||||
|
\"read_seq_1gb\": $read_seq_1gb
|
||||||
|
}"
|
10
doc/samples/tasks/nova/boot-runcommand-delete.json
Normal file
10
doc/samples/tasks/nova/boot-runcommand-delete.json
Normal file
@ -0,0 +1,10 @@
|
|||||||
|
{
|
||||||
|
"NovaServers.boot_runcommand_delete_server": [
|
||||||
|
{"args": {"flavor_id": "your flavor id here",
|
||||||
|
"image_id": "your image id here",
|
||||||
|
"script": "doc/samples/support/instance_dd_test.sh",
|
||||||
|
"interpreter": "/bin/sh",
|
||||||
|
"username": "ubuntu"},
|
||||||
|
"config": {"times": 2, "active_users": 1}}
|
||||||
|
]
|
||||||
|
}
|
@ -51,14 +51,16 @@ def _run_scenario_loop(args):
|
|||||||
cls.idle_time = 0
|
cls.idle_time = 0
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
scenario_output = None
|
||||||
with rutils.Timer() as timer:
|
with rutils.Timer() as timer:
|
||||||
getattr(cls, method_name)(**kwargs)
|
scenario_output = getattr(cls, method_name)(**kwargs)
|
||||||
error = None
|
error = None
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
error = utils.format_exc(e)
|
error = utils.format_exc(e)
|
||||||
finally:
|
finally:
|
||||||
return {"time": timer.duration() - cls.idle_time,
|
return {"time": timer.duration() - cls.idle_time,
|
||||||
"idle_time": cls.idle_time, "error": error}
|
"idle_time": cls.idle_time, "error": error,
|
||||||
|
"scenario_output": scenario_output}
|
||||||
|
|
||||||
|
|
||||||
class ScenarioRunner(object):
|
class ScenarioRunner(object):
|
||||||
|
@ -13,13 +13,20 @@
|
|||||||
# License for the specific language governing permissions and limitations
|
# License for the specific language governing permissions and limitations
|
||||||
# under the License.
|
# under the License.
|
||||||
|
|
||||||
|
import json
|
||||||
import jsonschema
|
import jsonschema
|
||||||
import random
|
import random
|
||||||
|
|
||||||
from rally.benchmark.scenarios.cinder import utils as cinder_utils
|
from rally.benchmark.scenarios.cinder import utils as cinder_utils
|
||||||
from rally.benchmark.scenarios.nova import utils
|
from rally.benchmark.scenarios.nova import utils
|
||||||
from rally.benchmark.scenarios import utils as scenario_utils
|
from rally.benchmark.scenarios import utils as scenario_utils
|
||||||
|
from rally.benchmark import utils as benchmark_utils
|
||||||
from rally import exceptions as rally_exceptions
|
from rally import exceptions as rally_exceptions
|
||||||
|
from rally.openstack.common.gettextutils import _ # noqa
|
||||||
|
from rally.openstack.common import log as logging
|
||||||
|
from rally import sshutils
|
||||||
|
|
||||||
|
LOG = logging.getLogger(__name__)
|
||||||
|
|
||||||
ACTION_BUILDER = scenario_utils.ActionBuilder(
|
ACTION_BUILDER = scenario_utils.ActionBuilder(
|
||||||
['hard_reboot', 'soft_reboot', 'stop_start', 'rescue_unrescue'])
|
['hard_reboot', 'soft_reboot', 'stop_start', 'rescue_unrescue'])
|
||||||
@ -53,6 +60,82 @@ class NovaServers(utils.NovaScenario,
|
|||||||
cls.sleep_between(min_sleep, max_sleep)
|
cls.sleep_between(min_sleep, max_sleep)
|
||||||
cls._delete_server(server)
|
cls._delete_server(server)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def boot_runcommand_delete_server(cls, image_id, flavor_id,
|
||||||
|
script, interpreter, network='private',
|
||||||
|
username='ubuntu', ip_version=4,
|
||||||
|
retries=60, port=22, **kwargs):
|
||||||
|
"""Boot server, run a script that outputs JSON, delete server.
|
||||||
|
|
||||||
|
Parameters:
|
||||||
|
script: script to run on the server, must output JSON mapping metric
|
||||||
|
names to values. See sample script below.
|
||||||
|
network: Network to choose address to connect to instance from
|
||||||
|
username: User to SSH to instance as
|
||||||
|
ip_version: Version of ip protocol to use for connection
|
||||||
|
|
||||||
|
returns: Dictionary containing two keys, data and errors. Data is JSON
|
||||||
|
data output by the script. Errors is raw data from the
|
||||||
|
script's standard error stream.
|
||||||
|
|
||||||
|
|
||||||
|
Example Script in doc/samples/support/instance_dd_test.sh
|
||||||
|
"""
|
||||||
|
server_name = cls._generate_random_name(16)
|
||||||
|
|
||||||
|
server = cls._boot_server(server_name, image_id, flavor_id,
|
||||||
|
key_name='rally_ssh_key', **kwargs)
|
||||||
|
|
||||||
|
if network not in server.addresses:
|
||||||
|
raise ValueError(
|
||||||
|
"Can't find cloud network %(network)s, so cannot boot "
|
||||||
|
"instance for Rally scenario boot-runcommand-delete. "
|
||||||
|
"Available networks: %(networks)s" % (
|
||||||
|
dict(network=network,
|
||||||
|
networks=server.addresses.keys()
|
||||||
|
)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
server_ip = [ip for ip in server.addresses[network] if
|
||||||
|
ip['version'] == ip_version][0]['addr']
|
||||||
|
ssh = sshutils.SSH(ip=server_ip, port=port, user=username,
|
||||||
|
key=cls.clients('ssh_key_pair')['private'],
|
||||||
|
key_type='string')
|
||||||
|
|
||||||
|
for retry in range(retries):
|
||||||
|
try:
|
||||||
|
LOG.debug(_('Execute script on server attempt '
|
||||||
|
'%(retry)i/%(retries)i') % dict(retry=retry,
|
||||||
|
retries=retries))
|
||||||
|
streams = list(ssh.execute_script(script=script,
|
||||||
|
interpreter=interpreter,
|
||||||
|
get_stdout=True,
|
||||||
|
get_stderr=True))
|
||||||
|
|
||||||
|
#NOTE(hughsaunders): Decode JSON script output
|
||||||
|
streams[sshutils.SSH.STDOUT_INDEX]\
|
||||||
|
= json.loads(streams[sshutils.SSH.STDOUT_INDEX])
|
||||||
|
break
|
||||||
|
except (rally_exceptions.SSHError,
|
||||||
|
rally_exceptions.TimeoutException, IOError) as e:
|
||||||
|
LOG.debug(_('Error running script on instance via SSH. '
|
||||||
|
'%(id)s/%(ip)s Attempt:%(retry)i, '
|
||||||
|
'Error: %(error)s') % dict(
|
||||||
|
id=server.id, ip=server_ip, retry=retry,
|
||||||
|
error=benchmark_utils.format_exc(e)))
|
||||||
|
cls.sleep_between(5, 5)
|
||||||
|
except ValueError:
|
||||||
|
LOG.error(_('Script %(script)s did not output valid JSON. ')
|
||||||
|
% dict(script=script))
|
||||||
|
|
||||||
|
cls._delete_server(server)
|
||||||
|
LOG.debug(_('Output streams from in-instance script execution: '
|
||||||
|
'stdout: %(stdout)s, stderr: $(stderr)s') % dict(
|
||||||
|
stdout=str(streams[sshutils.SSH.STDOUT_INDEX]),
|
||||||
|
stderr=str(streams[sshutils.SSH.STDERR_INDEX])))
|
||||||
|
return dict(data=streams[sshutils.SSH.STDOUT_INDEX],
|
||||||
|
errors=streams[sshutils.SSH.STDERR_INDEX])
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def boot_and_bounce_server(cls, image_id, flavor_id, **kwargs):
|
def boot_and_bounce_server(cls, image_id, flavor_id, **kwargs):
|
||||||
"""Tests booting a server then performing stop/start or hard/soft
|
"""Tests booting a server then performing stop/start or hard/soft
|
||||||
|
@ -192,6 +192,40 @@ class TaskCommands(object):
|
|||||||
table.add_row(['n/a', 'n/a', 'n/a', 0, len(raw)])
|
table.add_row(['n/a', 'n/a', 'n/a', 0, len(raw)])
|
||||||
print(table)
|
print(table)
|
||||||
|
|
||||||
|
#NOTE(hughsaunders): ssrs=scenario specific results
|
||||||
|
ssrs = []
|
||||||
|
for result in raw:
|
||||||
|
try:
|
||||||
|
ssrs.append(result['scenario_output']['data'])
|
||||||
|
except (KeyError, TypeError):
|
||||||
|
# No SSRs in this result
|
||||||
|
pass
|
||||||
|
if ssrs:
|
||||||
|
sys.stdout.flush()
|
||||||
|
keys = set()
|
||||||
|
for ssr in ssrs:
|
||||||
|
keys.update(ssr.keys())
|
||||||
|
|
||||||
|
ssr_table = prettytable.PrettyTable(
|
||||||
|
["Key", "max", "avg", "min"])
|
||||||
|
for key in keys:
|
||||||
|
values = [float(ssr[key]) for ssr in ssrs if key in ssr]
|
||||||
|
|
||||||
|
if values:
|
||||||
|
row = [str(key),
|
||||||
|
max(values),
|
||||||
|
sum(values) / len(values),
|
||||||
|
min(values)]
|
||||||
|
else:
|
||||||
|
row = [str(key)] + ['n/a'] * 3
|
||||||
|
ssr_table.add_row(row)
|
||||||
|
print("\nScenario Specific Results\n")
|
||||||
|
print(ssr_table)
|
||||||
|
|
||||||
|
for result in raw:
|
||||||
|
if result['scenario_output']['errors']:
|
||||||
|
print(result['scenario_output']['errors'])
|
||||||
|
|
||||||
@cliutils.args('--task-id', type=str, dest='task_id', help='uuid of task')
|
@cliutils.args('--task-id', type=str, dest='task_id', help='uuid of task')
|
||||||
@cliutils.args('--pretty', type=str, help=('pretty print (pprint) '
|
@cliutils.args('--pretty', type=str, help=('pretty print (pprint) '
|
||||||
'or json print (json)'))
|
'or json print (json)'))
|
||||||
|
@ -70,6 +70,51 @@ class NovaServersTestCase(test.TestCase):
|
|||||||
mock_sleep.assert_called_once_with(10, 20)
|
mock_sleep.assert_called_once_with(10, 20)
|
||||||
mock_delete.assert_called_once_with(fake_server)
|
mock_delete.assert_called_once_with(fake_server)
|
||||||
|
|
||||||
|
@mock.patch("json.loads")
|
||||||
|
@mock.patch("rally.benchmark.base.Scenario.clients")
|
||||||
|
@mock.patch("rally.sshutils.SSH.execute_script")
|
||||||
|
@mock.patch(NOVA_SERVERS + ".sleep_between")
|
||||||
|
@mock.patch(NOVA_SERVERS + "._generate_random_name")
|
||||||
|
@mock.patch(NOVA_SERVERS + "._delete_server")
|
||||||
|
@mock.patch(NOVA_SERVERS + "._boot_server")
|
||||||
|
def _verify_boot_runcommand_delete_server(
|
||||||
|
self, mock_boot, mock_delete, mock_random_name, mock_sleep,
|
||||||
|
mock_ssh_execute_script, mock_base_clients, mock_json_loads):
|
||||||
|
|
||||||
|
fake_server = fakes.FakeServer()
|
||||||
|
fake_server.addresses = dict(
|
||||||
|
private=[dict(
|
||||||
|
version=4,
|
||||||
|
addr="1.2.3.4"
|
||||||
|
)]
|
||||||
|
)
|
||||||
|
mock_boot.return_value = fake_server
|
||||||
|
mock_random_name.return_value = "random_name"
|
||||||
|
mock_ssh_execute_script.return_value = ('stdout', 'stderr')
|
||||||
|
mock_base_clients.return_value = dict(private='private-key-string')
|
||||||
|
|
||||||
|
servers.NovaServers.boot_runcommand_delete_server(
|
||||||
|
"img", 0, "script_path", "/bin/bash", fakearg="f")
|
||||||
|
|
||||||
|
mock_boot.assert_called_once_with(
|
||||||
|
"random_name", "img", 0, fakearg="f", key_name='rally_ssh_key')
|
||||||
|
mock_ssh_execute_script.assert_called_once_with(
|
||||||
|
script="script_path",
|
||||||
|
interpreter="/bin/bash",
|
||||||
|
get_stdout=True,
|
||||||
|
get_stderr=True
|
||||||
|
)
|
||||||
|
mock_json_loads.assert_called_once_with('stdout')
|
||||||
|
mock_delete.assert_called_once_with(fake_server)
|
||||||
|
|
||||||
|
fake_server.addresses = {}
|
||||||
|
self.assertRaises(
|
||||||
|
ValueError,
|
||||||
|
servers.NovaServers.boot_runcommand_delete_server,
|
||||||
|
"img", 0, "script_path", "/bin/bash",
|
||||||
|
fakearg="f"
|
||||||
|
)
|
||||||
|
|
||||||
@mock.patch(NOVA_SERVERS + "._generate_random_name")
|
@mock.patch(NOVA_SERVERS + "._generate_random_name")
|
||||||
@mock.patch(NOVA_SERVERS + "._boot_server")
|
@mock.patch(NOVA_SERVERS + "._boot_server")
|
||||||
@mock.patch("rally.benchmark.utils.osclients")
|
@mock.patch("rally.benchmark.utils.osclients")
|
||||||
@ -292,6 +337,9 @@ class NovaServersTestCase(test.TestCase):
|
|||||||
def test_boot_server_from_volume_and_delete(self):
|
def test_boot_server_from_volume_and_delete(self):
|
||||||
self._verify_boot_server_from_volume_and_delete()
|
self._verify_boot_server_from_volume_and_delete()
|
||||||
|
|
||||||
|
def test_boot_runcommand_delete_server(self):
|
||||||
|
self._verify_boot_runcommand_delete_server()
|
||||||
|
|
||||||
def test_boot_server_no_nics(self):
|
def test_boot_server_no_nics(self):
|
||||||
self._verify_boot_server(nic=None, assert_nic=False)
|
self._verify_boot_server(nic=None, assert_nic=False)
|
||||||
|
|
||||||
|
@ -82,7 +82,8 @@ class ScenarioTestCase(test.TestCase):
|
|||||||
{"times": times,
|
{"times": times,
|
||||||
"active_users": active_users,
|
"active_users": active_users,
|
||||||
"timeout": 2})
|
"timeout": 2})
|
||||||
expected = [{"time": 10, "idle_time": 0, "error": None}
|
expected = [{"time": 10, "idle_time": 0, "error": None,
|
||||||
|
"scenario_output": None}
|
||||||
for i in range(times)]
|
for i in range(times)]
|
||||||
self.assertEqual(results, expected)
|
self.assertEqual(results, expected)
|
||||||
|
|
||||||
@ -91,7 +92,8 @@ class ScenarioTestCase(test.TestCase):
|
|||||||
{"duration": duration,
|
{"duration": duration,
|
||||||
"active_users": active_users,
|
"active_users": active_users,
|
||||||
"timeout": 2})
|
"timeout": 2})
|
||||||
expected = [{"time": 10, "idle_time": 0, "error": None}
|
expected = [{"time": 10, "idle_time": 0, "error": None,
|
||||||
|
"scenario_output": None}
|
||||||
for i in range(active_users)]
|
for i in range(active_users)]
|
||||||
self.assertEqual(results, expected)
|
self.assertEqual(results, expected)
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user