openstack/rally
Code Issues Proposed changes

Benchmark scenarios for Rally: run task in instance

Browse Source 
This patch allows a user to define a scenario where Nova instances are
created, then a script which outputs arbitrary JSON is run, and then the
instance is destroyed.

This allows tests/benchmarks from the instance perspective, as well as
from OS APIs.

blueprint benchmark-scenarios

Change-Id: I4b1f77f5f8c55fe1da41b6ab0b23eb03303ba95f
This commit is contained in:
Hugh Saunders 2013-12-24 14:49:03 +00:00 committed by Gerrit Code Review
parent 22ca29d22e
commit de35f19279
7 changed files with 195 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
doc/samples/support/instance_dd_test.sh Normal file
View File

@ -0,0 +1,12 @@
#!/bin/sh
time_seconds(){ (time -p $1 ) 2>&1 |awk '/real/{print $2}'; }
file=/tmp/test.img
c=1000 #1GB
write_seq_1gb=$(time_seconds "dd if=/dev/zero of=$file bs=1M count=$c")
read_seq_1gb=$(time_seconds "dd if=$file of=/dev/null bs=1M")
[ -f $file ] && rm $file
echo "{
\"write_seq_1gb\": $write_seq_1gb,
\"read_seq_1gb\": $read_seq_1gb
}"

10
doc/samples/tasks/nova/boot-runcommand-delete.json Normal file
View File

@ -0,0 +1,10 @@
{
"NovaServers.boot_runcommand_delete_server": [
{"args": {"flavor_id": "your flavor id here",
"image_id": "your image id here",
"script": "doc/samples/support/instance_dd_test.sh",
"interpreter": "/bin/sh",
"username": "ubuntu"},
"config": {"times": 2, "active_users": 1}}
]
}

6
rally/benchmark/runner.py
View File

@ -51,14 +51,16 @@ def _run_scenario_loop(args):
cls.idle_time = 0
try:
scenario_output = None
with rutils.Timer() as timer:
getattr(cls, method_name)(**kwargs)
scenario_output = getattr(cls, method_name)(**kwargs)
error = None
except Exception as e:
error = utils.format_exc(e)
finally:
return {"time": timer.duration() - cls.idle_time,
"idle_time": cls.idle_time, "error": error}
"idle_time": cls.idle_time, "error": error,
"scenario_output": scenario_output}
class ScenarioRunner(object):

83
rally/benchmark/scenarios/nova/servers.py
View File

@ -13,13 +13,20 @@
# License for the specific language governing permissions and limitations
# under the License.
import json
import jsonschema
import random
from rally.benchmark.scenarios.cinder import utils as cinder_utils
from rally.benchmark.scenarios.nova import utils
from rally.benchmark.scenarios import utils as scenario_utils
from rally.benchmark import utils as benchmark_utils
from rally import exceptions as rally_exceptions
from rally.openstack.common.gettextutils import _ # noqa
from rally.openstack.common import log as logging
from rally import sshutils
LOG = logging.getLogger(__name__)
ACTION_BUILDER = scenario_utils.ActionBuilder(
['hard_reboot', 'soft_reboot', 'stop_start', 'rescue_unrescue'])
@ -53,6 +60,82 @@ class NovaServers(utils.NovaScenario,
cls.sleep_between(min_sleep, max_sleep)
cls._delete_server(server)
@classmethod
def boot_runcommand_delete_server(cls, image_id, flavor_id,
script, interpreter, network='private',
username='ubuntu', ip_version=4,
retries=60, port=22, **kwargs):
"""Boot server, run a script that outputs JSON, delete server.
Parameters:
script: script to run on the server, must output JSON mapping metric
names to values. See sample script below.
network: Network to choose address to connect to instance from
username: User to SSH to instance as
ip_version: Version of ip protocol to use for connection
returns: Dictionary containing two keys, data and errors. Data is JSON
data output by the script. Errors is raw data from the
script's standard error stream.
Example Script in doc/samples/support/instance_dd_test.sh
"""
server_name = cls._generate_random_name(16)
server = cls._boot_server(server_name, image_id, flavor_id,
key_name='rally_ssh_key', **kwargs)
if network not in server.addresses:
raise ValueError(
"Can't find cloud network %(network)s, so cannot boot "
"instance for Rally scenario boot-runcommand-delete. "
"Available networks: %(networks)s" % (
dict(network=network,
networks=server.addresses.keys()
)
)
)
server_ip = [ip for ip in server.addresses[network] if
ip['version'] == ip_version][0]['addr']
ssh = sshutils.SSH(ip=server_ip, port=port, user=username,
key=cls.clients('ssh_key_pair')['private'],
key_type='string')
for retry in range(retries):
try:
LOG.debug(_('Execute script on server attempt '
'%(retry)i/%(retries)i') % dict(retry=retry,
retries=retries))
streams = list(ssh.execute_script(script=script,
interpreter=interpreter,
get_stdout=True,
get_stderr=True))
#NOTE(hughsaunders): Decode JSON script output
streams[sshutils.SSH.STDOUT_INDEX]\
= json.loads(streams[sshutils.SSH.STDOUT_INDEX])
break
except (rally_exceptions.SSHError,
rally_exceptions.TimeoutException, IOError) as e:
LOG.debug(_('Error running script on instance via SSH. '
'%(id)s/%(ip)s Attempt:%(retry)i, '
'Error: %(error)s') % dict(
id=server.id, ip=server_ip, retry=retry,
error=benchmark_utils.format_exc(e)))
cls.sleep_between(5, 5)
except ValueError:
LOG.error(_('Script %(script)s did not output valid JSON. ')
% dict(script=script))
cls._delete_server(server)
LOG.debug(_('Output streams from in-instance script execution: '
'stdout: %(stdout)s, stderr: $(stderr)s') % dict(
stdout=str(streams[sshutils.SSH.STDOUT_INDEX]),
stderr=str(streams[sshutils.SSH.STDERR_INDEX])))
return dict(data=streams[sshutils.SSH.STDOUT_INDEX],
errors=streams[sshutils.SSH.STDERR_INDEX])
@classmethod
def boot_and_bounce_server(cls, image_id, flavor_id, **kwargs):
"""Tests booting a server then performing stop/start or hard/soft

34
rally/cmd/main.py
View File

@ -192,6 +192,40 @@ class TaskCommands(object):
table.add_row(['n/a', 'n/a', 'n/a', 0, len(raw)])
print(table)
#NOTE(hughsaunders): ssrs=scenario specific results
ssrs = []
for result in raw:
try:
ssrs.append(result['scenario_output']['data'])
except (KeyError, TypeError):
# No SSRs in this result
pass
if ssrs:
sys.stdout.flush()
keys = set()
for ssr in ssrs:
keys.update(ssr.keys())
ssr_table = prettytable.PrettyTable(
["Key", "max", "avg", "min"])
for key in keys:
values = [float(ssr[key]) for ssr in ssrs if key in ssr]
if values:
row = [str(key),
max(values),
sum(values) / len(values),
min(values)]
else:
row = [str(key)] + ['n/a'] * 3
ssr_table.add_row(row)
print("\nScenario Specific Results\n")
print(ssr_table)
for result in raw:
if result['scenario_output']['errors']:
print(result['scenario_output']['errors'])
@cliutils.args('--task-id', type=str, dest='task_id', help='uuid of task')
@cliutils.args('--pretty', type=str, help=('pretty print (pprint) '
'or json print (json)'))

48
tests/benchmark/scenarios/nova/test_servers.py
View File

@ -70,6 +70,51 @@ class NovaServersTestCase(test.TestCase):
mock_sleep.assert_called_once_with(10, 20)
mock_delete.assert_called_once_with(fake_server)
@mock.patch("json.loads")
@mock.patch("rally.benchmark.base.Scenario.clients")
@mock.patch("rally.sshutils.SSH.execute_script")
@mock.patch(NOVA_SERVERS + ".sleep_between")
@mock.patch(NOVA_SERVERS + "._generate_random_name")
@mock.patch(NOVA_SERVERS + "._delete_server")
@mock.patch(NOVA_SERVERS + "._boot_server")
def _verify_boot_runcommand_delete_server(
self, mock_boot, mock_delete, mock_random_name, mock_sleep,
mock_ssh_execute_script, mock_base_clients, mock_json_loads):
fake_server = fakes.FakeServer()
fake_server.addresses = dict(
private=[dict(
version=4,
addr="1.2.3.4"
)]
)
mock_boot.return_value = fake_server
mock_random_name.return_value = "random_name"
mock_ssh_execute_script.return_value = ('stdout', 'stderr')
mock_base_clients.return_value = dict(private='private-key-string')
servers.NovaServers.boot_runcommand_delete_server(
"img", 0, "script_path", "/bin/bash", fakearg="f")
mock_boot.assert_called_once_with(
"random_name", "img", 0, fakearg="f", key_name='rally_ssh_key')
mock_ssh_execute_script.assert_called_once_with(
script="script_path",
interpreter="/bin/bash",
get_stdout=True,
get_stderr=True
)
mock_json_loads.assert_called_once_with('stdout')
mock_delete.assert_called_once_with(fake_server)
fake_server.addresses = {}
self.assertRaises(
ValueError,
servers.NovaServers.boot_runcommand_delete_server,
"img", 0, "script_path", "/bin/bash",
fakearg="f"
)
@mock.patch(NOVA_SERVERS + "._generate_random_name")
@mock.patch(NOVA_SERVERS + "._boot_server")
@mock.patch("rally.benchmark.utils.osclients")
@ -292,6 +337,9 @@ class NovaServersTestCase(test.TestCase):
def test_boot_server_from_volume_and_delete(self):
self._verify_boot_server_from_volume_and_delete()
def test_boot_runcommand_delete_server(self):
self._verify_boot_runcommand_delete_server()
def test_boot_server_no_nics(self):
self._verify_boot_server(nic=None, assert_nic=False)

6
tests/benchmark/test_runner.py
View File

@ -82,7 +82,8 @@ class ScenarioTestCase(test.TestCase):
{"times": times,
"active_users": active_users,
"timeout": 2})
expected = [{"time": 10, "idle_time": 0, "error": None}
expected = [{"time": 10, "idle_time": 0, "error": None,
"scenario_output": None}
for i in range(times)]
self.assertEqual(results, expected)
@ -91,7 +92,8 @@ class ScenarioTestCase(test.TestCase):
{"duration": duration,
"active_users": active_users,
"timeout": 2})
expected = [{"time": 10, "idle_time": 0, "error": None}
expected = [{"time": 10, "idle_time": 0, "error": None,
"scenario_output": None}
for i in range(active_users)]
self.assertEqual(results, expected)