openstack/rally
Code Issues Proposed changes

Prepare instances for SSH access

Browse Source 
This patchset is in preparation for a scenario for running a
test/benchmark within an instance.

Change-Id: I94276237fb5b453b194e58ed00df589a3420a6ed
This commit is contained in:
Hugh Saunders
2013-12-24 20:29:48 +00:00
parent 3f5db4a953
commit 94415a3f40
8 changed files with 161 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
rally/benchmark/scenarios/nova/utils.py
View File

@@ -38,8 +38,15 @@ class NovaScenario(base.Scenario):
:returns: Created server object
"""
server = cls.clients("nova").servers.create(server_name, image_id,
flavor_id, **kwargs)
if 'security_groups' not in kwargs:
kwargs['security_groups'] = ['rally_open']
else:
if 'rally_open' not in kwargs['security_groups']:
kwargs['security_groups'].append('rally_open')
server = cls.clients("nova").servers.create(
server_name, image_id, flavor_id, **kwargs)
# NOTE(msdubov): It is reasonable to wait 5 secs before starting to
# check whether the server is ready => less API calls.
time.sleep(5)

59
rally/benchmark/utils.py
View File

@@ -112,6 +112,65 @@ def create_openstack_clients(users_endpoints, keys):
)) for cl in client_managers
]
_prepare_for_instance_ssh(clients)
return clients
def _prepare_for_instance_ssh(clients):
"""Generate and store SSH keys, allow access to port 22.
In order to run tests on instances it is necessary to have SSH access.
This function generates an SSH key pair per user which is stored in the
clients dictionary. The public key is also submitted to nova via the
novaclient.
A security group rule is created to allow access to instances on port 22.
"""
for client_dict in clients:
nova_client = client_dict['nova']
if ('rally_ssh_key' not in
[k.name for k in nova_client.keypairs.list()]):
keypair = nova_client.keypairs.create('rally_ssh_key')
client_dict['ssh_key_pair'] = dict(private=keypair.private_key,
public=keypair.public_key)
if 'rally_open' not in [sg.name for sg in
nova_client.security_groups.list()]:
rally_open = nova_client.security_groups.create(
'rally_open',
'Allow all access to VMs for benchmarking'
)
rally_open = nova_client.security_groups.find(name='rally_open')
rules_to_add = [dict(ip_protocol='tcp',
to_port=65536,
from_port=1,
ip_range=dict(cidr='0.0.0.0/0')),
dict(ip_protocol='udp',
to_port=65536,
from_port=1,
ip_range=dict(cidr='0.0.0.0/0')),
dict(ip_protocol='icmp',
to_port=255,
from_port=-1,
ip_range=dict(cidr='0.0.0.0/0'))
]
def rule_match(criteria, existing_rule):
return all(existing_rule[key] == value
for key, value in criteria.iteritems())
for new_rule in rules_to_add:
if not any(rule_match(new_rule, existing_rule) for existing_rule
in rally_open.rules):
nova_client.security_group_rules.create(
rally_open.id,
from_port=new_rule['from_port'],
to_port=new_rule['to_port'],
ip_protocol=new_rule['ip_protocol'],
cidr=new_rule['ip_range']['cidr'])
return clients

39
rally/sshutils.py
View File

@@ -20,6 +20,7 @@ import random
import select
import socket
import string
from StringIO import StringIO
import time
from rally import exceptions
@@ -31,27 +32,46 @@ LOG = logging.getLogger(__name__)
class SSH(object):
"""SSH common functions."""
STDOUT_INDEX = 0
STDERR_INDEX = 1
def __init__(self, ip, user, port=22, key=None, timeout=1800):
def __init__(self, ip, user, port=22, key=None, key_type="file",
timeout=1800):
"""Initialize SSH client with ip, username and the default values.
timeout - the timeout for execution of the command
key - path to private key file, or string containing actual key
key_type - "file" for key path, "string" for actual key
"""
self.ip = ip
self.port = port
self.user = user
self.timeout = timeout
self.client = None
if key:
self.key = key
else:
self.key = key
self.key_type = key_type
if not self.key:
#Guess location of user's private key if no key is specified.
self.key = os.path.expanduser('~/.ssh/id_rsa')
def _get_ssh_connection(self):
self.client = paramiko.SSHClient()
self.client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
self.client.connect(self.ip, username=self.user,
key_filename=self.key, port=self.port)
connect_params = {
'hostname': self.ip,
'port': self.port,
'username': self.user
}
# NOTE(hughsaunders): Set correct paramiko parameter names for each
# method of supplying a key.
if self.key_type == 'file':
connect_params['key_filename'] = self.key
else:
connect_params['pkey'] = paramiko.RSAKey(
file_obj=StringIO(self.key))
self.client.connect(**connect_params)
def _is_timed_out(self, start_time):
return (time.time() - self.timeout) > start_time
@@ -136,14 +156,17 @@ class SSH(object):
ftp.put(os.path.expanduser(source), destination)
ftp.close()
def execute_script(self, script, enterpreter='/bin/sh'):
def execute_script(self, script, interpreter='/bin/sh',
get_stdout=False, get_stderr=False):
"""Execute the specified local script on the remote server."""
destination = '/tmp/' + ''.join(
random.choice(string.lowercase) for i in range(16))
self.upload(script, destination)
self.execute('%s %s' % (enterpreter, destination))
streams = self.execute('%s %s' % (interpreter, destination),
get_stdout=get_stdout, get_stderr=get_stderr)
self.execute('rm %s' % destination)
return streams
def wait(self, timeout=120, interval=1):
"""Wait for the host will be available via ssh."""

2
tests/benchmark/scenarios/nova/test_utils.py
View File

@@ -58,7 +58,7 @@ class NovaScenarioTestCase(test.TestCase):
fc.get_nova_client = lambda: fake_nova
fsm = fakes.FakeServerManager(fake_nova.images)
fake_server = fsm.create("s1", "i1", 1)
fsm.create = lambda name, iid, fid: fake_server
fsm.create = lambda name, iid, fid, **kwargs: fake_server
fake_nova.servers = fsm
fake_image_id = fsm.create_image(fake_server, 'img')
fake_image = fsm.images.get(fake_image_id)

10
tests/benchmark/test_runner.py
View File

@@ -62,9 +62,10 @@ class ScenarioTestCase(test.TestCase):
endpoints = srunner._create_temp_tenants_and_users(
tenants, users_per_tenant)
self.assertEqual(len(endpoints), tenants * users_per_tenant)
endpoint_keys = set(["username", "password", "tenant_name", "uri"])
endpoint_keys = set(["username", "password", "tenant_name",
"uri"])
for endpoint in endpoints:
self.assertEqual(set(endpoint.keys()), endpoint_keys)
self.assertTrue(endpoint_keys.issubset(endpoint.keys()))
def test_run_scenario(self):
with mock.patch("rally.benchmark.utils.osclients") as mock_osclients:
@@ -98,7 +99,10 @@ class ScenarioTestCase(test.TestCase):
@mock.patch("rally.benchmark.utils.osclients")
@mock.patch("multiprocessing.pool.IMapIterator.next")
@mock.patch("rally.benchmark.runner.time.time")
def test_run_scenario_timeout(self, mock_time, mock_next, mock_osclients):
@mock.patch("rally.benchmark.utils._prepare_for_instance_ssh")
def test_run_scenario_timeout(self, mock_prepare_for_instance_ssh,
mock_time, mock_next, mock_osclients):
mock_time.side_effect = [1, 2, 3, 10]
mock_next.side_effect = multiprocessing.TimeoutError()
mock_osclients.Clients.return_value = fakes.FakeClients()

47
tests/fakes.py
View File

@@ -79,7 +79,17 @@ class FakeKeypair(FakeResource):
class FakeSecurityGroup(FakeResource):
pass
def __init__(self, manager=None):
super(FakeSecurityGroup, self).__init__(manager)
self.rules = []
class FakeSecurityGroupRule(FakeResource):
def __init__(self, name, **kwargs):
super(FakeSecurityGroupRule, self).__init__(name)
for key, value in kwargs.items():
setattr(self, key, value)
class FakeVolume(FakeResource):
@@ -126,6 +136,16 @@ class FakeManager(object):
resources.append(self.cache[uuid])
return resources
def find(self, **kwargs):
for resource in self.cache.values():
match = True
for key, value in kwargs.items():
if getattr(resource, key, None) != value:
match = False
break
if match:
return resource
class FakeServerManager(FakeManager):
@@ -145,7 +165,7 @@ class FakeServerManager(FakeManager):
server.name = name
return server
def create(self, name, image_id, flavor_id):
def create(self, name, image_id, flavor_id, **kwargs):
return self._create(name=name)
def create_image(self, server, name):
@@ -161,7 +181,7 @@ class FakeServerManager(FakeManager):
class FakeFailedServerManager(FakeServerManager):
def create(self, name, image_id, flavor_id):
def create(self, name, image_id, flavor_id, **kwargs):
return self._create(FakeFailedServer, name)
@@ -198,7 +218,7 @@ class FakeNetworkManager(FakeManager):
class FakeKeypairManager(FakeManager):
def create(self, name):
def create(self, name, public_key=None):
kp = FakeKeypair(self)
kp.name = name or kp.name
return self._cache(kp)
@@ -206,12 +226,28 @@ class FakeKeypairManager(FakeManager):
class FakeSecurityGroupManager(FakeManager):
def create(self, name):
def __init__(self):
super(FakeSecurityGroupManager, self).__init__()
self.create('default')
def create(self, name, description=""):
sg = FakeSecurityGroup(self)
sg.name = name or sg.name
sg.description = description
return self._cache(sg)
class FakeSecurityGroupRuleManager(FakeManager):
def __init__(self):
super(FakeSecurityGroupRuleManager, self).__init__()
def create(self, name, **kwargs):
sgr = FakeSecurityGroupRule(self, **kwargs)
sgr.name = name or sgr.name
return self._cache(sgr)
class FakeUsersManager(FakeManager):
def create(self, username, password, email, tenant_id):
@@ -292,6 +328,7 @@ class FakeNovaClient(object):
self.networks = FakeNetworkManager()
self.keypairs = FakeKeypairManager()
self.security_groups = FakeSecurityGroupManager()
self.security_group_rules = FakeSecurityGroupRuleManager()
class FakeKeystoneClient(object):

9
tests/serverprovider/providers/test_virsh.py
View File

@@ -16,6 +16,7 @@
import jsonschema
import mock
import netaddr
import os
from rally.openstack.common.fixture import mockpatch
from rally.openstack.common import test
@@ -43,15 +44,17 @@ class VirshProviderTestCase(test.BaseTestCase):
mock_subp.check_output.return_value = '10.0.0.1'
mock_ipaddress.return_value = '10.0.0.2'
server = self.provider.create_vm('name')
script_path = '%(virsh_path)s/virsh/get_domain_ip.sh' % dict(
virsh_path=os.path.split(virsh.__file__)[0])
mock_subp.assert_has_calls([
mock.call.check_call('virt-clone --connect=qemu+ssh://user@host/'
'system -o prefix -n name --auto-clone',
shell=True),
mock.call.check_call('virsh --connect=qemu+ssh://user@host/system '
'start name', shell=True),
mock.call.check_call('scp -o StrictHostKeyChecking=no rally/serve'
'rprovider/providers/virsh/get_domain_ip.sh u'
'ser@host:~/get_domain_ip.sh', shell=True),
mock.call.check_call('scp -o StrictHostKeyChecking=no %s u'
'ser@host:~/get_domain_ip.sh' % script_path,
shell=True),
mock.call.check_output('ssh -o StrictHostKeyChecking=no user@host '
'./get_domain_ip.sh name', shell=True),
])

9
tests/test_sshutils.py
View File

@@ -77,7 +77,7 @@ class SSHTestCase(test.TestCase):
self.ssh.upload('/tmp/s', '/tmp/d')
expected = [mock.call.set_missing_host_key_policy(self.policy),
mock.call.connect('example.net', username='root',
mock.call.connect(hostname='example.net', username='root',
key_filename=os.path.expanduser(
'~/.ssh/id_rsa'), port=22),
mock.call.open_sftp(),
@@ -94,8 +94,11 @@ class SSHTestCase(test.TestCase):
self.ssh.execute_script('/bin/script')
up.assert_called_once_with('/bin/script', '/tmp/aaaaaaaaaaaaaaaa')
ex.assert_has_calls([mock.call('/bin/sh /tmp/aaaaaaaaaaaaaaaa'),
mock.call('rm /tmp/aaaaaaaaaaaaaaaa')])
ex.assert_has_calls([
mock.call('/bin/sh /tmp/aaaaaaaaaaaaaaaa',
get_stderr=False, get_stdout=False),
mock.call('rm /tmp/aaaaaaaaaaaaaaaa')
])
@mock.patch('rally.sshutils.SSH.execute')
def test_wait(self, ex):