openstack
/
fuel-qa
Code Issues Proposed changes

Merge "Devops client for fuelweb"

This commit is contained in:
Jenkins 2017-03-21 12:36:11 +00:00 committed by Gerrit Code Review
parent 49fc7e54ac eeee1e1ffd
commit eb0b1fdc2b
3 changed files with 51 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

73
fuelweb_test/models/environment.py
View File

@ -21,7 +21,6 @@ from devops.helpers.helpers import tcp_ping_
from devops.helpers.helpers import wait_pass from devops.helpers.helpers import wait_pass
from devops.helpers.helpers import wait from devops.helpers.helpers import wait
from devops.helpers.metaclasses import SingletonMeta from devops.helpers.metaclasses import SingletonMeta
from devops.models import Environment
from keystoneauth1 import exceptions from keystoneauth1 import exceptions
from proboscis.asserts import assert_equal from proboscis.asserts import assert_equal
from proboscis.asserts import assert_true from proboscis.asserts import assert_true
@ -104,7 +103,7 @@ class EnvironmentModel(six.with_metaclass(SingletonMeta, object)):
@logwrap @logwrap
def add_syslog_server(self, cluster_id, port=5514): def add_syslog_server(self, cluster_id, port=5514):
self.fuel_web.add_syslog_server( self.fuel_web.add_syslog_server(
cluster_id, self.d_env.router(), port) cluster_id, self.d_env.get_default_gw(), port)
def bootstrap_nodes(self, devops_nodes, timeout=settings.BOOTSTRAP_TIMEOUT, def bootstrap_nodes(self, devops_nodes, timeout=settings.BOOTSTRAP_TIMEOUT,
skip_timesync=False): skip_timesync=False):
@ -147,8 +146,8 @@ class EnvironmentModel(six.with_metaclass(SingletonMeta, object)):
logger.info("Please wait while time on nodes: {0} " logger.info("Please wait while time on nodes: {0} "
"will be synchronized" "will be synchronized"
.format(', '.join(sorted(nodes_names)))) .format(', '.join(sorted(nodes_names))))
denv = DevopsClient().get_env(self.d_env.name) new_time = self.d_env.sync_time(node_names=nodes_names,
new_time = denv.sync_time(node_names=nodes_names, skip_sync=skip_sync) skip_sync=skip_sync)
for name in sorted(new_time): for name in sorted(new_time):
logger.info("New time on '{0}' = {1}".format(name, new_time[name])) logger.info("New time on '{0}' = {1}".format(name, new_time[name]))
@ -169,7 +168,7 @@ class EnvironmentModel(six.with_metaclass(SingletonMeta, object)):
'iface': iface_alias('eth0'), 'iface': iface_alias('eth0'),
'ip': node.get_ip_address_by_network_name('admin'), 'ip': node.get_ip_address_by_network_name('admin'),
'mask': self.d_env.get_network(name='admin').ip.netmask, 'mask': self.d_env.get_network(name='admin').ip.netmask,
'gw': self.d_env.router(), 'gw': self.d_env.get_default_gw(),
'hostname': ''.join((settings.FUEL_MASTER_HOSTNAME, 'hostname': ''.join((settings.FUEL_MASTER_HOSTNAME,
settings.DNS_SUFFIX)), settings.DNS_SUFFIX)),
'nat_interface': '', 'nat_interface': '',
@ -229,20 +228,21 @@ class EnvironmentModel(six.with_metaclass(SingletonMeta, object)):
from devops.error import DevopsObjNotFound from devops.error import DevopsObjNotFound
EnvDoesNotExist = DevopsObjNotFound EnvDoesNotExist = DevopsObjNotFound
except ImportError: except ImportError:
from devops.models import Environment
# pylint: disable=no-member # pylint: disable=no-member
EnvDoesNotExist = Environment.DoesNotExist EnvDoesNotExist = Environment.DoesNotExist
# pylint: enable=no-member # pylint: enable=no-member
try: try:
logger.info("Try to find environment '{0}'".format(env_name)) logger.info("Try to find environment '{0}'".format(env_name))
self._virt_env = Environment.get(name=env_name) self._virt_env = DevopsClient().get_env(env_name)
except EnvDoesNotExist: except EnvDoesNotExist:
logger.info("Try to create environment '{0}'".format(env_name)) logger.info("Try to create environment '{0}'".format(env_name))
if self._config: if self._config:
self._virt_env = Environment.create_environment( self._virt_env = DevopsClient().create_env_from_config(
full_config=self._config) config=self._config)
else: else:
self._virt_env = Environment.describe_environment( self._virt_env = DevopsClient().create_env(
boot_from=settings.ADMIN_BOOT_DEVICE) boot_from=settings.ADMIN_BOOT_DEVICE)
self._virt_env.define() self._virt_env.define()
logger.info("New environment '{0}' was defined".format(env_name)) logger.info("New environment '{0}' was defined".format(env_name))
@ -348,9 +348,9 @@ class EnvironmentModel(six.with_metaclass(SingletonMeta, object)):
new_login = settings.SSH_FUEL_CREDENTIALS['login'] new_login = settings.SSH_FUEL_CREDENTIALS['login']
new_password = settings.SSH_FUEL_CREDENTIALS['password'] new_password = settings.SSH_FUEL_CREDENTIALS['password']
try: try:
self.ssh_manager.execute_on_remote( self.ssh_manager.check_call(
ip=self.ssh_manager.admin_ip, ip=self.ssh_manager.admin_ip,
cmd='date' command='date'
) )
logger.debug('Accessing admin node using SSH: SUCCESS') logger.debug('Accessing admin node using SSH: SUCCESS')
except Exception: except Exception:
@ -363,10 +363,10 @@ class EnvironmentModel(six.with_metaclass(SingletonMeta, object)):
slave_login=settings.SSH_SLAVE_CREDENTIALS['login'], slave_login=settings.SSH_SLAVE_CREDENTIALS['login'],
slave_password=settings.SSH_SLAVE_CREDENTIALS['password'] slave_password=settings.SSH_SLAVE_CREDENTIALS['password']
) )
self.ssh_manager.execute_on_remote( self.ssh_manager.check_call(
ip=self.ssh_manager.admin_ip, ip=self.ssh_manager.admin_ip,
cmd='echo -e "{1}\\n{1}" | passwd {0}'.format(new_login, command='echo -e "{1}\\n{1}" | passwd {0}'.format(new_login,
new_password) new_password)
) )
self.ssh_manager.initialize( self.ssh_manager.initialize(
admin_ip=self.ssh_manager.admin_ip, admin_ip=self.ssh_manager.admin_ip,
@ -389,14 +389,15 @@ class EnvironmentModel(six.with_metaclass(SingletonMeta, object)):
self.fuel_web.client.get_releases() self.fuel_web.client.get_releases()
# TODO(akostrikov) CENTOS7 except exceptions.Unauthorized: # TODO(akostrikov) CENTOS7 except exceptions.Unauthorized:
except: except:
self.ssh_manager.execute_on_remote( self.ssh_manager.check_call(
ip=self.ssh_manager.admin_ip, ip=self.ssh_manager.admin_ip,
cmd='fuel user --newpass {0} --change-password'.format( command='fuel user --newpass {0} --change-password'.format(
settings.KEYSTONE_CREDS['password']) settings.KEYSTONE_CREDS['password'])
) )
config_file = self.ssh_manager.execute_on_remote( config_file_path = 'ls -1 $HOME/.config/fuel/fuel_client.yaml'
config_file = self.ssh_manager.check_call(
ip=self.ssh_manager.admin_ip, ip=self.ssh_manager.admin_ip,
cmd='ls -1 $HOME/.config/fuel/fuel_client.yaml')['stdout_str'] command=config_file_path)['stdout_str']
with YamlEditor(config_file, ip=self.admin_node_ip) as editor: with YamlEditor(config_file, ip=self.admin_node_ip) as editor:
editor.content["OS_USERNAME"] = \ editor.content["OS_USERNAME"] = \
@ -458,7 +459,7 @@ class EnvironmentModel(six.with_metaclass(SingletonMeta, object)):
nessus_node.start() nessus_node.start()
# wait while installation complete # wait while installation complete
self.admin_actions.modify_configs(self.d_env.router()) self.admin_actions.modify_configs(self.d_env.get_default_gw())
if CUSTOM_FUEL_SETTING_YAML: if CUSTOM_FUEL_SETTING_YAML:
self.admin_actions.update_fuel_setting_yaml( self.admin_actions.update_fuel_setting_yaml(
CUSTOM_FUEL_SETTING_YAML) CUSTOM_FUEL_SETTING_YAML)
@ -471,19 +472,19 @@ class EnvironmentModel(six.with_metaclass(SingletonMeta, object)):
cmd = """ cmd = """
echo -e '"SSL":\n "force_https": "true"' >> /etc/fuel/astute.yaml echo -e '"SSL":\n "force_https": "true"' >> /etc/fuel/astute.yaml
""" """
self.ssh_manager.execute_on_remote(admin_node_ip, cmd) self.ssh_manager.check_call(admin_node_ip, cmd)
cmd = "find / -name \"nginx_services.pp\"" cmd = "find / -name \"nginx_services.pp\""
puppet_manifest = \ puppet_manifest = \
self.ssh_manager.execute_on_remote( self.ssh_manager.check_call(
admin_node_ip, cmd)['stdout'][0].strip() admin_node_ip, cmd)['stdout'][0].strip()
cmd = 'puppet apply {0}'.format(puppet_manifest) cmd = 'puppet apply {0}'.format(puppet_manifest)
self.ssh_manager.execute_on_remote(admin_node_ip, cmd) self.ssh_manager.check_call(admin_node_ip, cmd)
cmd = """ cmd = """
systemctl status nginx.service | systemctl status nginx.service |
awk 'match($0, /\s+Active:.*\((\w+)\)/, a) {print a[1]}' awk 'match($0, /\s+Active:.*\((\w+)\)/, a) {print a[1]}'
""" """
wait(lambda: ( wait(lambda: (
self.ssh_manager.execute_on_remote( self.ssh_manager.check_call(
admin_node_ip, cmd)['stdout'][0] != 'dead'), interval=10, admin_node_ip, cmd)['stdout'][0] != 'dead'), interval=10,
timeout=30, timeout=30,
timeout_msg='Nginx service is dead after trying to enable ' timeout_msg='Nginx service is dead after trying to enable '
@ -550,13 +551,13 @@ class EnvironmentModel(six.with_metaclass(SingletonMeta, object)):
def kill_wait_for_external_config(self): def kill_wait_for_external_config(self):
kill_cmd = 'pkill -f "^wait_for_external_config"' kill_cmd = 'pkill -f "^wait_for_external_config"'
check_cmd = 'pkill -0 -f "^wait_for_external_config"; [[ $? -eq 1 ]]' check_cmd = 'pkill -0 -f "^wait_for_external_config"; [[ $? -eq 1 ]]'
self.ssh_manager.execute_on_remote( self.ssh_manager.check_call(
ip=self.ssh_manager.admin_ip, ip=self.ssh_manager.admin_ip,
cmd=kill_cmd command=kill_cmd
) )
self.ssh_manager.execute_on_remote( self.ssh_manager.check_call(
ip=self.ssh_manager.admin_ip, ip=self.ssh_manager.admin_ip,
cmd=check_cmd command=check_cmd
) )
def wait_bootstrap(self): def wait_bootstrap(self):
@ -606,9 +607,9 @@ class EnvironmentModel(six.with_metaclass(SingletonMeta, object)):
'is not based on Ubuntu!') 'is not based on Ubuntu!')
return return
bootstrap_images = self.ssh_manager.execute_on_remote( bootstrap_images = self.ssh_manager.check_call(
ip=self.ssh_manager.admin_ip, ip=self.ssh_manager.admin_ip,
cmd='fuel-bootstrap --quiet list' command='fuel-bootstrap --quiet list'
)['stdout'] )['stdout']
assert_true(any('active' in line for line in bootstrap_images), assert_true(any('active' in line for line in bootstrap_images),
'Ubuntu bootstrap image wasn\'t built and activated! ' 'Ubuntu bootstrap image wasn\'t built and activated! '
@ -662,10 +663,10 @@ class EnvironmentModel(six.with_metaclass(SingletonMeta, object)):
'yum update -y 2>>/var/log/yum-update-error.log' 'yum update -y 2>>/var/log/yum-update-error.log'
logger.info('Performing yum clean and update commands') logger.info('Performing yum clean and update commands')
update_result = self.ssh_manager.execute_on_remote( update_result = self.ssh_manager.check_call(
ip=self.ssh_manager.admin_ip, ip=self.ssh_manager.admin_ip,
cmd=update_command, command=update_command,
err_msg='Packages update failed, inspect logs for details') error_info='Packages update failed, inspect logs for details')
logger.info('Packages were updated successfully') logger.info('Packages were updated successfully')
@ -702,10 +703,10 @@ class EnvironmentModel(six.with_metaclass(SingletonMeta, object)):
# ssh connection hanging on massive output from puppet run. # ssh connection hanging on massive output from puppet run.
cmd = '/usr/share/fuel-utils/update-master-node.sh > /dev/null 2>&1' cmd = '/usr/share/fuel-utils/update-master-node.sh > /dev/null 2>&1'
self.ssh_manager.execute_on_remote( self.ssh_manager.check_call(
ip=self.ssh_manager.admin_ip, ip=self.ssh_manager.admin_ip,
cmd=cmd, command=cmd,
err_msg='Update failed, inspect logs for details', error_info='Update failed, inspect logs for details',
) )
logger.info('Update successful') logger.info('Update successful')

27
fuelweb_test/models/fuel_web_client.py
View File

@ -63,7 +63,6 @@ from fuelweb_test.helpers.decorators import retry
from fuelweb_test.helpers.decorators import update_fuel from fuelweb_test.helpers.decorators import update_fuel
from fuelweb_test.helpers.decorators import upload_manifests from fuelweb_test.helpers.decorators import upload_manifests
from fuelweb_test.helpers.security import SecurityChecks from fuelweb_test.helpers.security import SecurityChecks
from fuelweb_test.helpers.ssh_manager import SSHManager
from fuelweb_test.helpers.ssl_helpers import change_cluster_ssl_config from fuelweb_test.helpers.ssl_helpers import change_cluster_ssl_config
from fuelweb_test.helpers.ssl_helpers import copy_cert_from_master from fuelweb_test.helpers.ssl_helpers import copy_cert_from_master
from fuelweb_test.helpers.uca import change_cluster_uca_config from fuelweb_test.helpers.uca import change_cluster_uca_config
@ -105,9 +104,9 @@ class FuelWebClient29(object):
"""FuelWebClient.""" # TODO documentation """FuelWebClient.""" # TODO documentation
def __init__(self, environment): def __init__(self, environment):
self.ssh_manager = SSHManager()
self.admin_node_ip = self.ssh_manager.admin_ip
self._environment = environment self._environment = environment
self.ssh_manager = environment.ssh_manager
self.admin_node_ip = self.ssh_manager.admin_ip
keystone_url = "http://{0}:5000/v2.0".format(self.admin_node_ip) keystone_url = "http://{0}:5000/v2.0".format(self.admin_node_ip)
@ -655,7 +654,7 @@ class FuelWebClient29(object):
if help_data.FUEL_USE_LOCAL_NTPD\ if help_data.FUEL_USE_LOCAL_NTPD\
and ('ntp_list' not in settings)\ and ('ntp_list' not in settings)\
and checkers.is_ntpd_active( and checkers.is_ntpd_active(
self.ssh_manager.admin_ip, public_gw): self.admin_node_ip, public_gw):
attributes['editable']['external_ntp']['ntp_list']['value'] =\ attributes['editable']['external_ntp']['ntp_list']['value'] =\
[public_gw] [public_gw]
logger.info("Configuring cluster #{0}" logger.info("Configuring cluster #{0}"
@ -1259,8 +1258,7 @@ class FuelWebClient29(object):
@logwrap @logwrap
def get_ssh_for_node(self, node_name): def get_ssh_for_node(self, node_name):
return self.environment.d_env.get_ssh_to_remote( return self.environment.d_env.get_node_remote(node_name)
self.get_node_ip_by_devops_name(node_name))
@logwrap @logwrap
def get_ssh_for_role(self, nodes_dict, role): def get_ssh_for_role(self, nodes_dict, role):
@ -1268,9 +1266,13 @@ class FuelWebClient29(object):
nodes_dict.keys()))[0] nodes_dict.keys()))[0]
return self.get_ssh_for_node(node_name) return self.get_ssh_for_node(node_name)
@logwrap
def get_ssh_for_ip(self, ip):
return self.ssh_manager.get_remote(ip)
@logwrap @logwrap
def get_ssh_for_nailgun_node(self, nailgun_node): def get_ssh_for_nailgun_node(self, nailgun_node):
return self.environment.d_env.get_ssh_to_remote(nailgun_node['ip']) return self.get_ssh_for_ip(nailgun_node['ip'])
@logwrap @logwrap
def is_node_discovered(self, nailgun_node): def is_node_discovered(self, nailgun_node):
@ -2290,7 +2292,7 @@ class FuelWebClient29(object):
self.client.list_cluster_nodes(cluster_id)]) self.client.list_cluster_nodes(cluster_id)])
# 'mco find' returns '1' exit code if rabbitmq is not ready # 'mco find' returns '1' exit code if rabbitmq is not ready
out = self.ssh_manager.execute_on_remote( out = self.ssh_manager.execute_on_remote(
ip=self.ssh_manager.admin_ip, ip=self.admin_node_ip,
cmd='mco find', assert_ec_equal=[0, 1])['stdout_str'] cmd='mco find', assert_ec_equal=[0, 1])['stdout_str']
ready_nodes_uids = set(out.split('\n')) ready_nodes_uids = set(out.split('\n'))
unavailable_nodes = nodes_uids - ready_nodes_uids unavailable_nodes = nodes_uids - ready_nodes_uids
@ -2369,8 +2371,7 @@ class FuelWebClient29(object):
# Let's find nodes where are a time skew. It can be checked on # Let's find nodes where are a time skew. It can be checked on
# an arbitrary one. # an arbitrary one.
logger.debug("Looking up nodes with a time skew and try to fix them") logger.debug("Looking up nodes with a time skew and try to fix them")
with self.environment.d_env.get_ssh_to_remote( with self.get_ssh_for_nailgun_node(online_ceph_nodes[0]) as remote:
online_ceph_nodes[0]['ip']) as remote:
if ceph.is_clock_skew(remote): if ceph.is_clock_skew(remote):
skewed = ceph.get_node_fqdns_w_clock_skew(remote) skewed = ceph.get_node_fqdns_w_clock_skew(remote)
logger.warning("Time on nodes {0} are to be " logger.warning("Time on nodes {0} are to be "
@ -2413,9 +2414,7 @@ class FuelWebClient29(object):
logger.info('Waiting until Ceph service become up...') logger.info('Waiting until Ceph service become up...')
for node in online_ceph_nodes: for node in online_ceph_nodes:
with self.environment.d_env\ with self.get_ssh_for_nailgun_node(node) as remote:
.get_ssh_to_remote(node['ip']) as remote:
wait(lambda: ceph.check_service_ready(remote) is True, wait(lambda: ceph.check_service_ready(remote) is True,
interval=20, timeout=600, interval=20, timeout=600,
timeout_msg='Ceph service is not properly started' timeout_msg='Ceph service is not properly started'
@ -2425,7 +2424,7 @@ class FuelWebClient29(object):
self.check_ceph_time_skew(cluster_id, offline_nodes) self.check_ceph_time_skew(cluster_id, offline_nodes)
node = online_ceph_nodes[0] node = online_ceph_nodes[0]
with self.environment.d_env.get_ssh_to_remote(node['ip']) as remote: with self.get_ssh_for_nailgun_node(node) as remote:
if not ceph.is_health_ok(remote): if not ceph.is_health_ok(remote):
if ceph.is_pgs_recovering(remote) and len(offline_nodes) > 0: if ceph.is_pgs_recovering(remote) and len(offline_nodes) > 0:
logger.info('Ceph is being recovered after osd node(s)' logger.info('Ceph is being recovered after osd node(s)'

2
fuelweb_test/tests/base_test_case.py
View File

@ -280,7 +280,7 @@ class TestBasic(object):
ssh.execute_on_remote(ssh.admin_ip, cmd=cmd) ssh.execute_on_remote(ssh.admin_ip, cmd=cmd)
self.env.wait_for_external_config() self.env.wait_for_external_config()
self.env.admin_actions.modify_configs(self.env.d_env.router()) self.env.admin_actions.modify_configs(self.env.d_env.get_default_gw())
if CUSTOM_FUEL_SETTING_YAML: if CUSTOM_FUEL_SETTING_YAML:
self.env.admin_actions.update_fuel_setting_yaml( self.env.admin_actions.update_fuel_setting_yaml(
CUSTOM_FUEL_SETTING_YAML) CUSTOM_FUEL_SETTING_YAML)