fuel-qa/fuelweb_test/tests/test_rh_compute.py

705 lines
27 KiB
Python

# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# TODO: We need to use sshmanager instead of executing bare commands
# bp link: https://blueprints.launchpad.net/fuel/+spec/sshmanager-integration
from __future__ import division
import re
from devops.error import TimeoutError
from devops.helpers.helpers import tcp_ping
from devops.helpers.helpers import wait
from proboscis import asserts
from proboscis import test
from fuelweb_test.helpers import checkers
from fuelweb_test.helpers.decorators import log_snapshot_after_test
from fuelweb_test.helpers import os_actions
from fuelweb_test import settings
from fuelweb_test import logger
from fuelweb_test.tests.base_test_case import SetupEnvironment
from fuelweb_test.tests.base_test_case import TestBasic
@test(groups=["rh", "rh.ha", "rh.basic"])
class RhHA(TestBasic):
"""RH-based compute tests"""
@staticmethod
def wait_for_slave_provision(node_ip, timeout=10 * 60):
"""Wait for a target node provision.
:param node_ip: IP address of target node.
:param timeout: Timeout for wait function.
"""
wait(lambda: tcp_ping(node_ip, 22),
timeout=timeout, timeout_msg="Node doesn't appear in network")
@staticmethod
def wait_for_slave_network_down(node_ip, timeout=10 * 20):
"""Wait for a target node network down.
:param node_ip: IP address of target node.
:param timeout: Timeout for wait function.
"""
wait(lambda: (not tcp_ping(node_ip, 22)), interval=1,
timeout=timeout, timeout_msg="Node doesn't gone offline")
def warm_restart_nodes(self, devops_nodes):
logger.info('Reboot (warm restart) nodes '
'{0}'.format([n.name for n in devops_nodes]))
self.warm_shutdown_nodes(devops_nodes)
self.warm_start_nodes(devops_nodes)
def warm_shutdown_nodes(self, devops_nodes):
logger.info('Shutting down (warm) nodes '
'{0}'.format([n.name for n in devops_nodes]))
for node in devops_nodes:
logger.debug('Shutdown node {0}'.format(node.name))
with self.fuel_web.get_ssh_for_node(node.name) as remote:
remote.execute('/sbin/shutdown -Ph now & exit')
for node in devops_nodes:
ip = self.fuel_web.get_node_ip_by_devops_name(node.name)
logger.info('Wait a {0} node offline status'.format(node.name))
try:
self.wait_for_slave_network_down(ip)
except TimeoutError:
asserts.assert_false(
tcp_ping(ip, 22),
'Node {0} has not become '
'offline after warm shutdown'.format(node.name))
node.destroy()
def warm_start_nodes(self, devops_nodes):
logger.info('Starting nodes '
'{0}'.format([n.name for n in devops_nodes]))
for node in devops_nodes:
node.start()
for node in devops_nodes:
ip = self.fuel_web.get_node_ip_by_devops_name(node.name)
try:
self.wait_for_slave_provision(ip)
except TimeoutError:
asserts.assert_true(
tcp_ping(ip, 22),
'Node {0} has not become online '
'after warm start'.format(node.name))
logger.debug('Node {0} became online.'.format(node.name))
@staticmethod
def connect_rh_image(slave):
"""Upload RH image into a target node.
:param slave: Target node name.
"""
path = settings.RH_IMAGE_PATH + settings.RH_IMAGE
def find_system_drive(node):
drives = node.disk_devices
for drive in drives:
if drive.device == 'disk' and 'system' in drive.volume.name:
return drive
raise Exception('Can not find suitable volume to proceed')
system_disk = find_system_drive(slave)
vol_path = system_disk.volume.get_path()
try:
system_disk.volume.upload(path)
except Exception as e:
logger.error(e)
logger.debug("Volume path: {0}".format(vol_path))
logger.debug("Image path: {0}".format(path))
@staticmethod
def verify_image_connected(remote):
"""Check that correct image connected to a target node system volume.
:param remote: Remote node to proceed.
"""
cmd = "cat /etc/redhat-release"
result = remote.execute(cmd)
logger.debug(result)
asserts.assert_equal(result['exit_code'], 0, "Image doesn't connected")
@staticmethod
def register_rh_subscription(remote):
"""Register RH subscription.
:param remote: Remote node to proceed.
"""
reg_command = (
"/usr/sbin/subscription-manager register "
"--username={0} --password={1}".format(
settings.RH_LICENSE_USERNAME,
settings.RH_LICENSE_PASSWORD)
)
if settings.RH_SERVER_URL:
reg_command = reg_command + " --serverurl={0}".format(
settings.RH_SERVER_URL)
if settings.RH_REGISTERED_ORG_NAME:
reg_command = reg_command + " --org={0}".format(
settings.RH_REGISTERED_ORG_NAME)
if settings.RH_RELEASE:
reg_command = reg_command + " --release={0}".format(
settings.RH_RELEASE)
if settings.RH_ACTIVATION_KEY:
reg_command = reg_command + " --activationkey={0}".format(
settings.RH_ACTIVATION_KEY)
if settings.RH_POOL_HASH:
result = remote.execute(reg_command)
logger.debug(result)
asserts.assert_equal(result['exit_code'], 0,
'RH registration failed')
reg_pool_cmd = ("/usr/sbin/subscription-manager "
"attach --pool={0}".format(settings.RH_POOL_HASH))
result = remote.execute(reg_pool_cmd)
logger.debug(result)
asserts.assert_equal(result['exit_code'], 0,
'Can not attach node to subscription pool')
else:
cmd = reg_command + " --auto-attach"
result = remote.execute(cmd)
logger.debug(result)
asserts.assert_equal(result['exit_code'], 0,
'RH registration with auto-attaching failed')
@staticmethod
def enable_rh_repos(remote):
"""Enable Red Hat mirrors on a target node.
:param remote: Remote node for proceed.
"""
cmd = ("yum-config-manager --enable rhel-{0}-server-optional-rpms && "
"yum-config-manager --enable rhel-{0}-server-extras-rpms &&"
"yum-config-manager --enable rhel-{0}-server-rh-common-rpms"
.format(settings.RH_MAJOR_RELEASE))
result = remote.execute(cmd)
logger.debug(result)
asserts.assert_equal(result['exit_code'], 0,
'Enabling RH repos failed')
@staticmethod
def set_hostname(remote, host_number=1):
"""Set hostname with domain for a target node.
:param host_number: Node index nubmer (1 by default).
:param remote: Remote node for proceed.
"""
hostname = "rh-{0}.test.domain.local".format(host_number)
cmd = ("sysctl kernel.hostname={0} && "
"echo '{0}' > /etc/hostname".format(hostname))
result = remote.execute(cmd)
logger.debug(result)
asserts.assert_equal(result['exit_code'], 0,
'Setting up hostname for node failed')
@staticmethod
def puppet_apply(puppets, remote):
"""Apply list of puppets on a target node.
:param puppets: <list> of puppets.
:param remote: Remote node for proceed.
"""
logger.debug("Applying puppets...")
for puppet in puppets:
logger.debug('Applying: {0}'.format(puppet))
result = remote.execute(
'puppet apply -vd -l /var/log/puppet.log {0}'.format(puppet))
if result['exit_code'] != 0:
logger.debug("Failed on task: {0}".format(puppet))
logger.debug("STDERR:\n {0}".format(result['stderr']))
logger.debug("STDOUT:\n {0}".format(result['stdout']))
asserts.assert_equal(
result['exit_code'], 0, 'Puppet run failed. '
'Task: {0}'.format(puppet))
def apply_first_part_puppet(self, remote):
"""Apply first part of puppet modular tasks on terget node.
:param remote: Remote node for proceed.
"""
first_puppet_run = [
"/etc/puppet/modules/osnailyfacter/modular/hiera/hiera.pp",
"/etc/puppet/modules/osnailyfacter/modular/globals/globals.pp",
"/etc/puppet/modules/osnailyfacter/modular/firewall/firewall.pp",
"/etc/puppet/modules/osnailyfacter/modular/tools/tools.pp"
]
self.puppet_apply(first_puppet_run, remote)
@staticmethod
def apply_networking_puppet(remote):
"""Apply networking puppet on a target node.
Puppet task will executed in screen to prevent disconnections while
interfaces configuring.
:param remote: Remote node for proceed.
"""
iface_check = "test -f /etc/sysconfig/network-scripts/ifcfg-eth0"
result = remote.execute(iface_check)
if result['exit_code'] == 0:
remove_iface = "rm -f /etc/sysconfig/network-scripts/ifcfg-eth0"
result = remote.execute(remove_iface)
logger.debug(result)
prep = "screen -dmS netconf"
result = remote.execute(prep)
logger.debug(result)
asserts.assert_equal(result['exit_code'], 0, 'Can not create screen')
net_puppet = ('screen -r netconf -p 0 -X stuff '
'$"puppet apply -vd -l /var/log/puppet.log '
'/etc/puppet/modules/osnailyfacter/modular/'
'netconfig/netconfig.pp && touch ~/success ^M"')
result = remote.execute(net_puppet)
if result['exit_code'] != 0:
logger.debug("STDERR:\n {0}".format(result['stderr']))
logger.debug("STDOUT:\n {0}".format(result['stdout']))
asserts.assert_equal(
result['exit_code'], 0, 'Can not create screen with '
'netconfig task')
@staticmethod
def check_netconfig_success(remote, timeout=10 * 20):
"""Check that netconfig.pp modular task is succeeded.
:param remote: Remote node for proceed.
:param timeout: Timeout for wait function.
"""
def file_checker(connection):
cmd = "test -f ~/success"
result = connection.execute(cmd)
logger.debug(result)
if result['exit_code'] != 0:
return False
else:
return True
wait(lambda: file_checker(remote), timeout=timeout,
timeout_msg='Netconfig puppet task unsuccessful')
def apply_last_part_puppet(self, remote):
"""Apply final part of puppet modular tasks on a target node.
:param remote: Remote node for proceed.
"""
last_puppet_run = [
"/etc/puppet/modules/osnailyfacter/modular/roles/compute.pp",
"/etc/puppet/modules/osnailyfacter/modular/"
"openstack-network/common-config.pp",
"/etc/puppet/modules/osnailyfacter/modular/"
"openstack-network/plugins/ml2.pp",
"/etc/puppet/modules/osnailyfacter/modular/"
"openstack-network/agents/l3.pp",
"/etc/puppet/modules/osnailyfacter/modular/"
"openstack-network/agents/metadata.pp",
"/etc/puppet/modules/osnailyfacter/modular/"
"openstack-network/compute-nova.pp",
"/etc/puppet/modules/osnailyfacter/modular/"
"astute/enable_compute.pp"
]
self.puppet_apply(last_puppet_run, remote)
@staticmethod
def backup_required_information(remote, ip):
"""Back up required information for compute from target node.
:param remote: Remote Fuel master node.
:param ip: Target node ip to back up from.
"""
logger.debug('Target node ip: {0}'.format(ip))
cmd = ("cd ~/ && mkdir rh_backup; "
"scp -r {0}:/root/.ssh rh_backup/. ; "
"scp {0}:/etc/astute.yaml rh_backup/ ; "
"scp -r {0}:/var/lib/astute/nova rh_backup/").format(ip)
result = remote.execute(cmd)
logger.debug(result['stdout'])
logger.debug(result['stderr'])
asserts.assert_equal(result['exit_code'], 0,
'Can not back up required information from node')
logger.debug("Backed up ssh-keys and astute.yaml")
@staticmethod
def clean_string(string):
"""Clean string of redundant characters.
:param string: String.
:return:
"""
k = str(string)
pattern = "^\s+|\[|\]|\n|,|'|\r|\s+$"
res = re.sub(pattern, '', k)
res = res.strip('/\\n')
# NOTE(freerunner): Using sub twice to collect key without extra
# whitespaces.
res = re.sub(pattern, '', res)
res = res.strip('/\\n')
return res
def restore_information(self, ip, remote_admin, remote_slave):
"""Restore information on a target node.
:param ip: Remote node ip.
:param remote_admin: Remote admin node for proceed.
:param remote_slave: Remote slave node for proceed.
"""
cmd = "cat ~/rh_backup/.ssh/authorized_keys"
result = remote_admin.execute(cmd)
key = result['stdout']
logger.debug(result)
asserts.assert_equal(result['exit_code'], 0,
'Can not get backed up ssh key.')
key = self.clean_string(key)
cmd = "mkdir ~/.ssh; echo '{0}' >> ~/.ssh/authorized_keys".format(key)
result = remote_slave.execute(cmd)
logger.debug(result['stdout'])
logger.debug(result['stderr'])
asserts.assert_equal(result['exit_code'], 0,
'Can not recover ssh key for node')
cmd = "cd ~/rh_backup && scp astute.yaml {0}@{1}:/etc/.".format(
settings.RH_IMAGE_USER, ip)
logger.debug("Restoring astute.yaml for node with ip {0}".format(ip))
result = remote_admin.execute(cmd)
logger.debug(result)
asserts.assert_equal(result['exit_code'], 0,
'Can not restore astute.yaml')
cmd = "mkdir -p /var/lib/astute"
logger.debug("Prepare node for restoring nova ssh-keys")
result = remote_slave.execute(cmd)
logger.debug(result)
asserts.assert_equal(result['exit_code'], 0, 'Preparation failed')
cmd = (
"cd ~/rh_backup && scp -r nova {0}@{1}:/var/lib/astute/.".format(
settings.RH_IMAGE_USER, ip)
)
logger.debug("Restoring nova ssh-keys")
result = remote_admin.execute(cmd)
logger.debug(result)
asserts.assert_equal(result['exit_code'], 0,
'Can not restore ssh-keys for nova')
@staticmethod
def install_yum_components(remote):
"""Install required yum components on a target node.
:param remote: Remote node for proceed.
"""
cmd = "yum install yum-utils yum-priorities -y"
result = remote.execute(cmd)
logger.debug(result)
asserts.assert_equal(result['exit_code'], 0, 'Can not install required'
'yum components.')
@staticmethod
def set_repo_for_perestroika(remote):
"""Set Perestroika repos.
:param remote: Remote node for proceed.
"""
repo = settings.PERESTROIKA_REPO
cmd = ("curl {0}".format(repo))
result = remote.execute(cmd)
logger.debug(result)
asserts.assert_equal(result['exit_code'], 0,
'Perestroika repos unavailable from node.')
cmd = ("echo '[mos]\n"
"name=mos\n"
"type=rpm-md\n"
"baseurl={0}\n"
"gpgcheck=0\n"
"enabled=1\n"
"priority=5' >"
"/etc/yum.repos.d/mos.repo && "
"yum clean all".format(repo))
result = remote.execute(cmd)
logger.debug(result)
asserts.assert_equal(result['exit_code'], 0,
'Can not create config file for repo')
@staticmethod
def check_hiera_installation(remote):
"""Check hiera installation on node.
:param remote: Remote node for proceed.
"""
cmd = "yum list installed | grep hiera"
logger.debug('Checking hiera installation...')
result = remote.execute(cmd)
if result['exit_code'] == 0:
cmd = "yum remove hiera -y"
logger.debug('Found existing installation of hiera. Removing...')
result = remote.execute(cmd)
asserts.assert_equal(result['exit_code'], 0, 'Can not remove '
'hiera')
cmd = "ls /etc/hiera"
logger.debug('Checking hiera files for removal...')
result = remote.execute(cmd)
if result['exit_code'] == 0:
logger.debug('Found redundant hiera files. Removing...')
cmd = "rm -rf /etc/hiera"
result = remote.execute(cmd)
asserts.assert_equal(result['exit_code'], 0,
'Can not remove hiera files')
@staticmethod
def check_rsync_installation(remote):
"""Check rsync installation on node.
:param remote: Remote node for proceed.
"""
cmd = "yum list installed | grep rsync"
logger.debug("Checking rsync installation...")
result = remote.execute(cmd)
if result['exit_code'] != 0:
logger.debug("Rsync is not found. Installing rsync...")
cmd = "yum clean all && yum install rsync -y"
result = remote.execute(cmd)
logger.debug(result)
asserts.assert_equal(result['exit_code'], 0, 'Can not install '
'rsync on node.')
@staticmethod
def remove_old_compute_services(remote, hostname):
"""Remove old redundant services which was removed from services base.
:param remote: Remote node for proceed.
:param hostname: Old compute hostname.
"""
cmd = ("source ~/openrc && for i in $(nova service-list | "
"awk '/%s/{print $2}'); do nova service-delete $i; "
"done" % hostname)
result = remote.execute(cmd)
logger.debug(result)
asserts.assert_equal(result['exit_code'], 0, 'Can not remove '
'old nova computes')
cmd = ("source ~/openrc && for i in $(neutron agent-list | "
"awk '/%s/{print $2}'); do neutron agent-delete $i; "
"done" % hostname)
result = remote.execute(cmd)
logger.debug(result)
asserts.assert_equal(result['exit_code'], 0, 'Can not remove '
'old neutron agents')
@staticmethod
def install_ruby_puppet(remote):
"""Install ruby and puppet on a target node.
:param remote: Remote node for proceed.
"""
puppet_install_cmd = "yum install puppet ruby -y"
result = remote.execute(puppet_install_cmd)
logger.debug(result)
asserts.assert_equal(result['exit_code'], 0,
'Ruby and puppet installation failed')
@staticmethod
def rsync_puppet_modules(remote, ip):
"""Rsync puppet modules from remote node to node with specified ip.
:param remote: Remote node for proceed.
:param ip: IP address of a target node where to sync.
"""
cmd = ("rsync -avz /etc/puppet/modules/* "
"{0}@{1}:/etc/puppet/modules/".format(settings.RH_IMAGE_USER,
ip))
result = remote.execute(cmd)
logger.debug(cmd)
asserts.assert_equal(result['exit_code'], 0,
'Rsync puppet modules failed')
def save_node_hostname(self, remote):
"""Save hostname of a node.
:param remote: Remote node for proceed.
:return: Node hostname.
"""
cmd = "hostname"
result = remote.execute(cmd)
asserts.assert_equal(result['exit_code'], 0, 'Can not get hostname '
'for remote')
nodename = self.clean_string(result['stdout'])
return nodename
@test(depends_on=[SetupEnvironment.prepare_slaves_5],
groups=["deploy_rh_compute_ha_tun"])
@log_snapshot_after_test
def deploy_rh_based_compute(self):
"""Deploy RH-based compute in HA mode with Neutron VXLAN
Scenario:
1. Check required image.
2. Revert snapshot 'ready_with_5_slaves'.
3. Create a Fuel cluster.
4. Update cluster nodes with required roles.
5. Deploy the Fuel cluster.
6. Run OSTF.
7. Backup astute.yaml and ssh keys from compute.
8. Boot compute with RH image.
9. Prepare node for Puppet run.
10. Execute modular tasks for compute.
11. Run OSTF.
Duration: 150m
Snapshot: deploy_rh_compute_ha_tun
"""
self.show_step(1, initialize=True)
logger.debug('Check MD5 sum of RH 7 image')
check_image = checkers.check_image(
settings.RH_IMAGE,
settings.RH_IMAGE_MD5,
settings.RH_IMAGE_PATH)
asserts.assert_true(check_image,
'Provided image is incorrect. '
'Please, check image path and md5 sum of it.')
self.show_step(2)
self.env.revert_snapshot("ready_with_5_slaves")
self.show_step(3)
logger.debug('Create Fuel cluster RH-based compute tests')
data = {
'net_provider': 'neutron',
'net_segment_type': settings.NEUTRON_SEGMENT['tun'],
'tenant': 'RhHA',
'user': 'RhHA',
'password': 'RhHA'
}
cluster_id = self.fuel_web.create_cluster(
name=self.__class__.__name__,
mode=settings.DEPLOYMENT_MODE,
settings=data
)
self.show_step(4)
self.fuel_web.update_nodes(
cluster_id,
{
'slave-01': ['controller'],
'slave-02': ['controller'],
'slave-03': ['controller'],
'slave-04': ['compute']
}
)
self.show_step(5)
self.fuel_web.deploy_cluster_wait(cluster_id)
cluster_vip = self.fuel_web.get_public_vip(cluster_id)
os_conn = os_actions.OpenStackActions(
cluster_vip, data['user'], data['password'], data['tenant'])
self.show_step(6)
self.fuel_web.run_ostf(cluster_id=cluster_id,
test_sets=['ha', 'smoke', 'sanity'])
self.show_step(7)
compute = self.fuel_web.get_nailgun_cluster_nodes_by_roles(
cluster_id, ['compute'])[0]
controller_name = 'slave-01'
controller_ip = self.fuel_web.get_nailgun_node_by_name(
controller_name)['ip']
logger.debug('Got node: {0}'.format(compute))
target_node_name = compute['name'].split('_')[0]
logger.debug('Target node name: {0}'.format(target_node_name))
target_node = self.env.d_env.get_node(name=target_node_name)
logger.debug('DevOps Node: {0}'.format(target_node))
target_node_ip = self.fuel_web.get_nailgun_node_by_name(
target_node_name)['ip']
logger.debug('Acquired ip: {0} for node: {1}'.format(
target_node_ip, target_node_name))
with self.env.d_env.get_ssh_to_remote(target_node_ip) as remote:
old_hostname = self.save_node_hostname(remote)
with self.env.d_env.get_admin_remote() as remote:
self.backup_required_information(remote, target_node_ip)
self.show_step(8)
target_node.destroy()
asserts.assert_false(target_node.driver.node_active(node=target_node),
'Target node still active')
self.connect_rh_image(target_node)
target_node.start()
asserts.assert_true(target_node.driver.node_active(node=target_node),
'Target node did not start')
self.wait_for_slave_provision(target_node_ip)
with self.env.d_env.get_ssh_to_remote(target_node_ip) as remote:
self.verify_image_connected(remote)
self.show_step(9)
with self.env.d_env.get_admin_remote() as remote_admin:
with self.env.d_env.get_ssh_to_remote(target_node_ip) as \
remote_slave:
self.restore_information(target_node_ip,
remote_admin, remote_slave)
with self.env.d_env.get_ssh_to_remote(target_node_ip) as remote:
self.set_hostname(remote)
if not settings.CENTOS_DUMMY_DEPLOY:
self.register_rh_subscription(remote)
self.install_yum_components(remote)
if not settings.CENTOS_DUMMY_DEPLOY:
self.enable_rh_repos(remote)
self.set_repo_for_perestroika(remote)
self.check_hiera_installation(remote)
self.install_ruby_puppet(remote)
self.check_rsync_installation(remote)
with self.env.d_env.get_admin_remote() as remote:
self.rsync_puppet_modules(remote, target_node_ip)
self.show_step(10)
with self.env.d_env.get_ssh_to_remote(target_node_ip) as remote:
self.apply_first_part_puppet(remote)
with self.env.d_env.get_ssh_to_remote(target_node_ip) as remote:
self.apply_networking_puppet(remote)
with self.env.d_env.get_ssh_to_remote(target_node_ip) as remote:
self.check_netconfig_success(remote)
self.apply_last_part_puppet(remote)
with self.env.d_env.get_ssh_to_remote(controller_ip) as remote:
self.remove_old_compute_services(remote, old_hostname)
self.fuel_web.assert_cluster_ready(os_conn, smiles_count=13)
self.show_step(11)
self.fuel_web.run_ostf(cluster_id=cluster_id,
test_sets=['ha', 'smoke', 'sanity'])
self.env.make_snapshot("ready_ha_with_rh_compute", is_make=True)