fuel-qa/fuelweb_test/tests/tests_extra_computes/base_extra_computes.py

#    Copyright 2015 Mirantis, Inc.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from __future__ import division
import re

from devops.error import TimeoutError
from devops.helpers.helpers import tcp_ping
from devops.helpers.helpers import wait
from proboscis import asserts

from fuelweb_test import settings
from fuelweb_test import logger
from fuelweb_test.tests.base_test_case import TestBasic


class ExtraComputesBase(TestBasic):
    """Extra computes tests base"""

    def check_slaves_are_ready(self):
        devops_nodes = [node for node in self.env.d_env.nodes().slaves
                        if node.driver.node_active(node)]

        for node in devops_nodes:
            ip = self.fuel_web.get_node_ip_by_devops_name(node.name)
            try:
                self.wait_for_slave_provision(ip)
            except TimeoutError:
                asserts.assert_true(
                    tcp_ping(ip, 22),
                    'Node {0} has not become online '
                    'after revert'.format(node.name))
            logger.debug('Node {0} became online.'.format(node.name))
        return True

    @staticmethod
    def wait_for_slave_provision(node_ip, timeout=10 * 60):
        """Wait for a target node provision.

        :param node_ip: IP address of target node.
        :param timeout: Timeout for wait function.
        """
        wait(lambda: tcp_ping(node_ip, 22),
             timeout=timeout, timeout_msg="Node doesn't appear in network")

    @staticmethod
    def wait_for_slave_network_down(node_ip, timeout=10 * 20):
        """Wait for a target node network down.

        :param node_ip: IP address of target node.
        :param timeout: Timeout for wait function.
        """
        wait(lambda: (not tcp_ping(node_ip, 22)), interval=1,
             timeout=timeout, timeout_msg="Node doesn't gone offline")

    def warm_restart_nodes(self, devops_nodes):
        logger.info('Reboot (warm restart) nodes '
                    '{0}'.format([n.name for n in devops_nodes]))
        self.warm_shutdown_nodes(devops_nodes)
        self.warm_start_nodes(devops_nodes)

    def warm_shutdown_nodes(self, devops_nodes):
        logger.info('Shutting down (warm) nodes '
                    '{0}'.format([n.name for n in devops_nodes]))
        for node in devops_nodes:
            ip = self.fuel_web.get_node_ip_by_devops_name(node.name)
            logger.debug('Shutdown node {0}'.format(node.name))
            self.ssh_manager.execute(ip, '/sbin/shutdown -Ph now & exit')

        for node in devops_nodes:
            ip = self.fuel_web.get_node_ip_by_devops_name(node.name)
            logger.info('Wait a {0} node offline status'.format(node.name))
            try:
                self.wait_for_slave_network_down(ip)
            except TimeoutError:
                asserts.assert_false(
                    tcp_ping(ip, 22),
                    'Node {0} has not become '
                    'offline after warm shutdown'.format(node.name))
            node.destroy()

    def warm_start_nodes(self, devops_nodes):
        logger.info('Starting nodes '
                    '{0}'.format([n.name for n in devops_nodes]))
        for node in devops_nodes:
            node.start()
        for node in devops_nodes:
            ip = self.fuel_web.get_node_ip_by_devops_name(node.name)
            try:
                self.wait_for_slave_provision(ip)
            except TimeoutError:
                asserts.assert_true(
                    tcp_ping(ip, 22),
                    'Node {0} has not become online '
                    'after warm start'.format(node.name))
            logger.info('Node {0} became online.'.format(node.name))

    @staticmethod
    def connect_extra_compute_image(slave):
        """Upload extra compute image into a target node.

        :param slave: Target node name.
        """
        path = settings.EXTRA_COMP_IMAGE_PATH + settings.EXTRA_COMP_IMAGE

        def find_system_drive(node):
            drives = node.disk_devices
            for drive in drives:
                if drive.device == 'disk' and 'system' in drive.volume.name:
                    return drive
            raise Exception('Can not find suitable volume to proceed')

        system_disk = find_system_drive(slave)
        vol_path = system_disk.volume.get_path()

        try:
            system_disk.volume.upload(path)
        except Exception as e:
            logger.error(e)
        logger.debug("Volume path: {0}".format(vol_path))
        logger.debug("Image path: {0}".format(path))

    def verify_image_connected(self, ip, types='rh'):
        """Check that correct image connected to a target node system volume.

        :param ip: Remote node ip to proceed.
        :param types: rh or ol
        """
        if types is 'rh':
            cmd = "cat /etc/redhat-release"
        else:
            cmd = "cat /etc/oracle-release"
        self.ssh_manager.execute_on_remote(
            ip, cmd, err_msg="Image doesn't connected")

    def register_rh_subscription(self, ip):
        """Register RH subscription.

        :param ip: Remote node ip to proceed.
        """
        reg_command = (
            "/usr/sbin/subscription-manager register "
            "--username={0} --password={1}".format(
                settings.RH_LICENSE_USERNAME,
                settings.RH_LICENSE_PASSWORD)
        )

        if settings.RH_SERVER_URL:
            reg_command += " --serverurl={0}".format(
                settings.RH_SERVER_URL)

        if settings.RH_REGISTERED_ORG_NAME:
            reg_command += " --org={0}".format(
                settings.RH_REGISTERED_ORG_NAME)

        if settings.RH_RELEASE:
            reg_command += " --release={0}".format(
                settings.RH_RELEASE)

        if settings.RH_ACTIVATION_KEY:
            reg_command += " --activationkey={0}".format(
                settings.RH_ACTIVATION_KEY)

        if settings.RH_POOL_HASH:
            self.ssh_manager.execute_on_remote(
                ip, reg_command, err_msg='RH registration failed')
            reg_pool_cmd = ("/usr/sbin/subscription-manager "
                            "attach --pool={0}".format(settings.RH_POOL_HASH))
            self.ssh_manager.execute_on_remote(
                ip, reg_pool_cmd,
                err_msg='Can not attach node to subscription pool')
        else:
            cmd = reg_command + " --auto-attach"
            self.ssh_manager.execute_on_remote(
                ip, cmd, err_msg='RH registration with auto-attaching failed')

    def enable_extra_compute_repos(self, ip, types='rh'):
        """Enable requested family mirrors on a target node.

        :param ip: Remote node ip for proceed.
        :param types: rh or ol
        """
        if types is 'rh':
            cmd = (
                "yum-config-manager --enable rhel-{0}-server-optional-rpms &&"
                " yum-config-manager --enable rhel-{0}-server-extras-rpms &&"
                " yum-config-manager --enable rhel-{0}-server-rh-common-rpms"
                .format(settings.RH_MAJOR_RELEASE))
        else:
            cmd = ("yum-config-manager --enable ol{0}_addons && "
                   "yum-config-manager --enable ol{0}_optional_latest"
                   .format(settings.OL_MAJOR_RELEASE))

        self.ssh_manager.execute_on_remote(
            ip, cmd, err_msg='Enabling requested family repos failed')

    def set_hostname(self, ip, types='rh', host_number=1):
        """Set hostname with domain for a target node.

        :param host_number: Node index nubmer (1 by default).
        :param types: rh or ol
        :param ip: Remote node ip for proceed.
        """
        hostname = "{0}-{1}.test.domain.local".format(types, host_number)
        cmd = ("sysctl kernel.hostname={0} && "
               "echo '{0}' > /etc/hostname".format(hostname))

        self.ssh_manager.execute_on_remote(
            ip, cmd, err_msg='Setting up hostname for node failed')
        return hostname

    def puppet_apply(self, puppets, ip):
        """Apply list of puppets on a target node.

        :param puppets: <list> of puppets.
        :param ip: Remote node ip for proceed.
        """
        logger.debug("Applying puppets...")
        for puppet in puppets:
            logger.debug('Applying: {0}'.format(puppet))
            self.ssh_manager.execute_on_remote(
                ip,
                'puppet apply -vd -l /var/log/puppet.log {0}'.format(puppet),
                err_msg='Puppet run failed. Task: {0}'.format(puppet))

    def apply_first_part_puppet(self, ip):
        """Apply first part of puppet modular tasks on target node.

        :param ip: Remote node ip for proceed.
        """
        first_puppet_run = [
            "/etc/puppet/modules/osnailyfacter/modular/hiera/hiera.pp",
            "/etc/puppet/modules/osnailyfacter/modular/"
            "hiera/override_configuration.pp",
            "/etc/puppet/modules/osnailyfacter/modular/"
            "netconfig/reserved_ports.pp",
            "/etc/puppet/modules/osnailyfacter/modular/fuel_pkgs/fuel_pkgs.pp",
            "/etc/puppet/modules/osnailyfacter/modular/globals/globals.pp",
            "/etc/puppet/modules/osnailyfacter/modular/tools/tools.pp"
        ]

        self.puppet_apply(first_puppet_run, ip)

    def apply_networking_puppet(self, ip):
        """Apply networking puppet on a target node.

        Puppet task will executed in screen to prevent disconnections while
        interfaces configuring.

        :param ip: Remote node ip for proceed.
        """
        iface_check = "test -f /etc/sysconfig/network-scripts/ifcfg-eth0"
        result = self.ssh_manager.execute(ip, iface_check)
        if result['exit_code'] == 0:
            remove_iface = "rm -f /etc/sysconfig/network-scripts/ifcfg-eth0"
            self.ssh_manager.execute_on_remote(ip, remove_iface)
        prep = "screen -dmS netconf"
        self.ssh_manager.execute_on_remote(ip, prep,
                                           err_msg='Can not create screen')

        net_puppet = ('screen -r netconf -p 0 -X stuff '
                      '$"puppet apply -vd -l /var/log/puppet.log '
                      '/etc/puppet/modules/osnailyfacter/modular/'
                      'netconfig/netconfig.pp && touch ~/success ^M"')
        self.ssh_manager.execute_on_remote(
            ip, net_puppet,
            err_msg='Can not create screen with netconfig task')

    def check_netconfig_success(self, ip, timeout=10 * 20):
        """Check that netconfig.pp modular task is succeeded.

        :param ip: Remote node ip for proceed.
        :param timeout: Timeout for wait function.
        """

        def file_checker(target_ip):
            cmd = "test -f ~/success"
            result = self.ssh_manager.execute(target_ip, cmd)
            logger.debug(result)
            if result['exit_code'] != 0:
                return False
            else:
                return True
        wait(lambda: file_checker(ip), timeout=timeout,
             timeout_msg='Netconfig puppet task unsuccessful')

    def apply_last_part_puppet(self, ip, ceph=False):
        """Apply final part of puppet modular tasks on a target node.

        :param ip: Remote node ip for proceed.
        """
        last_puppet_run = [
            "/etc/puppet/modules/osnailyfacter/modular/firewall/firewall.pp",
            "/etc/puppet/modules/osnailyfacter/modular/hosts/hosts.pp",
            "/etc/puppet/modules/osnailyfacter/modular/roles/compute.pp",
            "/etc/puppet/modules/osnailyfacter/modular/"
            "openstack-network/common-config.pp",
            "/etc/puppet/modules/osnailyfacter/modular/"
            "openstack-network/plugins/ml2.pp",
            "/etc/puppet/modules/osnailyfacter/modular/"
            "openstack-network/agents/l3.pp",
            "/etc/puppet/modules/osnailyfacter/modular/"
            "openstack-network/agents/metadata.pp",
            "/etc/puppet/modules/osnailyfacter/modular/"
            "openstack-network/compute-nova.pp",
            "/etc/puppet/modules/osnailyfacter/modular/"
            "roles/enable_compute.pp",
            "/etc/puppet/modules/osnailyfacter/modular/dns/dns-client.pp",
            "/etc/puppet/modules/osnailyfacter/modular/netconfig/"
            "configure_default_route.pp"
        ]

        if ceph:
            last_puppet_run.append("/etc/puppet/modules/osnailyfacter/"
                                   "modular/ceph/ceph_compute.pp")
        last_puppet_run.append("/etc/puppet/modules/osnailyfacter/modular/"
                               "ntp/ntp-client.pp")

        self.puppet_apply(last_puppet_run, ip)

    def backup_required_information(self, ip, target_ip, node=1, ceph=False):
        """Back up required information for compute from target node.

        :param ip: Remote Fuel master node ip.
        :param target_ip: Target node ip to back up from.
        :param node: Node number
        :param ceph: Enabled or disabled Ceph storage.
        """

        logger.debug('Target node ip: {0}'.format(target_ip))
        cmd = ("cd ~/ && mkdir rh_backup-{1}; "
               "scp -r {0}:/root/.ssh rh_backup-{1}/. ; "
               "scp {0}:/etc/astute.yaml rh_backup-{1}/ ; "
               "scp -r {0}:/var/lib/astute/nova rh_backup-{1}/"
               .format(target_ip, node))
        if ceph:
            cmd += (" ; scp -r {0}:/var/lib/astute/ceph rh_backup-{1}/"
                    .format(target_ip, node))
        self.ssh_manager.execute_on_remote(
            ip, cmd, err_msg='Can not back up required information from node')
        logger.debug("Backed up ssh-keys and astute.yaml")

    @staticmethod
    def clean_string(string, twice=True):
        """Clean string of redundant characters.

        :param string: String.
        :param twice: Boolean. Use function twice or not.
        :return:
        """
        k = str(string)
        pattern = "^\s+|\[|\]|\n|,|'|\r|\s+$"
        res = re.sub(pattern, '', k)
        if twice:
            res = res.strip('/\\n')
            # NOTE(freerunner): Using sub twice to collect key without extra
            # whitespaces.
            res = re.sub(pattern, '', res)
            res = res.strip('/\\n')
        return res

    def restore_information(self, ip, remote_admin_ip, ceph=False, node=1):

        """Restore information on a target node.

        :param ip: Remote node ip.
        :param remote_admin_ip: Remote admin node for proceed.
        """

        cmd = "cat ~/rh_backup-{0}/.ssh/authorized_keys".format(node)
        result = self.ssh_manager.execute_on_remote(
            remote_admin_ip, cmd,
            err_msg='Can not get backed up ssh key.')
        key = result['stdout']

        key = self.clean_string(key)

        cmd = "mkdir ~/.ssh; echo '{0}' >> ~/.ssh/authorized_keys".format(key)
        self.ssh_manager.execute_on_remote(
            ip, cmd, err_msg='Can not recover ssh key for node')

        cmd = "cd ~/rh_backup-{2} && scp astute.yaml {0}@{1}:/etc/.".format(
            settings.EXTRA_COMP_IMAGE_USER, ip, node)
        logger.debug("Restoring astute.yaml for node with ip {0}".format(ip))
        self.ssh_manager.execute_on_remote(
            remote_admin_ip, cmd, err_msg='Can not restore astute.yaml')

        cmd = "mkdir -p /var/lib/astute"
        logger.debug("Prepare node for restoring nova ssh-keys")
        self.ssh_manager.execute_on_remote(ip, cmd,
                                           err_msg='Preparation failed')

        cmd = (
            "cd ~/rh_backup-{2} && scp -r nova {0}@{1}:/var/lib/astute/.".
            format(settings.EXTRA_COMP_IMAGE_USER, ip, node)
        )
        logger.debug("Restoring nova ssh-keys")
        self.ssh_manager.execute_on_remote(
            remote_admin_ip, cmd, err_msg='Can not restore ssh-keys for nova')

        if ceph:
            cmd = (
                "cd ~/rh_backup-{2} && scp -r ceph {0}@{1}:/var/lib/astute/."
                .format(settings.EXTRA_COMP_IMAGE_USER, ip, node)
            )
            logger.debug("Restoring ceph ssh-keys")
            self.ssh_manager.execute_on_remote(
                remote_admin_ip, cmd,
                err_msg='Can not restore ssh-keys for ceph')

    def install_yum_components(self, ip):
        """Install required yum components on a target node.

        :param ip: Remote node ip for proceed.
        """
        cmd = "yum install yum-utils yum-priorities -y"
        self.ssh_manager.execute_on_remote(
            ip, cmd, err_msg='Can not install required yum components.')

    def set_repo_for_perestroika(self, ip):
        """Set Perestroika repos.

        :param ip: Remote node ip for proceed.
        """
        repo = settings.PERESTROIKA_REPO
        cmd = ("curl {0}".format(repo))

        self.ssh_manager.execute_on_remote(
            ip, cmd, err_msg='Perestroika repos unavailable from node.')

        cmd = ("echo '[mos]\n"
               "name=mos\n"
               "type=rpm-md\n"
               "baseurl={0}\n"
               "gpgcheck=0\n"
               "enabled=1\n"
               "priority=5' >"
               "/etc/yum.repos.d/mos.repo && "
               "yum clean all".format(repo))
        self.ssh_manager.execute_on_remote(
            ip, cmd, err_msg='Can not create config file for repo')

    def check_hiera_installation(self, ip):
        """Check hiera installation on node.

        :param ip: Remote node ip for proceed.
        """
        cmd = "yum list installed | grep hiera"
        logger.debug('Checking hiera installation...')
        result = self.ssh_manager.execute(ip, cmd)
        if result['exit_code'] == 0:
            cmd = "yum remove hiera -y"
            logger.debug('Found existing installation of hiera. Removing...')
            result = self.ssh_manager.execute(ip, cmd)
            asserts.assert_equal(result['exit_code'], 0, 'Can not remove '
                                                         'hiera')
            cmd = "ls /etc/hiera"
            logger.debug('Checking hiera files for removal...')
            result = self.ssh_manager.execute(ip, cmd)
            if result['exit_code'] == 0:
                logger.debug('Found redundant hiera files. Removing...')
                cmd = "rm -rf /etc/hiera"
                self.ssh_manager.execute_on_remote(
                    ip, cmd, err_msg='Can not remove hiera files')

    def check_rsync_installation(self, ip):
        """Check rsync installation on node.

        :param ip: Remote node ip for proceed.
        """
        cmd = "yum list installed | grep rsync"
        logger.debug("Checking rsync installation...")
        result = self.ssh_manager.execute(ip, cmd)
        if result['exit_code'] != 0:
            logger.debug("Rsync is not found. Installing rsync...")
            cmd = "yum clean all && yum install rsync -y"
            self.ssh_manager.execute_on_remote(
                ip, cmd, err_msg='Can not install rsync on node.')

    def remove_old_compute_services(self, ip, hostname):
        """Remove old redundant services which was removed from services base.

        :param ip: Remote node ip for proceed.
        :param hostname: Old compute hostname.
        """
        cmd = ("source ~/openrc && for i in $(nova service-list | "
               "awk '/{:s}/{{print $2}}'); do nova service-delete $i; "
               "done".format(hostname))
        self.ssh_manager.execute_on_remote(
            ip, cmd, err_msg='Can not remove old nova computes')

        cmd = ("source ~/openrc && for i in $(neutron agent-list | "
               "awk '/{:s}/{{print $2}}'); do neutron agent-delete $i; "
               "done".format(hostname))
        self.ssh_manager.execute_on_remote(
            ip, cmd, err_msg='Can not remove old neutron agents')

    def install_ruby_puppet(self, ip):
        """Install ruby and puppet on a target node.

        :param ip: Remote node ip for proceed.
        """
        puppet_install_cmd = "yum install puppet ruby -y"
        self.ssh_manager.execute_on_remote(
            ip, puppet_install_cmd,
            err_msg='Ruby and puppet installation failed')

    def rsync_puppet_modules(self, master_node_ip, ip):
        """Rsync puppet modules from remote node to node with specified ip.

        :param master_node_ip: Remote node ip for proceed.
        :param ip: IP address of a target node where to sync.
        """
        cmd = ("rsync -avz /etc/puppet/modules/* "
               "{0}@{1}:/etc/puppet/modules/".
               format(settings.EXTRA_COMP_IMAGE_USER, ip))
        self.ssh_manager.execute_on_remote(
            master_node_ip, cmd, err_msg='Rsync puppet modules failed')

    def save_node_hostname(self, ip):
        """Save hostname of a node.

        :param ip: Remote node ip for proceed.
        :return: Node hostname.
        """
        cmd = "hostname"
        result = self.ssh_manager.execute_on_remote(
            ip, cmd, err_msg='Can not get hostname for remote')
        nodename = self.clean_string(result['stdout'][0], twice=False)
        return nodename

    def backup_hosts_file(self, ip, target_ip):
        """Backing up hosts file

        :param ip: Remote node ip to backup to.
        :param target_ip: Node ip to backup from.
        """
        cmd = "cd ~/ && scp {0}:/etc/hosts .".format(target_ip)
        self.ssh_manager.execute_on_remote(
            ip, cmd, err_msg='Can not save hosts file from remote')

    def prepare_hosts_file(self, ip, old_host, new_host):
        cmd = "cd ~/ && sed -i 's/{0}/{1}/g' hosts".format(old_host, new_host)
        self.ssh_manager.execute_on_remote(
            ip, cmd, err_msg='Can not prepare hosts file.')

    def restore_hosts_file(self, ip, target_ip):
        """Restore host file

        :param ip: Node ip to restore from.
        :param target_ip: Node ip to restore to.
        """
        cmd = "cd ~/ && scp hosts {0}:/etc/".format(target_ip)
        self.ssh_manager.execute_on_remote(
            ip, cmd, err_msg='Can not restore hosts file.')

    def clean_custom_repos(self, ip):
        """Remove custom repo files

        :param ip: Node ip to clean
        """
        cmd = "cd /etc/yum.repos.d && rm -f *mir* && yum clean all"
        self.ssh_manager.execute_on_remote(
            ip, cmd, err_msg='Can not clean custom repos.')