# Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import functools import logging import shutil import socket import sys import time from distutils import version from octane.util import helpers from octane.util import ssh LOG = logging.getLogger(__name__) def preserve_partition(node, partition): disks = node.get_attribute('disks') for disk in disks: for vol in disk['volumes']: if vol['name'] == partition: vol.update({ 'keep': True, 'keep_data': True, }) node.upload_node_attribute('disks', disks) def get_ip(network_name, node): for net in node.data['network_data']: if net['name'] == network_name: return net['ip'] def get_ips(network_name, nodes): get_network_ip = functools.partial(get_ip, network_name) return map(get_network_ip, nodes) def get_hostnames(nodes): return [node.data['fqdn'] for node in nodes] def tar_files(filename, node, *files): cmd = ['tar', '-czvP'] cmd.extend(files) with ssh.popen(cmd, stdout=ssh.PIPE, node=node) as proc: with open(filename, 'wb') as f: shutil.copyfileobj(proc.stdout, f) def untar_files(filename, node): cmd = ['tar', '-xzv', '-C', '/'] with ssh.popen(cmd, stdin=ssh.PIPE, node=node) as proc: with open(filename, 'rb') as f: shutil.copyfileobj(f, proc.stdin) def get_hostname_remotely(node): hostname = ssh.call_output(['hostname'], node=node) return hostname[:-1] def get_nova_node_handle(node): version_num = node.env.data.get('fuel_version') if not version_num: raise Exception("Cannot determine Fuel version for node {0}" .format(node.data['id'])) if version.StrictVersion(version_num) < version.StrictVersion('6.1'): return "node-{0}".format(node.data['id']) return node.data['fqdn'] def reboot_nodes(nodes, timeout=600): old_clients = dict((node, ssh.get_client(node)) for node in nodes) for node in nodes: ssh.call(['reboot'], node=node) node_ids_str = ", ".join(str(node.data['id']) for node in nodes) LOG.debug("Sent reboot command to nodes: %s", node_ids_str) wait_list = set(nodes) start = time.time() while wait_list: time.sleep(10) done = set() for node in wait_list: if time.time() - start > timeout: failed = ", ".join(str(node.data['id']) for node in wait_list) raise Exception( "Timeout waiting for nodes {0} to reboot".format( failed)) try: new_client = ssh.get_client(node) except socket.error: LOG.debug("Failed to connect to node %s", node.data['id'], exc_info=sys.exc_info()) node.update() # IP could've been changed, use new IP next time continue if new_client != old_clients[node]: done.add(node) wait_list -= done def wait_for_mcollective_start(nodes, timeout=600): start_at = time.time() wait_list = set(nodes) node_ids_str = ", ".join(str(node.data['id']) for node in nodes) LOG.info("Wait for mcollective start on nodes {0}".format(node_ids_str)) while wait_list: time.sleep(10) done = set() for node in wait_list: try: ssh.call(['service', 'mcollective', 'status'], node=node) except Exception as e: LOG.debug(e) else: done.add(node) if time.time() - start_at > timeout: failed = ", ".join(str(node.data['id'] for node in wait_list)) raise Exception("Timeout waiting for nodes {0} to start" " mcollective".format(failed)) wait_list -= done def add_compute_upgrade_levels(node, version): sftp = ssh.sftp(node) with ssh.update_file(sftp, '/etc/nova/nova.conf') as (old, new): add_upgrade_levels = True in_section = False for line in old: if line.startswith("[upgrade_levels]"): add_upgrade_levels = False in_section = True new.write(line) new.write("compute={0}\n".format(version)) continue if in_section and line.startswith("["): in_section = False if in_section and line.startswith("compute="): LOG.warning( "Skipping line so not to duplicate compute " "upgrade level setting: %s" % line.rstrip()) continue new.write(line) if add_upgrade_levels: new.write("[upgrade_levels]\ncompute={0}\n".format(version)) def remove_compute_upgrade_levels(node): sftp = ssh.sftp(node) with ssh.update_file(sftp, '/etc/nova/nova.conf') as (old, new): for line in old: if line.startswith("compute="): continue new.write(line) def is_live_migration_supported(node): sftp = ssh.sftp(node) with sftp.open('/etc/nova/nova.conf') as config: for line in config: if line.strip().startswith("live_migration_flag") \ and "VIR_MIGRATE_LIVE" in line: return True return False def restart_nova_services(node): nova_services = ssh.call_output(["service", "--status-all"], node=node) for service_line in nova_services.splitlines(): service_line = service_line.strip() _, status, _, service = service_line.split() if status == "+" and service.startswith("nova"): ssh.call(["service", service, "restart"], node=node) class AbsentParametersError(Exception): msg = "Could not get parameters from the file " \ "node-{node_id}[{filename}]: {parameters}" def __init__(self, node_id, filename, parameters): super(AbsentParametersError, self).__init__(self.msg.format( node_id=node_id, filename=filename, parameters=", ".join(parameters), )) def get_parameters(node, filename, parameters_to_get, ensure=True): with ssh.sftp(node).open(filename) as fp: parameters = helpers.get_parameters(fp, parameters_to_get) if ensure: required_parameters = set(parameters_to_get) current_parameters = set(parameters) absent_parameters = required_parameters - current_parameters if absent_parameters: flat_parameters = [] for aparam in absent_parameters: for param in parameters_to_get[aparam]: flat_parameters.append("/".join(param)) raise AbsentParametersError( node.data["id"], filename, flat_parameters) return parameters