33a8b5abb2
The backend name for volumes was changed since Kilo. To make these volumes back manageable in Mitaka their hostnames have to be changed. The `cinder-manage volume update_host` command is suitable to do that. Change-Id: Ieeadd81c714186d58a3e3dfe5fd1223c40d16996 Partial-Bug: #1624341
215 lines
7.2 KiB
Python
215 lines
7.2 KiB
Python
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
import functools
|
|
import logging
|
|
import shutil
|
|
import socket
|
|
import sys
|
|
import time
|
|
|
|
from distutils import version
|
|
from octane.util import helpers
|
|
from octane.util import ssh
|
|
|
|
LOG = logging.getLogger(__name__)
|
|
|
|
|
|
def preserve_partition(node, partition):
|
|
disks = node.get_attribute('disks')
|
|
for disk in disks:
|
|
for vol in disk['volumes']:
|
|
if vol['name'] == partition:
|
|
vol.update({
|
|
'keep': True,
|
|
'keep_data': True,
|
|
})
|
|
|
|
node.upload_node_attribute('disks', disks)
|
|
|
|
|
|
def get_ip(network_name, node):
|
|
for net in node.data['network_data']:
|
|
if net['name'] == network_name:
|
|
return net['ip']
|
|
|
|
|
|
def get_ips(network_name, nodes):
|
|
get_network_ip = functools.partial(get_ip, network_name)
|
|
return map(get_network_ip, nodes)
|
|
|
|
|
|
def get_hostnames(nodes):
|
|
return [node.data['fqdn'] for node in nodes]
|
|
|
|
|
|
def tar_files(filename, node, *files):
|
|
cmd = ['tar', '-czvP']
|
|
cmd.extend(files)
|
|
with ssh.popen(cmd, stdout=ssh.PIPE, node=node) as proc:
|
|
with open(filename, 'wb') as f:
|
|
shutil.copyfileobj(proc.stdout, f)
|
|
|
|
|
|
def untar_files(filename, node):
|
|
cmd = ['tar', '-xzv', '-C', '/']
|
|
with ssh.popen(cmd, stdin=ssh.PIPE, node=node) as proc:
|
|
with open(filename, 'rb') as f:
|
|
shutil.copyfileobj(f, proc.stdin)
|
|
|
|
|
|
def get_hostname_remotely(node):
|
|
hostname = ssh.call_output(['hostname'], node=node)
|
|
return hostname[:-1]
|
|
|
|
|
|
def get_nova_node_handle(node):
|
|
version_num = node.env.data.get('fuel_version')
|
|
if not version_num:
|
|
raise Exception("Cannot determine Fuel version for node {0}"
|
|
.format(node.data['id']))
|
|
if version.StrictVersion(version_num) < version.StrictVersion('6.1'):
|
|
return "node-{0}".format(node.data['id'])
|
|
return node.data['fqdn']
|
|
|
|
|
|
def reboot_nodes(nodes, timeout=600):
|
|
old_clients = dict((node, ssh.get_client(node)) for node in nodes)
|
|
for node in nodes:
|
|
ssh.call(['reboot'], node=node)
|
|
node_ids_str = ", ".join(str(node.data['id']) for node in nodes)
|
|
LOG.debug("Sent reboot command to nodes: %s", node_ids_str)
|
|
wait_list = set(nodes)
|
|
start = time.time()
|
|
while wait_list:
|
|
time.sleep(10)
|
|
done = set()
|
|
for node in wait_list:
|
|
if time.time() - start > timeout:
|
|
failed = ", ".join(str(node.data['id']) for node in wait_list)
|
|
raise Exception(
|
|
"Timeout waiting for nodes {0} to reboot".format(
|
|
failed))
|
|
try:
|
|
new_client = ssh.get_client(node)
|
|
except socket.error:
|
|
LOG.debug("Failed to connect to node %s", node.data['id'],
|
|
exc_info=sys.exc_info())
|
|
node.update() # IP could've been changed, use new IP next time
|
|
continue
|
|
if new_client != old_clients[node]:
|
|
done.add(node)
|
|
wait_list -= done
|
|
|
|
|
|
def wait_for_mcollective_start(nodes, timeout=600):
|
|
start_at = time.time()
|
|
wait_list = set(nodes)
|
|
node_ids_str = ", ".join(str(node.data['id']) for node in nodes)
|
|
LOG.info("Wait for mcollective start on nodes {0}".format(node_ids_str))
|
|
while wait_list:
|
|
time.sleep(10)
|
|
done = set()
|
|
for node in wait_list:
|
|
try:
|
|
ssh.call(['service', 'mcollective', 'status'], node=node)
|
|
except Exception as e:
|
|
LOG.debug(e)
|
|
else:
|
|
done.add(node)
|
|
if time.time() - start_at > timeout:
|
|
failed = ", ".join(str(node.data['id'] for node in wait_list))
|
|
raise Exception("Timeout waiting for nodes {0} to start"
|
|
" mcollective".format(failed))
|
|
wait_list -= done
|
|
|
|
|
|
def add_compute_upgrade_levels(node, version):
|
|
sftp = ssh.sftp(node)
|
|
with ssh.update_file(sftp, '/etc/nova/nova.conf') as (old, new):
|
|
add_upgrade_levels = True
|
|
in_section = False
|
|
for line in old:
|
|
if line.startswith("[upgrade_levels]"):
|
|
add_upgrade_levels = False
|
|
in_section = True
|
|
new.write(line)
|
|
new.write("compute={0}\n".format(version))
|
|
continue
|
|
if in_section and line.startswith("["):
|
|
in_section = False
|
|
if in_section and line.startswith("compute="):
|
|
LOG.warning(
|
|
"Skipping line so not to duplicate compute "
|
|
"upgrade level setting: %s" % line.rstrip())
|
|
continue
|
|
new.write(line)
|
|
if add_upgrade_levels:
|
|
new.write("[upgrade_levels]\ncompute={0}\n".format(version))
|
|
|
|
|
|
def remove_compute_upgrade_levels(node):
|
|
sftp = ssh.sftp(node)
|
|
with ssh.update_file(sftp, '/etc/nova/nova.conf') as (old, new):
|
|
for line in old:
|
|
if line.startswith("compute="):
|
|
continue
|
|
new.write(line)
|
|
|
|
|
|
def is_live_migration_supported(node):
|
|
sftp = ssh.sftp(node)
|
|
with sftp.open('/etc/nova/nova.conf') as config:
|
|
for line in config:
|
|
if line.strip().startswith("live_migration_flag") \
|
|
and "VIR_MIGRATE_LIVE" in line:
|
|
return True
|
|
return False
|
|
|
|
|
|
def restart_nova_services(node):
|
|
nova_services = ssh.call_output(["service", "--status-all"], node=node)
|
|
for service_line in nova_services.splitlines():
|
|
service_line = service_line.strip()
|
|
_, status, _, service = service_line.split()
|
|
if status == "+" and service.startswith("nova"):
|
|
ssh.call(["service", service, "restart"], node=node)
|
|
|
|
|
|
class AbsentParametersError(Exception):
|
|
msg = "Could not get parameters from the file " \
|
|
"node-{node_id}[{filename}]: {parameters}"
|
|
|
|
def __init__(self, node_id, filename, parameters):
|
|
super(AbsentParametersError, self).__init__(self.msg.format(
|
|
node_id=node_id,
|
|
filename=filename,
|
|
parameters=", ".join(parameters),
|
|
))
|
|
|
|
|
|
def get_parameters(node, filename, parameters_to_get, ensure=True):
|
|
with ssh.sftp(node).open(filename) as fp:
|
|
parameters = helpers.get_parameters(fp, parameters_to_get)
|
|
if ensure:
|
|
required_parameters = set(parameters_to_get)
|
|
current_parameters = set(parameters)
|
|
absent_parameters = required_parameters - current_parameters
|
|
if absent_parameters:
|
|
flat_parameters = []
|
|
for aparam in absent_parameters:
|
|
for param in parameters_to_get[aparam]:
|
|
flat_parameters.append("/".join(param))
|
|
raise AbsentParametersError(
|
|
node.data["id"], filename, flat_parameters)
|
|
return parameters
|