6293 lines
234 KiB
Python
Executable File
6293 lines
234 KiB
Python
Executable File
#
|
||
# Copyright (c) 2019 Wind River Systems, Inc.
|
||
#
|
||
# SPDX-License-Identifier: Apache-2.0
|
||
#
|
||
|
||
|
||
import copy
|
||
import math
|
||
import os
|
||
import random
|
||
import re
|
||
import time
|
||
import ipaddress
|
||
import pexpect
|
||
from contextlib import contextmanager
|
||
|
||
from consts.auth import Tenant, TestFileServer, HostLinuxUser
|
||
from consts.stx import VMStatus, NovaCLIOutput, EXT_IP, ImageStatus, \
|
||
VMNetwork, EventLogID, GuestImages, Networks, FlavorSpec, VimEventID
|
||
from consts.filepaths import VMPath, UserData, TestServerPath
|
||
from consts.proj_vars import ProjVar
|
||
from consts.timeout import VMTimeout, CMDTimeout
|
||
from utils import exceptions, cli, table_parser, multi_thread
|
||
from utils import local_host
|
||
from utils.clients.ssh import NATBoxClient, VMSSHClient, ControllerClient, \
|
||
Prompt, get_cli_client
|
||
from utils.clients.local import LocalHostClient
|
||
from utils.guest_scripts.scripts import TisInitServiceScript
|
||
from utils.multi_thread import MThread, Events
|
||
from utils.tis_log import LOG
|
||
from keywords import network_helper, nova_helper, cinder_helper, host_helper, \
|
||
glance_helper, common, system_helper, \
|
||
storage_helper
|
||
from testfixtures.fixture_resources import ResourceCleanup
|
||
from testfixtures.recover_hosts import HostsToRecover
|
||
|
||
|
||
def set_vm(vm_id, name=None, state=None, con_ssh=None, auth_info=None,
|
||
fail_ok=False, **properties):
|
||
"""
|
||
Set vm with given parameters - name, state, and/or properties
|
||
Args:
|
||
vm_id:
|
||
name:
|
||
state:
|
||
con_ssh:
|
||
auth_info:
|
||
fail_ok:
|
||
**properties:
|
||
|
||
Returns (tuple):
|
||
(0, )
|
||
|
||
"""
|
||
args_dict = {
|
||
'--name': name,
|
||
'--state': state.lower() if state else None,
|
||
'--property': properties,
|
||
}
|
||
args = '{} {}'.format(common.parse_args(args_dict, repeat_arg=True), vm_id)
|
||
LOG.info("Setting vm with args: {}".format(args))
|
||
code, output = cli.openstack('server set', args, ssh_client=con_ssh,
|
||
fail_ok=fail_ok, auth_info=auth_info)
|
||
if code > 0:
|
||
return 1, output
|
||
|
||
msg = "VM {} is set successfully.".format(vm_id)
|
||
LOG.info(msg)
|
||
return 0, msg
|
||
|
||
|
||
def unset_vm(vm_id, properties, con_ssh=None, auth_info=None, fail_ok=False):
|
||
"""
|
||
Unset given properties for VM
|
||
Args:
|
||
vm_id:
|
||
properties:
|
||
con_ssh:
|
||
auth_info:
|
||
fail_ok:
|
||
|
||
Returns (tuple):
|
||
(1, <std_err>) - cli rejected
|
||
(0, "VM <vm_id> properties unset successfully: <properties>")
|
||
|
||
"""
|
||
if isinstance(properties, str):
|
||
properties = (properties,)
|
||
|
||
args = '{} {}'.format(
|
||
common.parse_args({'--property': properties}, repeat_arg=True), vm_id)
|
||
LOG.info("Unsetting vm {} properties: {}".format(vm_id, properties))
|
||
code, output = cli.openstack('server unset', args, ssh_client=con_ssh,
|
||
fail_ok=fail_ok, auth_info=auth_info)
|
||
if code > 0:
|
||
return 1, output
|
||
|
||
msg = "VM {} properties unset successfully: {}".format(vm_id, properties)
|
||
LOG.info(msg)
|
||
return 0, msg
|
||
|
||
|
||
def get_any_vms(count=None, con_ssh=None, auth_info=None, all_tenants=False,
|
||
rtn_new=False):
|
||
"""
|
||
Get a list of ids of any active vms.
|
||
|
||
Args:
|
||
count (int): number of vms ids to return. If None, all vms for
|
||
specific tenant will be returned. If num of
|
||
existing vm is less than count additional vm will be created to match
|
||
the count
|
||
con_ssh (SSHClient):
|
||
auth_info (dict):
|
||
all_tenants (bool): whether to get any vms from all tenants or just
|
||
admin tenant if auth_info is set to Admin
|
||
rtn_new (bool): whether to return an extra list containing only the
|
||
newly created vms
|
||
|
||
Returns (list):
|
||
vms(list) # rtn_new=False
|
||
[vms(list), new_vms(list)] # rtn_new=True
|
||
|
||
"""
|
||
vms = get_vms(con_ssh=con_ssh, auth_info=auth_info,
|
||
all_projects=all_tenants, Status='ACTIVE')
|
||
if count is None:
|
||
if rtn_new:
|
||
vms = [vms, []]
|
||
return vms
|
||
diff = count - len(vms)
|
||
if diff <= 0:
|
||
vms = random.sample(vms, count)
|
||
if rtn_new:
|
||
vms = [vms, []]
|
||
return vms
|
||
|
||
new_vms = []
|
||
for i in range(diff):
|
||
new_vm = boot_vm(con_ssh=con_ssh, auth_info=auth_info)[1]
|
||
vms.append(new_vm)
|
||
new_vms.append(new_vm)
|
||
|
||
if rtn_new:
|
||
vms = [vms, new_vms]
|
||
return vms
|
||
|
||
|
||
def create_image_from_vm(vm_id, image_name=None, wait=True,
|
||
expt_cinder_snapshot=None,
|
||
fail_ok=False, con_ssh=None, auth_info=None,
|
||
cleanup=None):
|
||
"""
|
||
Create glance image from an existing vm
|
||
Args:
|
||
vm_id:
|
||
image_name:
|
||
wait:
|
||
expt_cinder_snapshot (bool): if vm was booted from cinder volume,
|
||
then a cinder snapshot is expected
|
||
fail_ok:
|
||
con_ssh:
|
||
auth_info:
|
||
cleanup (None|str): valid scopes: function, class, module, session
|
||
|
||
Returns (tuple):
|
||
|
||
"""
|
||
LOG.info("Creating image from vm {}".format(vm_id))
|
||
args_dict = {'--name': image_name, '--wait': wait}
|
||
args = '{} {}'.format(common.parse_args(args_dict), vm_id)
|
||
code, out = cli.openstack('server image create', args, ssh_client=con_ssh,
|
||
fail_ok=fail_ok, auth_info=auth_info)
|
||
|
||
table_ = table_parser.table(out)
|
||
image_id = table_parser.get_value_two_col_table(table_, 'id')
|
||
cinder_snapshot_id = None
|
||
if cleanup and image_id:
|
||
ResourceCleanup.add('image', image_id, scope=cleanup)
|
||
|
||
if code > 0:
|
||
return 1, out, cinder_snapshot_id
|
||
|
||
post_name = table_parser.get_value_two_col_table(table_, 'name')
|
||
if image_name and image_name != post_name:
|
||
raise exceptions.NovaError(
|
||
"Create image does not expected name. Actual {}, expected: "
|
||
"{}".format(post_name, image_name))
|
||
|
||
LOG.info(
|
||
"Wait for created image {} to reach active state".format(post_name))
|
||
glance_helper.wait_for_image_status(image_id, status=ImageStatus.ACTIVE,
|
||
con_ssh=con_ssh, auth_info=auth_info)
|
||
|
||
image_size = table_parser.get_value_two_col_table(table_, 'size')
|
||
if str(image_size) == '0' or expt_cinder_snapshot:
|
||
cinder_snapshotname = "snapshot for {}".format(post_name)
|
||
vol_snapshots = cinder_helper.get_vol_snapshots(
|
||
name=cinder_snapshotname)
|
||
if not vol_snapshots:
|
||
raise exceptions.CinderError(
|
||
"cinder snapshot expected, but was not found: {}".format(
|
||
cinder_snapshotname))
|
||
cinder_snapshot_id = vol_snapshots[0]
|
||
if cleanup:
|
||
ResourceCleanup.add('vol_snapshot', cinder_snapshot_id)
|
||
|
||
LOG.info("glance image {} successfully created from vm {}".format(post_name,
|
||
vm_id))
|
||
return 0, image_id, cinder_snapshot_id
|
||
|
||
|
||
def add_security_group(vm_id, security_group, fail_ok=False, con_ssh=None,
|
||
auth_info=None):
|
||
"""
|
||
Add given security group to vm
|
||
Args:
|
||
vm_id:
|
||
security_group:
|
||
fail_ok:
|
||
con_ssh:
|
||
auth_info:
|
||
|
||
Returns (tuple):
|
||
|
||
"""
|
||
LOG.info("Adding security group {} to vm {}".format(security_group, vm_id))
|
||
args = '{} {}'.format(vm_id, security_group)
|
||
code, output = cli.openstack('server add security group', args,
|
||
ssh_client=con_ssh, fail_ok=fail_ok,
|
||
auth_info=auth_info)
|
||
if code > 0:
|
||
return 1, output
|
||
|
||
msg = "Security group {} added to VM {} successfully".format(security_group,
|
||
vm_id)
|
||
LOG.info(msg)
|
||
return 0, msg
|
||
|
||
|
||
def wait_for_vol_attach(vm_id, vol_id, timeout=VMTimeout.VOL_ATTACH,
|
||
con_ssh=None, auth_info=None, fail_ok=False):
|
||
"""
|
||
Wait for volume attachment appear in openstack server show as well as
|
||
opentstack volume show
|
||
Args:
|
||
vm_id:
|
||
vol_id:
|
||
timeout:
|
||
con_ssh:
|
||
auth_info:
|
||
fail_ok:
|
||
|
||
Returns (bool):
|
||
|
||
"""
|
||
end_time = time.time() + timeout
|
||
while time.time() < end_time:
|
||
vols_attached = get_vm_volumes(vm_id=vm_id, con_ssh=con_ssh,
|
||
auth_info=auth_info)
|
||
if vol_id in vols_attached:
|
||
cinder_helper.wait_for_volume_status(vol_id, status='in-use',
|
||
timeout=120, fail_ok=False,
|
||
con_ssh=con_ssh,
|
||
auth_info=auth_info)
|
||
return True
|
||
time.sleep(5)
|
||
else:
|
||
msg = "Volume {} is not shown in nova show {} in {} seconds".format(
|
||
vol_id, vm_id, timeout)
|
||
LOG.warning(msg)
|
||
if not fail_ok:
|
||
raise exceptions.VMError(msg)
|
||
return False
|
||
|
||
|
||
def attach_vol_to_vm(vm_id, vol_id=None, device=None, mount=False, con_ssh=None,
|
||
auth_info=None, fail_ok=False,
|
||
cleanup=None):
|
||
"""
|
||
Attach a volume to VM
|
||
Args:
|
||
vm_id (str):
|
||
vol_id (str|None): volume to attach. When None, a non-bootable volume
|
||
will be created to attach to given vm
|
||
device (str|None): whether to specify --device in cmd
|
||
mount (bool): if True, login to vm and attempt to mount the device
|
||
after attached. Best effort only.
|
||
con_ssh:
|
||
auth_info:
|
||
fail_ok:
|
||
cleanup:
|
||
|
||
Returns:
|
||
|
||
"""
|
||
if not vol_id:
|
||
vol_id = \
|
||
cinder_helper.create_volume(bootable=False, auth_info=auth_info,
|
||
con_ssh=con_ssh, cleanup=cleanup)[1]
|
||
|
||
LOG.info("Attaching volume {} to vm {}".format(vol_id, vm_id))
|
||
args = '{}{} {}'.format('--device {} '.format(device) if device else '',
|
||
vm_id, vol_id)
|
||
code, output = cli.openstack('server add volume', args, ssh_client=con_ssh,
|
||
fail_ok=fail_ok, auth_info=auth_info)
|
||
if code > 0:
|
||
return 1, output
|
||
|
||
LOG.info(
|
||
"Waiting for attached volume to appear in openstack server show and "
|
||
"volume show")
|
||
wait_for_vol_attach(vm_id=vm_id, vol_id=vol_id, con_ssh=con_ssh,
|
||
auth_info=auth_info)
|
||
|
||
if mount:
|
||
LOG.info("Mount attached volume {} to vm {}".format(vol_id, vm_id))
|
||
guest = get_vm_image_name(vm_id)
|
||
if not (guest and 'cgcs-guest' in guest):
|
||
attached_devs = get_vm_volume_attachments(vm_id=vm_id,
|
||
field='device',
|
||
vol_id=vol_id,
|
||
auth_info=auth_info,
|
||
con_ssh=con_ssh)
|
||
device_name = attached_devs[0]
|
||
device = device_name.split('/')[-1]
|
||
LOG.info(
|
||
"Volume {} is attached to VM {} as {}".format(vol_id, vm_id,
|
||
device_name))
|
||
mount_attached_volume(vm_id, device, vm_image_name=guest)
|
||
|
||
return 0, vol_id
|
||
|
||
|
||
def is_attached_volume_mounted(vm_id, rootfs, vm_image_name=None, vm_ssh=None):
|
||
"""
|
||
Checks if an attached volume is mounted in VM
|
||
Args:
|
||
vm_id (str): - the vm uuid where the volume is attached to
|
||
rootfs (str) - the device name of the attached volume like vda, vdb,
|
||
vdc, ....
|
||
vm_image_name (str): - the guest image the vm is booted with
|
||
vm_ssh (VMSSHClient): ssh client session to vm
|
||
Returns: bool
|
||
|
||
"""
|
||
|
||
if vm_image_name is None:
|
||
vm_image_name = get_vm_image_name(vm_id)
|
||
|
||
cmd = "mount | grep {} | wc -l".format(rootfs)
|
||
mounted_msg = "Filesystem /dev/{} is mounted: {}".format(rootfs, vm_id)
|
||
not_mount_msg = "Filesystem /dev/{} is not mounted: {}".format(rootfs,
|
||
vm_id)
|
||
if vm_ssh:
|
||
cmd_output = vm_ssh.exec_sudo_cmd(cmd)[1]
|
||
if cmd_output != '0':
|
||
LOG.info(mounted_msg)
|
||
return True
|
||
LOG.info(not_mount_msg)
|
||
return False
|
||
|
||
with ssh_to_vm_from_natbox(vm_id, vm_image_name=vm_image_name) as vm_ssh:
|
||
|
||
cmd_output = vm_ssh.exec_sudo_cmd(cmd)[1]
|
||
if cmd_output != '0':
|
||
LOG.info(mounted_msg)
|
||
return True
|
||
LOG.info(not_mount_msg)
|
||
return False
|
||
|
||
|
||
def get_vm_volume_attachments(vm_id, vol_id=None, field='device', con_ssh=None,
|
||
auth_info=Tenant.get('admin')):
|
||
"""
|
||
Get volume attachments for given vm
|
||
Args:
|
||
vm_id:
|
||
vol_id:
|
||
field:
|
||
con_ssh:
|
||
auth_info:
|
||
|
||
Returns (list):
|
||
|
||
"""
|
||
# No replacement in openstack client
|
||
table_ = table_parser.table(
|
||
cli.nova('volume-attachments', vm_id, ssh_client=con_ssh,
|
||
auth_info=auth_info)[1])
|
||
return table_parser.get_values(table_, field, **{'volume id': vol_id})
|
||
|
||
|
||
def mount_attached_volume(vm_id, rootfs, vm_image_name=None):
|
||
"""
|
||
Mounts an attached volume on VM
|
||
Args:
|
||
vm_id (str): - the vm uuid where the volume is attached to
|
||
rootfs (str) - the device name of the attached volume like vda, vdb,
|
||
vdc, ....
|
||
vm_image_name (str): - the guest image the vm is booted with
|
||
|
||
Returns: bool
|
||
|
||
"""
|
||
wait_for_vm_pingable_from_natbox(vm_id)
|
||
if vm_image_name is None:
|
||
vm_image_name = get_vm_image_name(vm_id)
|
||
|
||
with ssh_to_vm_from_natbox(vm_id, vm_image_name=vm_image_name) as vm_ssh:
|
||
|
||
if not is_attached_volume_mounted(vm_id, rootfs,
|
||
vm_image_name=vm_image_name,
|
||
vm_ssh=vm_ssh):
|
||
LOG.info("Creating ext4 file system on /dev/{} ".format(rootfs))
|
||
cmd = "mkfs -t ext4 /dev/{}".format(rootfs)
|
||
rc, output = vm_ssh.exec_cmd(cmd)
|
||
if rc != 0:
|
||
msg = "Failed to create filesystem on /dev/{} for vm " \
|
||
"{}: {}".format(rootfs, vm_id, output)
|
||
LOG.warning(msg)
|
||
return False
|
||
LOG.info("Mounting /dev/{} to /mnt/volume".format(rootfs))
|
||
cmd = "test -e /mnt/volume"
|
||
rc, output = vm_ssh.exec_cmd(cmd)
|
||
mount_cmd = ''
|
||
if rc == 1:
|
||
mount_cmd += "mkdir -p /mnt/volume; mount /dev/{} " \
|
||
"/mnt/volume".format(rootfs)
|
||
else:
|
||
mount_cmd += "mount /dev/{} /mnt/volume".format(rootfs)
|
||
|
||
rc, output = vm_ssh.exec_cmd(mount_cmd)
|
||
if rc != 0:
|
||
msg = "Failed to mount /dev/{} for vm {}: {}".format(rootfs,
|
||
vm_id,
|
||
output)
|
||
LOG.warning(msg)
|
||
return False
|
||
|
||
LOG.info(
|
||
"Adding /dev/{} mounting point in /etc/fstab".format(rootfs))
|
||
cmd = "echo \"/dev/{} /mnt/volume ext4 defaults 0 0\" >> " \
|
||
"/etc/fstab".format(rootfs)
|
||
|
||
rc, output = vm_ssh.exec_cmd(cmd)
|
||
if rc != 0:
|
||
msg = "Failed to add /dev/{} mount point to /etc/fstab for " \
|
||
"vm {}: {}".format(rootfs, vm_id, output)
|
||
LOG.warning(msg)
|
||
|
||
LOG.info(
|
||
"/dev/{} is mounted to /mnt/volume for vm {}".format(rootfs,
|
||
vm_id))
|
||
return True
|
||
else:
|
||
LOG.info(
|
||
"/dev/{} is already mounted to /mnt/volume for vm {}".format(
|
||
rootfs, vm_id))
|
||
return True
|
||
|
||
|
||
def get_vm_devices_via_virsh(vm_id, con_ssh=None):
|
||
"""
|
||
Get vm disks in dict format via 'virsh domblklist <instance_name>'
|
||
Args:
|
||
vm_id (str):
|
||
con_ssh:
|
||
|
||
Returns (dict): vm disks per type.
|
||
Examples:
|
||
{'root_img': {'vda': '/dev/nova-local/a746beb9-08e4-4b08-af2a
|
||
-000c8ca72851_disk'},
|
||
'attached_vol': {'vdb': '/dev/disk/by-path/ip-192.168.205.106:3260-iscsi
|
||
-iqn.2010-10.org.openstack:volume-...'},
|
||
'swap': {},
|
||
'eph': {}}
|
||
|
||
"""
|
||
vm_host = get_vm_host(vm_id=vm_id, con_ssh=con_ssh)
|
||
inst_name = get_vm_instance_name(vm_id=vm_id, con_ssh=con_ssh)
|
||
|
||
with host_helper.ssh_to_host(vm_host, con_ssh=con_ssh) as host_ssh:
|
||
output = host_ssh.exec_sudo_cmd('virsh domblklist {}'.format(inst_name),
|
||
fail_ok=False)[1]
|
||
disk_lines = output.split('-------------------------------\n', 1)[
|
||
-1].splitlines()
|
||
|
||
disks = {}
|
||
root_line = disk_lines.pop(0)
|
||
root_dev, root_source = root_line.split()
|
||
if re.search('openstack:volume|cinder-volumes|/dev/sd', root_source):
|
||
disk_type = 'root_vol'
|
||
else:
|
||
disk_type = 'root_img'
|
||
disks[disk_type] = {root_dev: root_source}
|
||
LOG.info("Root disk: {}".format(disks))
|
||
|
||
disks.update({'eph': {}, 'swap': {}, 'attached_vol': {}})
|
||
for line in disk_lines:
|
||
dev, source = line.split()
|
||
if re.search('disk.swap', source):
|
||
disk_type = 'swap'
|
||
elif re.search('openstack:volume|cinder-volumes|/dev/sd', source):
|
||
disk_type = 'attached_vol'
|
||
elif re.search('disk.eph|disk.local', source):
|
||
disk_type = 'eph'
|
||
else:
|
||
raise exceptions.CommonError(
|
||
"Unknown disk in virsh: {}. Automation update "
|
||
"required.".format(
|
||
line))
|
||
disks[disk_type][dev] = source
|
||
|
||
LOG.info("disks for vm {}: {}".format(vm_id, disks))
|
||
return disks
|
||
|
||
|
||
def get_vm_boot_volume_via_virsh(vm_id, con_ssh=None):
|
||
"""
|
||
Get cinder volume id where the vm is booted from via virsh cmd.
|
||
Args:
|
||
vm_id (str):
|
||
con_ssh (SSHClient):
|
||
|
||
Returns (str|None): vol_id or None if vm is not booted from cinder volume
|
||
|
||
"""
|
||
disks = get_vm_devices_via_virsh(vm_id=vm_id, con_ssh=con_ssh)
|
||
root_vol = disks.get('root_vol', {})
|
||
if not root_vol:
|
||
LOG.info("VM is not booted from volume. Return None")
|
||
return
|
||
|
||
root_vol = list(root_vol.values())[0]
|
||
root_vol = re.findall('openstack:volume-(.*)-lun', root_vol)[0]
|
||
LOG.info("vm {} is booted from cinder volume {}".format(vm_id, root_vol))
|
||
return root_vol
|
||
|
||
|
||
def auto_mount_vm_devices(vm_id, devices, guest_os=None, check_first=True,
|
||
vm_ssh=None):
|
||
"""
|
||
Mount and auto mount devices on vm
|
||
Args:
|
||
vm_id (str): - the vm uuid where the volume is attached to
|
||
devices (str|list) - the device name(s). such as vdc or [vda, vdb]
|
||
guest_os (str): - the guest image the vm is booted with. such as
|
||
tis-centos-guest
|
||
check_first (bool): where to check if the device is already mounted
|
||
and auto mounted before mount and automount
|
||
vm_ssh (VMSSHClient):
|
||
"""
|
||
if isinstance(devices, str):
|
||
devices = [devices]
|
||
|
||
def _auto_mount(vm_ssh_):
|
||
_mounts = []
|
||
for disk in devices:
|
||
fs = '/dev/{}'.format(disk)
|
||
mount_on, fs_type = storage_helper.mount_partition(
|
||
ssh_client=vm_ssh_, disk=disk, partition=fs)
|
||
storage_helper.auto_mount_fs(ssh_client=vm_ssh_, fs=fs,
|
||
mount_on=mount_on, fs_type=fs_type,
|
||
check_first=check_first)
|
||
_mounts.append(mount_on)
|
||
return _mounts
|
||
|
||
if vm_ssh:
|
||
mounts = _auto_mount(vm_ssh_=vm_ssh)
|
||
else:
|
||
with ssh_to_vm_from_natbox(vm_id, vm_image_name=guest_os) as vm_ssh:
|
||
mounts = _auto_mount(vm_ssh_=vm_ssh)
|
||
|
||
return mounts
|
||
|
||
|
||
def touch_files(vm_id, file_dirs, file_name=None, content=None, guest_os=None):
|
||
"""
|
||
touch files from vm in specified dirs,and adds same content to all
|
||
touched files.
|
||
Args:
|
||
vm_id (str):
|
||
file_dirs (list): e.g., ['/', '/mnt/vdb']
|
||
file_name (str|None): defaults to 'test_file.txt' if set to None
|
||
content (str|None): defaults to "I'm a test file" if set to None
|
||
guest_os (str|None): default guest assumed to set to None
|
||
|
||
Returns (tuple): (<file_paths_for_touched_files>, <file_content>)
|
||
|
||
"""
|
||
if not file_name:
|
||
file_name = 'test_file.txt'
|
||
if not content:
|
||
content = "I'm a test file"
|
||
|
||
if isinstance(file_dirs, str):
|
||
file_dirs = [file_dirs]
|
||
file_paths = []
|
||
with ssh_to_vm_from_natbox(vm_id=vm_id, vm_image_name=guest_os) as vm_ssh:
|
||
for file_dir in file_dirs:
|
||
file_path = "{}/{}".format(file_dir, file_name)
|
||
file_path = file_path.replace('//', '/')
|
||
vm_ssh.exec_sudo_cmd(
|
||
'mkdir -p {}; touch {}'.format(file_dir, file_path),
|
||
fail_ok=False)
|
||
time.sleep(3)
|
||
vm_ssh.exec_sudo_cmd('echo "{}" >> {}'.format(content, file_path),
|
||
fail_ok=False)
|
||
output = \
|
||
vm_ssh.exec_sudo_cmd('cat {}'.format(file_path),
|
||
fail_ok=False)[1]
|
||
# TO DELETE: Debugging purpose only
|
||
vm_ssh.exec_sudo_cmd('mount | grep vd')
|
||
assert content in output, "Expected content {} is not in {}. " \
|
||
"Actual content: {}". \
|
||
format(content, file_path, output)
|
||
file_paths.append(file_path)
|
||
|
||
vm_ssh.exec_sudo_cmd('sync')
|
||
return file_paths, content
|
||
|
||
|
||
def auto_mount_vm_disks(vm_id, disks=None, guest_os=None):
|
||
"""
|
||
Auto mount non-root vm disks and return all the mount points including
|
||
root dir
|
||
Args:
|
||
vm_id (str):
|
||
disks (dict|None): disks returned by get_vm_devices_via_virsh()
|
||
guest_os (str|None): when None, default guest is assumed.
|
||
|
||
Returns (list): list of mount points. e.g., ['/', '/mnt/vdb']
|
||
|
||
"""
|
||
if not disks:
|
||
disks_to_check = get_vm_devices_via_virsh(vm_id=vm_id)
|
||
else:
|
||
disks_to_check = copy.deepcopy(disks)
|
||
|
||
root_disk = disks_to_check.pop('root_vol', {})
|
||
if not root_disk:
|
||
disks_to_check.pop('root_img')
|
||
|
||
# add root dir
|
||
mounted_on = ['/']
|
||
devs_to_mount = []
|
||
for val in disks_to_check.values():
|
||
devs_to_mount += list(val.keys())
|
||
|
||
LOG.info("Devices to mount: {}".format(devs_to_mount))
|
||
if devs_to_mount:
|
||
mounted_on += auto_mount_vm_devices(vm_id=vm_id, devices=devs_to_mount,
|
||
guest_os=guest_os)
|
||
else:
|
||
LOG.info("No non-root disks to mount for vm {}".format(vm_id))
|
||
|
||
return mounted_on
|
||
|
||
|
||
vif_map = {
|
||
'e1000': 'normal',
|
||
'rt18139': 'normal',
|
||
'virtio': 'normal',
|
||
'avp': 'normal',
|
||
'pci-sriov': 'direct',
|
||
'pci-passthrough': 'direct-physical'}
|
||
|
||
|
||
def _convert_vnics(nics, con_ssh, auth_info, cleanup):
|
||
"""
|
||
Conversion from wrs vif-model to upstream implementation
|
||
Args:
|
||
nics (list|tuple|dict):
|
||
con_ssh
|
||
auth_info
|
||
cleanup (None|str)
|
||
|
||
Returns (list):
|
||
|
||
"""
|
||
converted_nics = []
|
||
for nic in nics:
|
||
nic = dict(nic) # Do not modify original nic param
|
||
if 'vif-model' in nic:
|
||
vif_model = nic.pop('vif-model')
|
||
if vif_model:
|
||
vnic_type = vif_map[vif_model]
|
||
vif_model_ = vif_model if (
|
||
system_helper.is_avs() and vnic_type == 'normal')\
|
||
else None
|
||
if 'port-id' in nic:
|
||
port_id = nic['port-id']
|
||
current_vnic_type, current_vif_model = \
|
||
network_helper.get_port_values(
|
||
port=port_id,
|
||
fields=('binding_vnic_type', 'binding_profile'),
|
||
con_ssh=con_ssh,
|
||
auth_info=auth_info)
|
||
if current_vnic_type != vnic_type or (
|
||
vif_model_ and vif_model_ not in current_vif_model):
|
||
network_helper.set_port(port_id, vnic_type=vnic_type,
|
||
con_ssh=con_ssh,
|
||
auth_info=auth_info,
|
||
wrs_vif=vif_model_)
|
||
else:
|
||
net_id = nic.pop('net-id')
|
||
port_name = common.get_unique_name(
|
||
'port_{}'.format(vif_model))
|
||
port_id = network_helper.create_port(net_id, name=port_name,
|
||
wrs_vif=vif_model_,
|
||
vnic_type=vnic_type,
|
||
con_ssh=con_ssh,
|
||
auth_info=auth_info,
|
||
cleanup=cleanup)[1]
|
||
nic['port-id'] = port_id
|
||
converted_nics.append(nic)
|
||
|
||
return converted_nics
|
||
|
||
def boot_vm_openstack(name=None, flavor=None, block_device_mapping=None,
|
||
source=None, source_id=None, image_id=None,
|
||
image_property=None, security_groups=None, key_name=None,
|
||
inject_file=None, user_data=None, avail_zone=None,
|
||
nics=None, network=None, port=None, hint=None,
|
||
config_drive=False, min_count=None, max_count=None,
|
||
reuse_vol=False, guest_os='', wait=True,
|
||
fail_ok=False, auth_info=None, con_ssh=None, cleanup=None,
|
||
**properties):
|
||
"""
|
||
Boot a vm with given parameters using opnstack
|
||
Args:
|
||
name (str): New server name
|
||
flavor (str): Create server with this flavor (name or ID)
|
||
block_device_mapping (str): Create a block device on the server.
|
||
Block device mapping in the format
|
||
<dev-name>=<id>:<type>:<size(GB)>:<delete-on-terminate>
|
||
<dev-name>: block device name, like: vdb, xvdc (required)
|
||
<id>: Name or ID of the volume, volume snapshot or image (required)
|
||
<type>: volume, snapshot or image; default: volume (optional)
|
||
<size(GB)>: volume size if create from image or snapshot (optional)
|
||
<delete-on-terminate>: true or false; default: false (optional)
|
||
source (str): 'image', 'volume', 'snapshot'
|
||
source_id (str): id of the specified source. such as volume_id, image_id, or snapshot_id
|
||
image_id (str): id of glance image. Will not be used if source is image and source_id is
|
||
specified
|
||
image_property (str): <key=value> Image property to be matched
|
||
security_groups (str|list|tuple): Security group/groups
|
||
to assign to this server (name or ID)
|
||
key_name (str): Keypair to inject into this server (optional extension)
|
||
inject_file (str|list|tuple): File/Files to inject into image before boot
|
||
user_data (str|list): User data file to serve from the metadata server
|
||
avail_zone (str): Select an availability zone for the server
|
||
nics (list|tuple): Create NIC's on the server.
|
||
each nic:
|
||
<net-id=net-uuid,v4-fixed-ip=ip-addr,v6-fixed-ip=ip-addr, port-id=port-uuid,auto,none>,
|
||
<vif-model=model,vif-pci-address=pci-address>
|
||
Examples: [{'net-id': <net_id1>, 'vif-model': <vif1>},
|
||
{'net-id': <net_id2>, 'vif-model': <vif2>}, ...]
|
||
Notes: valid vif-models:
|
||
virtio, avp, e1000, pci-passthrough, pci-sriov, rtl8139, ne2k_pci, pcnet
|
||
network (str|list|tuple): Create a NIC on the server and connect it to network.
|
||
This is a wrapper for the ‘–nic net-id=<network>’ parameter that provides simple syntax
|
||
for the standard use case of connecting a new server to a given network.
|
||
For more advanced use cases, refer to the ‘–nic’ parameter.
|
||
port (str|list|tuple): Create a NIC on the server and connect it to port.
|
||
This is a wrapper for the ‘–nic port-id=<port>’ parameter that provides simple syntax
|
||
for the standard use case of connecting a new server to a given port.
|
||
For more advanced use cases, refer to the ‘–nic’ parameter.
|
||
hint (dict): key/value pair(s) sent to scheduler for custom use, such as
|
||
group=<server_group_id>
|
||
config_drive (<config-drive-volume>|True): Use specified volume as the config drive,
|
||
or ‘True’ to use an ephemeral drive
|
||
min_count (int): Minimum number of servers to launch (default=1)
|
||
max_count (int): Maximum number of servers to launch (default=1)
|
||
reuse_vol (bool): whether or not to reuse the existing volume (default False)
|
||
guest_os (str): Valid values: 'cgcs-guest', 'ubuntu_14', 'centos_6', 'centos_7', etc.
|
||
This will be overriden by image_id if specified.
|
||
wait (bool): Wait for build to complete (default True)
|
||
fail_ok (bool):
|
||
auth_info:
|
||
con_ssh:
|
||
cleanup (str|None): valid values: 'module', 'session', 'function', 'class',
|
||
vm (and volume) will be deleted as part of teardown
|
||
|
||
Returns (tuple): (rtn_code(int), new_vm_id_if_any(str), message(str),
|
||
new_vol_id_if_any(str))
|
||
(0, vm_id, 'VM is booted successfully') # vm is created
|
||
successfully and in Active state.
|
||
(1, vm_id, <stderr>) # boot vm cli command failed, but vm is
|
||
still booted
|
||
(2, vm_id, "VM boot started, check skipped (wait={}).") # boot vm cli
|
||
accepted, but vm building is not
|
||
100% completed. Only applicable when wait=False
|
||
(3, vm_id, "VM <uuid> did not reach ACTIVE state within <seconds>. VM
|
||
status: <status>")
|
||
# vm is not in Active state after created.
|
||
(4, '', <stderr>): create vm cli command failed, vm is not booted
|
||
|
||
"""
|
||
# Prechecks
|
||
valid_cleanups = (None, 'function', 'class', 'module', 'session')
|
||
if cleanup not in valid_cleanups:
|
||
raise ValueError(
|
||
"Invalid scope provided. Choose from: {}".format(valid_cleanups))
|
||
|
||
if user_data is None and guest_os and not re.search(
|
||
GuestImages.TIS_GUEST_PATTERN, guest_os):
|
||
# create userdata cloud init file to run right after vm
|
||
# initialization to get ip on interfaces other than eth0.
|
||
user_data = _create_cloud_init_if_conf(guest_os, nics_num=len(nics))
|
||
|
||
if user_data and user_data.startswith('~'):
|
||
user_data = user_data.replace('~', HostLinuxUser.get_home(), 1)
|
||
|
||
if inject_file and inject_file.startswith('~'):
|
||
inject_file = inject_file.replace('~', HostLinuxUser.get_home(), 1)
|
||
|
||
if guest_os == 'vxworks':
|
||
LOG.tc_step("Add HPET Timer extra spec to flavor")
|
||
extra_specs = {FlavorSpec.HPET_TIMER: 'True'}
|
||
properties.update(extra_specs)
|
||
|
||
LOG.info("Processing boot_vm_openstack args...")
|
||
# Handle mandatory arg - name
|
||
tenant = common.get_tenant_name(auth_info=auth_info)
|
||
if name is None:
|
||
name = 'vm'
|
||
name = "{}-{}".format(tenant, name)
|
||
name = common.get_unique_name(name, resource_type='vm')
|
||
|
||
# Handle mandatory arg - key_name
|
||
key_name = key_name if key_name is not None else get_default_keypair(
|
||
auth_info=auth_info, con_ssh=con_ssh)
|
||
|
||
# Handle mandatory arg - flavor
|
||
if flavor is None:
|
||
flavor = nova_helper.get_basic_flavor(auth_info=auth_info,
|
||
con_ssh=con_ssh,
|
||
guest_os=guest_os)
|
||
|
||
# Handle mandatory arg - nics
|
||
if not nics:
|
||
mgmt_net_id = network_helper.get_mgmt_net_id(auth_info=auth_info,
|
||
con_ssh=con_ssh)
|
||
if not mgmt_net_id:
|
||
raise exceptions.NeutronError("Cannot find management network")
|
||
nics = [{'net-id': mgmt_net_id}]
|
||
|
||
if 'edge' not in guest_os and 'vxworks' not in guest_os:
|
||
tenant_net_id = network_helper.get_tenant_net_id(
|
||
auth_info=auth_info, con_ssh=con_ssh)
|
||
if tenant_net_id:
|
||
nics.append({'net-id': tenant_net_id})
|
||
|
||
if isinstance(nics, dict):
|
||
nics = [nics]
|
||
nics = _convert_vnics(nics, con_ssh=con_ssh, auth_info=auth_info,
|
||
cleanup=cleanup)
|
||
|
||
# Handle mandatory arg - boot source
|
||
volume_id = snapshot_id = image = None
|
||
if source is None:
|
||
if min_count is None and max_count is None:
|
||
source = 'volume'
|
||
else:
|
||
source = 'image'
|
||
|
||
elif source.lower() == 'snapshot' and not block_device_mapping:
|
||
snapshot_id = source_id
|
||
if not snapshot_id:
|
||
snapshot_id = cinder_helper.get_vol_snapshots(
|
||
auth_info=auth_info, con_ssh=con_ssh)
|
||
if not snapshot_id:
|
||
raise ValueError(
|
||
"snapshot id is required to boot vm; however no "
|
||
"snapshot exists on the system.")
|
||
snapshot_id = snapshot_id[0]
|
||
block_device_mapping = {"vdb": "{}:snapshot".format(snapshot_id)}
|
||
vol_size, vol_id = cinder_helper.get_volume_snapshot_values(snapshot_id, ["size", "volume_id"])
|
||
img_id = cinder_helper.get_volume_show_values(vol_id, "volume_image_metadata")[0]["image_id"]
|
||
image = img_id
|
||
if vol_size:
|
||
block_device_mapping["vdb"] = "{}:{}".format(block_device_mapping["vdb"], vol_size)
|
||
elif source.lower() == 'volume':
|
||
if source_id:
|
||
volume_id = source_id
|
||
else:
|
||
vol_name = 'vol-' + name
|
||
if reuse_vol:
|
||
volume_id = cinder_helper.get_any_volume(
|
||
new_name=vol_name,
|
||
auth_info=auth_info,
|
||
con_ssh=con_ssh,
|
||
cleanup=cleanup)
|
||
else:
|
||
volume_id = cinder_helper.create_volume(
|
||
name=vol_name,
|
||
source_id=image_id,
|
||
auth_info=auth_info,
|
||
con_ssh=con_ssh,
|
||
guest_image=guest_os,
|
||
cleanup=cleanup)[1]
|
||
elif source.lower() == 'image':
|
||
# image property is not compatible with image
|
||
if not image_property:
|
||
image = source_id if source_id else image_id
|
||
if not image:
|
||
img_name = guest_os if guest_os else GuestImages.DEFAULT['guest']
|
||
image = glance_helper.get_image_id_from_name(img_name,
|
||
strict=True,
|
||
fail_ok=False)
|
||
|
||
# create cmd
|
||
non_repeat_args = {'--flavor': flavor,
|
||
'--block-device-mapping': block_device_mapping,
|
||
'--image': image,
|
||
'--image_property': image_property,
|
||
'--volume': volume_id,
|
||
'--min-count': str(min_count) if min_count is not None else None,
|
||
'--max-count': str(max_count) if max_count is not None else None,
|
||
'--key-name': key_name,
|
||
'--user-data': user_data,
|
||
'--availability_zone': avail_zone,
|
||
'--config-drive': str(config_drive) if config_drive else None,
|
||
'--wait': wait,
|
||
}
|
||
non_repeat_args = common.parse_args(non_repeat_args, repeat_arg=False,
|
||
vals_sep=',')
|
||
|
||
repeat_args = {
|
||
'--nic': nics,
|
||
'--network': network,
|
||
'--port': port,
|
||
'--file': inject_file,
|
||
'--security-groups': security_groups,
|
||
'--hint': hint,
|
||
'--property': properties
|
||
}
|
||
repeat_args = common.parse_args(repeat_args, repeat_arg=True, vals_sep=',')
|
||
|
||
pre_boot_vms = []
|
||
if not (min_count is None and max_count is None):
|
||
name_str = name + '-'
|
||
pre_boot_vms = get_vms(auth_info=auth_info, con_ssh=con_ssh,
|
||
strict=False, name=name_str)
|
||
|
||
args_ = ' '.join([non_repeat_args, repeat_args, name])
|
||
LOG.info("Booting VM {} with args: {}".format(name, args_))
|
||
exitcode, output = cli.openstack('server create', positional_args=args_,
|
||
ssh_client=con_ssh, fail_ok=True,
|
||
auth_info=auth_info,
|
||
timeout=VMTimeout.BOOT_VM)
|
||
|
||
if min_count is None and max_count is None:
|
||
table_ = table_parser.table(output)
|
||
vm_id = table_parser.get_value_two_col_table(table_, 'id')
|
||
if cleanup and vm_id:
|
||
ResourceCleanup.add('vm', vm_id, scope=cleanup, del_vm_vols=False)
|
||
# if source="snapshot":
|
||
# ResourceCleanup.add('snapshot', snapshot_id, scope=cleanup, del_vm_vols=False)
|
||
|
||
if exitcode == 1:
|
||
if vm_id:
|
||
# print out vm show for debugging purpose
|
||
cli.openstack('server show', vm_id, ssh_client=con_ssh,
|
||
auth_info=Tenant.get('admin'))
|
||
if not fail_ok:
|
||
raise exceptions.VMOperationFailed(output)
|
||
|
||
if vm_id:
|
||
return 1, vm_id, output # vm_id = '' if cli is rejected
|
||
# without vm created
|
||
return 4, '', output
|
||
|
||
LOG.info("Post action check...")
|
||
vm_status = get_vm_values(vm_id, 'status', strict=True, con_ssh=con_ssh,
|
||
auth_info=auth_info)[0]
|
||
if wait:
|
||
if vm_status != VMStatus.ACTIVE:
|
||
message = "VM did not reach {} state: {}".format(VMStatus.ACTIVE, vm_status)
|
||
if fail_ok:
|
||
LOG.warning(message)
|
||
return 2, vm_id, message
|
||
else:
|
||
raise exceptions.VMPostCheckFailed(message)
|
||
else:
|
||
LOG.info("VM {} started to create, \
|
||
check skipped because of wait argument wait={}, \
|
||
vm status is: {}".format(vm_id, wait, vm_status))
|
||
return 2, vm_id, "VM boot started, \
|
||
check skipped (wait={}), \
|
||
vm status is: {}".format(wait, vm_status)
|
||
LOG.info("VM {} is booted successfully.".format(vm_id))
|
||
return 0, vm_id, 'VM is booted successfully'
|
||
else:
|
||
name_str = name + '-'
|
||
post_boot_vms = get_vms(auth_info=auth_info, con_ssh=con_ssh,
|
||
strict=False, name=name_str)
|
||
vm_ids = list(set(post_boot_vms) - set(pre_boot_vms))
|
||
if cleanup and vm_ids:
|
||
ResourceCleanup.add('vm', vm_ids, scope=cleanup, del_vm_vols=False)
|
||
|
||
if exitcode == 1:
|
||
return 1, vm_ids, output
|
||
|
||
|
||
for instance_id in vm_ids:
|
||
vm_status = get_vm_values(instance_id, 'status', strict=True, con_ssh=con_ssh,
|
||
auth_info=auth_info)[0]
|
||
if wait:
|
||
if vm_status != VMStatus.ACTIVE:
|
||
msg = "VMs failed to reach {} state: {}".format(VMStatus.ACTIVE, vm_status)
|
||
if fail_ok:
|
||
LOG.warning(msg)
|
||
return 3, vm_ids, msg
|
||
else:
|
||
LOG.warning("VM {} started to create, \
|
||
check skipped because of wait argument wait={}, \
|
||
vm status is: {}".format(vm_id, wait, vm_status))
|
||
LOG.info("VMs booted successfully: {}".format(vm_ids))
|
||
return 0, vm_ids, "VMs are booted successfully"
|
||
|
||
|
||
def boot_vm(name=None, flavor=None, source=None, source_id=None, image_id=None,
|
||
min_count=None, nics=None, hint=None,
|
||
max_count=None, key_name=None, swap=None, ephemeral=None,
|
||
user_data=None, block_device=None,
|
||
block_device_mapping=None, security_groups=None, vm_host=None,
|
||
avail_zone=None, file=None,
|
||
config_drive=False, meta=None, tags=None,
|
||
fail_ok=False, auth_info=None, con_ssh=None, reuse_vol=False,
|
||
guest_os='', poll=True, cleanup=None):
|
||
"""
|
||
Boot a vm with given parameters
|
||
Args:
|
||
name (str):
|
||
flavor (str):
|
||
source (str): 'image', 'volume', 'snapshot', or 'block_device'
|
||
source_id (str): id of the specified source. such as volume_id,
|
||
image_id, or snapshot_id
|
||
image_id (str): id of glance image. Will not be used if source is
|
||
image and source_id is specified
|
||
min_count (int):
|
||
max_count (int):
|
||
key_name (str):
|
||
swap (int|None):
|
||
ephemeral (int):
|
||
user_data (str|list):
|
||
vm_host (str): which host to place the vm
|
||
avail_zone (str): availability zone for vm host, Possible values:
|
||
'nova', 'stxauto', etc
|
||
block_device (dict|list|tuple): dist or list of dict, each dictionary
|
||
is a block device.
|
||
e.g, {'source': 'volume', 'volume_id': xxxx, ...}
|
||
block_device_mapping (str): Block device mapping in the format
|
||
'<dev-name>=<id>:<type>:<size(GB)>:<delete-on-
|
||
terminate>'.
|
||
auth_info (dict):
|
||
con_ssh (SSHClient):
|
||
security_groups (str|list|tuple): add nova boot option
|
||
--security-groups $(sec_group_name)
|
||
nics (list): nics to be created for the vm
|
||
each nic: <net-id=net-uuid,net-name=network-name,
|
||
v4-fixed-ip=ip-addr,v6-fixed-ip=ip-addr,
|
||
port-id=port-uuid,vif-model=model>,
|
||
vif-pci-address=pci-address>
|
||
Examples: [{'net-id': <net_id1>, 'vif-model': <vif1>}, {'net-id':
|
||
<net_id2>, 'vif-model': <vif2>}, ...]
|
||
Notes: valid vif-models:
|
||
virtio, avp, e1000, pci-passthrough, pci-sriov, rtl8139,
|
||
ne2k_pci, pcnet
|
||
|
||
hint (dict): key/value pair(s) sent to scheduler for custom use. such
|
||
as group=<server_group_id>
|
||
file (str): <dst-path=src-path> To store files from local <src-path>
|
||
to <dst-path> on the new server.
|
||
config_drive (bool): To enable config drive.
|
||
meta (dict): key/value pairs for vm meta data. e.g.,
|
||
{'sw:wrs:recovery_priority': 1, ...}
|
||
tags (None|str|tuple|list)
|
||
fail_ok (bool):
|
||
reuse_vol (bool): whether or not to reuse the existing volume
|
||
guest_os (str): Valid values: 'cgcs-guest', 'ubuntu_14', 'centos_6',
|
||
'centos_7', etc.
|
||
This will be overriden by image_id if specified.
|
||
poll (bool):
|
||
cleanup (str|None): valid values: 'module', 'session', 'function',
|
||
'class', vm (and volume) will be deleted as
|
||
part of teardown
|
||
|
||
Returns (tuple): (rtn_code(int), new_vm_id_if_any(str), message(str),
|
||
new_vol_id_if_any(str))
|
||
(0, vm_id, 'VM is booted successfully') # vm is created
|
||
successfully and in Active state.
|
||
(1, vm_id, <stderr>) # boot vm cli command failed, but vm is
|
||
still booted
|
||
(2, vm_id, "VM building is not 100% complete.") # boot vm cli
|
||
accepted, but vm building is not
|
||
100% completed. Only applicable when poll=True
|
||
(3, vm_id, "VM <uuid> did not reach ACTIVE state within <seconds>. VM
|
||
status: <status>")
|
||
# vm is not in Active state after created.
|
||
(4, '', <stderr>): create vm cli command failed, vm is not booted
|
||
|
||
"""
|
||
valid_cleanups = (None, 'function', 'class', 'module', 'session')
|
||
if cleanup not in valid_cleanups:
|
||
raise ValueError(
|
||
"Invalid scope provided. Choose from: {}".format(valid_cleanups))
|
||
|
||
LOG.info("Processing boot_vm args...")
|
||
# Handle mandatory arg - name
|
||
tenant = common.get_tenant_name(auth_info=auth_info)
|
||
if name is None:
|
||
name = 'vm'
|
||
name = "{}-{}".format(tenant, name)
|
||
name = common.get_unique_name(name, resource_type='vm')
|
||
|
||
# Handle mandatory arg - key_name
|
||
key_name = key_name if key_name is not None else get_default_keypair(
|
||
auth_info=auth_info, con_ssh=con_ssh)
|
||
|
||
# Handle mandatory arg - flavor
|
||
if flavor is None:
|
||
flavor = nova_helper.get_basic_flavor(auth_info=auth_info,
|
||
con_ssh=con_ssh,
|
||
guest_os=guest_os)
|
||
|
||
if guest_os == 'vxworks':
|
||
LOG.tc_step("Add HPET Timer extra spec to flavor")
|
||
extra_specs = {FlavorSpec.HPET_TIMER: 'True'}
|
||
nova_helper.set_flavor(flavor=flavor, **extra_specs)
|
||
|
||
# Handle mandatory arg - nics
|
||
if not nics:
|
||
mgmt_net_id = network_helper.get_mgmt_net_id(auth_info=auth_info,
|
||
con_ssh=con_ssh)
|
||
if not mgmt_net_id:
|
||
raise exceptions.NeutronError("Cannot find management network")
|
||
nics = [{'net-id': mgmt_net_id}]
|
||
|
||
if 'edge' not in guest_os and 'vxworks' not in guest_os:
|
||
tenant_net_id = network_helper.get_tenant_net_id(
|
||
auth_info=auth_info, con_ssh=con_ssh)
|
||
if tenant_net_id:
|
||
nics.append({'net-id': tenant_net_id})
|
||
|
||
if isinstance(nics, dict):
|
||
nics = [nics]
|
||
nics = _convert_vnics(nics, con_ssh=con_ssh, auth_info=auth_info,
|
||
cleanup=cleanup)
|
||
|
||
# Handle mandatory arg - boot source
|
||
volume_id = snapshot_id = image = None
|
||
if source != 'block_device':
|
||
if source is None:
|
||
if min_count is None and max_count is None:
|
||
source = 'volume'
|
||
else:
|
||
source = 'image'
|
||
|
||
if source.lower() == 'volume':
|
||
if source_id:
|
||
volume_id = source_id
|
||
else:
|
||
vol_name = 'vol-' + name
|
||
if reuse_vol:
|
||
volume_id = cinder_helper.get_any_volume(
|
||
new_name=vol_name,
|
||
auth_info=auth_info,
|
||
con_ssh=con_ssh,
|
||
cleanup=cleanup)
|
||
else:
|
||
volume_id = cinder_helper.create_volume(
|
||
name=vol_name,
|
||
source_id=image_id,
|
||
auth_info=auth_info,
|
||
con_ssh=con_ssh,
|
||
guest_image=guest_os,
|
||
cleanup=cleanup)[1]
|
||
|
||
elif source.lower() == 'image':
|
||
image = source_id if source_id else image_id
|
||
if not image:
|
||
img_name = guest_os if guest_os else GuestImages.DEFAULT[
|
||
'guest']
|
||
image = glance_helper.get_image_id_from_name(img_name,
|
||
strict=True,
|
||
fail_ok=False)
|
||
|
||
elif source.lower() == 'snapshot':
|
||
snapshot_id = source_id
|
||
if not snapshot_id:
|
||
snapshot_id = cinder_helper.get_vol_snapshots(
|
||
auth_info=auth_info, con_ssh=con_ssh)
|
||
if not snapshot_id:
|
||
raise ValueError(
|
||
"snapshot id is required to boot vm; however no "
|
||
"snapshot exists on the system.")
|
||
snapshot_id = snapshot_id[0]
|
||
|
||
if vm_host and not avail_zone:
|
||
avail_zone = 'nova'
|
||
if avail_zone and vm_host:
|
||
avail_zone = '{}:{}'.format(avail_zone, vm_host)
|
||
|
||
if user_data is None and guest_os and not re.search(
|
||
GuestImages.TIS_GUEST_PATTERN, guest_os):
|
||
# create userdata cloud init file to run right after vm
|
||
# initialization to get ip on interfaces other than eth0.
|
||
user_data = _create_cloud_init_if_conf(guest_os, nics_num=len(nics))
|
||
|
||
if user_data and user_data.startswith('~'):
|
||
user_data = user_data.replace('~', HostLinuxUser.get_home(), 1)
|
||
|
||
if file and file.startswith('~'):
|
||
file = file.replace('~', HostLinuxUser.get_home(), 1)
|
||
|
||
# create cmd
|
||
non_repeat_args = {'--flavor': flavor,
|
||
'--image': image,
|
||
'--boot-volume': volume_id,
|
||
'--snapshot': snapshot_id,
|
||
'--min-count': str(
|
||
min_count) if min_count is not None else None,
|
||
'--max-count': str(
|
||
max_count) if max_count is not None else None,
|
||
'--key-name': key_name,
|
||
'--swap': swap,
|
||
'--user-data': user_data,
|
||
'--ephemeral': ephemeral,
|
||
'--availability-zone': avail_zone,
|
||
'--file': file,
|
||
'--config-drive': str(
|
||
config_drive) if config_drive else None,
|
||
'--block-device-mapping': block_device_mapping,
|
||
'--security-groups': security_groups,
|
||
'--tags': tags,
|
||
'--poll': poll,
|
||
}
|
||
non_repeat_args = common.parse_args(non_repeat_args, repeat_arg=False,
|
||
vals_sep=',')
|
||
|
||
repeat_args = {
|
||
'--meta': meta,
|
||
'--nic': nics,
|
||
'--hint': hint,
|
||
'--block-device': block_device,
|
||
}
|
||
repeat_args = common.parse_args(repeat_args, repeat_arg=True, vals_sep=',')
|
||
|
||
pre_boot_vms = []
|
||
if not (min_count is None and max_count is None):
|
||
name_str = name + '-'
|
||
pre_boot_vms = get_vms(auth_info=auth_info, con_ssh=con_ssh,
|
||
strict=False, name=name_str)
|
||
|
||
args_ = ' '.join([non_repeat_args, repeat_args, name])
|
||
LOG.info("Booting VM {} with args: {}".format(name, args_))
|
||
exitcode, output = cli.nova('boot', positional_args=args_,
|
||
ssh_client=con_ssh, fail_ok=True,
|
||
auth_info=auth_info,
|
||
timeout=VMTimeout.BOOT_VM)
|
||
|
||
tmout = VMTimeout.STATUS_CHANGE
|
||
if min_count is None and max_count is None:
|
||
table_ = table_parser.table(output)
|
||
vm_id = table_parser.get_value_two_col_table(table_, 'id')
|
||
if cleanup and vm_id:
|
||
ResourceCleanup.add('vm', vm_id, scope=cleanup, del_vm_vols=False)
|
||
|
||
if exitcode == 1:
|
||
if vm_id:
|
||
# print out vm show for debugging purpose
|
||
cli.openstack('server show', vm_id, ssh_client=con_ssh,
|
||
auth_info=Tenant.get('admin'))
|
||
if not fail_ok:
|
||
raise exceptions.VMOperationFailed(output)
|
||
|
||
if vm_id:
|
||
return 1, vm_id, output # vm_id = '' if cli is rejected
|
||
# without vm created
|
||
return 4, '', output
|
||
|
||
LOG.info("Post action check...")
|
||
if poll and "100% complete" not in output:
|
||
message = "VM building is not 100% complete."
|
||
if fail_ok:
|
||
LOG.warning(message)
|
||
return 2, vm_id, "VM building is not 100% complete."
|
||
else:
|
||
raise exceptions.VMOperationFailed(message)
|
||
|
||
if not wait_for_vm_status(vm_id=vm_id, status=VMStatus.ACTIVE,
|
||
timeout=tmout, con_ssh=con_ssh,
|
||
auth_info=auth_info, fail_ok=True):
|
||
vm_status = \
|
||
get_vm_values(vm_id, 'status', strict=True, con_ssh=con_ssh,
|
||
auth_info=auth_info)[0]
|
||
message = "VM {} did not reach ACTIVE state within {}. VM " \
|
||
"status: {}".format(vm_id, tmout, vm_status)
|
||
if fail_ok:
|
||
LOG.warning(message)
|
||
return 3, vm_id, message
|
||
else:
|
||
raise exceptions.VMPostCheckFailed(message)
|
||
|
||
LOG.info("VM {} is booted successfully.".format(vm_id))
|
||
|
||
return 0, vm_id, 'VM is booted successfully'
|
||
|
||
else:
|
||
name_str = name + '-'
|
||
post_boot_vms = get_vms(auth_info=auth_info, con_ssh=con_ssh,
|
||
strict=False, name=name_str)
|
||
vm_ids = list(set(post_boot_vms) - set(pre_boot_vms))
|
||
if cleanup and vm_ids:
|
||
ResourceCleanup.add('vm', vm_ids, scope=cleanup, del_vm_vols=False)
|
||
|
||
if exitcode == 1:
|
||
return 1, vm_ids, output
|
||
|
||
result, vms_in_state, vms_failed_to_reach_state = wait_for_vms_values(
|
||
vm_ids, fail_ok=True, timeout=tmout,
|
||
con_ssh=con_ssh,
|
||
auth_info=Tenant.get('admin'))
|
||
if not result:
|
||
msg = "VMs failed to reach ACTIVE state: {}".format(
|
||
vms_failed_to_reach_state)
|
||
if fail_ok:
|
||
LOG.warning(msg=msg)
|
||
return 3, vm_ids, msg
|
||
|
||
LOG.info("VMs booted successfully: {}".format(vm_ids))
|
||
return 0, vm_ids, "VMs are booted successfully"
|
||
|
||
|
||
def wait_for_vm_pingable_from_natbox(vm_id, timeout=200, fail_ok=False,
|
||
con_ssh=None, use_fip=False):
|
||
"""
|
||
Wait for ping vm from natbox succeeds.
|
||
|
||
Args:
|
||
vm_id (str): id of the vm to ping
|
||
timeout (int): max retry time for pinging vm
|
||
fail_ok (bool): whether to raise exception if vm cannot be ping'd
|
||
successfully from natbox within timeout
|
||
con_ssh (SSHClient): TiS server ssh handle
|
||
use_fip (bool): whether or not to ping floating ip only if any
|
||
|
||
Returns (bool): True if ping vm succeeded, False otherwise.
|
||
|
||
"""
|
||
ping_end_time = time.time() + timeout
|
||
while time.time() < ping_end_time:
|
||
if ping_vms_from_natbox(vm_ids=vm_id, fail_ok=True, con_ssh=con_ssh,
|
||
num_pings=3, use_fip=use_fip)[0]:
|
||
# give it sometime to settle after vm booted and became pingable
|
||
time.sleep(5)
|
||
return True
|
||
else:
|
||
msg = "Ping from NatBox to vm {} failed for {} seconds.".format(vm_id,
|
||
timeout)
|
||
if fail_ok:
|
||
LOG.warning(msg)
|
||
return False
|
||
else:
|
||
time_stamp = common.get_date_in_format(ssh_client=con_ssh,
|
||
date_format='%Y%m%d_%H-%M')
|
||
f_path = '{}/{}-{}'.format(ProjVar.get_var('PING_FAILURE_DIR'),
|
||
time_stamp, ProjVar.get_var('TEST_NAME'))
|
||
common.write_to_file(f_path,
|
||
"=================={}===============\n".format(
|
||
msg))
|
||
ProjVar.set_var(PING_FAILURE=True)
|
||
get_console_logs(vm_ids=vm_id, sep_file=f_path)
|
||
network_helper.collect_networking_info(vms=vm_id, sep_file=f_path,
|
||
time_stamp=time_stamp)
|
||
raise exceptions.VMNetworkError(msg)
|
||
|
||
|
||
def __merge_dict(base_dict, merge_dict):
|
||
# identical to {**base_dict, **merge_dict} in python3.6+
|
||
d = dict(base_dict) # id() will be different, making a copy
|
||
for k in merge_dict:
|
||
d[k] = merge_dict[k]
|
||
return d
|
||
|
||
|
||
def get_default_keypair(auth_info=None, con_ssh=None):
|
||
"""
|
||
Get keypair for specific tenant.
|
||
|
||
Args:
|
||
auth_info (dict): If None, default tenant will be used.
|
||
con_ssh (SSHClient):
|
||
|
||
Returns (str): key name
|
||
|
||
"""
|
||
if auth_info is None:
|
||
auth_info = Tenant.get_primary()
|
||
|
||
keypair_name = auth_info['nova_keypair']
|
||
existing_keypairs = nova_helper.get_keypairs(name=keypair_name,
|
||
con_ssh=con_ssh,
|
||
auth_info=auth_info)
|
||
if existing_keypairs:
|
||
return existing_keypairs[0]
|
||
|
||
# Assume that public key file already exists since it should have been
|
||
# set up in session config.
|
||
# In the case of public key file does not exist, there should be existing
|
||
# nova keypair, so it should not
|
||
# reach this step. Config done via setups.setup_keypair()
|
||
keyfile_stx_final = ProjVar.get_var('STX_KEYFILE_SYS_HOME')
|
||
public_key_stx = '{}.pub'.format(keyfile_stx_final)
|
||
LOG.info("Create nova keypair {} using public key {}".format(
|
||
keypair_name, public_key_stx))
|
||
nova_helper.create_keypair(keypair_name, public_key=public_key_stx,
|
||
auth_info=auth_info, con_ssh=con_ssh)
|
||
|
||
return keypair_name
|
||
|
||
|
||
def live_migrate_vm(vm_id, destination_host='', con_ssh=None,
|
||
block_migrate=None, force=None, fail_ok=False,
|
||
auth_info=Tenant.get('admin')):
|
||
"""
|
||
|
||
Args:
|
||
vm_id (str):
|
||
destination_host (str): such as compute-0, compute-1
|
||
con_ssh (SSHClient):
|
||
block_migrate (bool): whether to add '--block-migrate' to command
|
||
force (None|bool): force live migrate
|
||
fail_ok (bool): if fail_ok, return a numerical number to indicate the
|
||
execution status
|
||
One exception is if the live-migration command exit_code > 1,
|
||
which indicating the command itself may
|
||
be incorrect. In this case CLICommandFailed exception will be
|
||
thrown regardless of the fail_ok flag.
|
||
auth_info (dict):
|
||
|
||
Returns (tuple): (return_code (int), error_msg_if_migration_rejected (str))
|
||
(0, 'Live migration is successful.'):
|
||
live migration succeeded and post migration checking passed
|
||
(1, <cli stderr>): # This scenario is changed to host did not change
|
||
as excepted
|
||
live migration request rejected as expected. e.g., no available
|
||
destination host,
|
||
or live migrate a vm with block migration
|
||
(2, <cli stderr>): live migration request rejected due to unknown
|
||
reason.
|
||
(3, 'Post action check failed: VM is in ERROR state.'):
|
||
live migration command executed successfully, but VM is in Error
|
||
state after migration
|
||
(4, 'Post action check failed: VM is not in original state.'):
|
||
live migration command executed successfully, but VM is not in
|
||
before-migration-state
|
||
(5, 'Post action check failed: VM host did not change!'): (this
|
||
scenario is removed from Newton)
|
||
live migration command executed successfully, but VM is still on
|
||
the same host after migration
|
||
(6, <cli_stderr>) This happens when vote_note_to_migrate is set for
|
||
vm, or pci device is used in vm, etc
|
||
|
||
For the first two scenarios, results will be returned regardless of the
|
||
fail_ok flag.
|
||
For scenarios other than the first two, returns are only applicable if
|
||
fail_ok=True
|
||
|
||
Examples:
|
||
1) If a test case is meant to test live migration with a specific
|
||
flavor which would block the migration, the
|
||
following call can be made:
|
||
|
||
return_code, msg = live_migrate_vm(vm_id, fail_ok=True)
|
||
expected_err_str = "your error string"
|
||
assert return_code in [1, 2]
|
||
assert expected_err_str in msg
|
||
|
||
2) For a test that needs to live migrate
|
||
|
||
"""
|
||
optional_arg = ''
|
||
|
||
if block_migrate:
|
||
optional_arg += '--block-migrate'
|
||
|
||
if force:
|
||
optional_arg += '--force'
|
||
|
||
before_host = get_vm_host(vm_id, con_ssh=con_ssh)
|
||
before_status = get_vm_values(vm_id, 'status', strict=True, con_ssh=con_ssh,
|
||
auth_info=Tenant.get('admin'))[0]
|
||
if not before_status == VMStatus.ACTIVE:
|
||
LOG.warning("Non-active VM status before live migrate: {}".format(
|
||
before_status))
|
||
|
||
extra_str = ''
|
||
if not destination_host == '':
|
||
extra_str = ' to ' + destination_host
|
||
positional_args = ' '.join(
|
||
[optional_arg.strip(), str(vm_id), destination_host]).strip()
|
||
LOG.info(
|
||
"Live migrating VM {} from {}{} started.".format(vm_id, before_host,
|
||
extra_str))
|
||
LOG.info("nova live-migration {}".format(positional_args))
|
||
# auto host/block migration selection unavailable in openstack client
|
||
exit_code, output = cli.nova('live-migration',
|
||
positional_args=positional_args,
|
||
ssh_client=con_ssh, fail_ok=fail_ok,
|
||
auth_info=auth_info)
|
||
|
||
if exit_code == 1:
|
||
return 6, output
|
||
|
||
LOG.info("Waiting for VM status change to {} with best effort".format(
|
||
VMStatus.MIGRATING))
|
||
in_mig_state = wait_for_vm_status(vm_id, status=VMStatus.MIGRATING,
|
||
timeout=60, fail_ok=True)
|
||
if not in_mig_state:
|
||
LOG.warning(
|
||
"VM did not reach {} state after triggering live-migration".format(
|
||
VMStatus.MIGRATING))
|
||
|
||
LOG.info("Waiting for VM status change to original state {}".format(
|
||
before_status))
|
||
end_time = time.time() + VMTimeout.LIVE_MIGRATE_COMPLETE
|
||
while time.time() < end_time:
|
||
time.sleep(2)
|
||
status = get_vm_values(vm_id, 'status', strict=True, con_ssh=con_ssh,
|
||
auth_info=Tenant.get('admin'))[0]
|
||
if status == before_status:
|
||
LOG.info("Live migrate vm {} completed".format(vm_id))
|
||
break
|
||
elif status == VMStatus.ERROR:
|
||
if fail_ok:
|
||
return 3, "Post action check failed: VM is in ERROR state."
|
||
nova_helper.get_migration_list_table(con_ssh=con_ssh,
|
||
auth_info=auth_info)
|
||
raise exceptions.VMPostCheckFailed(
|
||
"VM {} is in {} state after live migration. Original state "
|
||
"before live migration is: {}".format(vm_id, VMStatus.ERROR,
|
||
before_status))
|
||
else:
|
||
if fail_ok:
|
||
return 4, "Post action check failed: VM is not in original state."
|
||
else:
|
||
nova_helper.get_migration_list_table(con_ssh=con_ssh,
|
||
auth_info=auth_info)
|
||
raise exceptions.TimeoutException(
|
||
"VM {} did not reach original state within {} seconds after "
|
||
"live migration".format(vm_id, VMTimeout.LIVE_MIGRATE_COMPLETE))
|
||
|
||
after_host = before_host
|
||
for i in range(3):
|
||
after_host = get_vm_host(vm_id, con_ssh=con_ssh)
|
||
if after_host != before_host:
|
||
break
|
||
time.sleep(3)
|
||
|
||
if before_host == after_host:
|
||
LOG.warning(
|
||
"Live migration of vm {} failed. Checking if this is expected "
|
||
"failure...".format(
|
||
vm_id))
|
||
if _is_live_migration_allowed(vm_id, vm_host=before_host,
|
||
block_migrate=block_migrate) and \
|
||
(destination_host or get_dest_host_for_live_migrate(vm_id)):
|
||
if fail_ok:
|
||
return 1, "Unknown live migration failure"
|
||
else:
|
||
nova_helper.get_migration_list_table(con_ssh=con_ssh,
|
||
auth_info=auth_info)
|
||
raise exceptions.VMPostCheckFailed(
|
||
"Unexpected failure of live migration!")
|
||
else:
|
||
LOG.debug(
|
||
"System does not allow live migrating vm {} as "
|
||
"expected.".format(
|
||
vm_id))
|
||
return 2, "Live migration failed as expected"
|
||
|
||
LOG.info(
|
||
"VM {} successfully migrated from {} to {}".format(vm_id, before_host,
|
||
after_host))
|
||
return 0, "Live migration is successful."
|
||
|
||
|
||
def _is_live_migration_allowed(vm_id, vm_host, con_ssh=None,
|
||
block_migrate=None):
|
||
vm_info = VMInfo.get_vm_info(vm_id, con_ssh=con_ssh)
|
||
storage_backing = vm_info.get_storage_type()
|
||
if not storage_backing:
|
||
storage_backing = host_helper.get_host_instance_backing(host=vm_host,
|
||
con_ssh=con_ssh)
|
||
|
||
vm_boot_from = vm_info.boot_info['type']
|
||
|
||
if storage_backing == 'local_image':
|
||
if block_migrate and vm_boot_from == 'volume' and not \
|
||
vm_info.has_local_disks():
|
||
LOG.warning(
|
||
"Live block migration is not supported for boot-from-volume "
|
||
"vm with local_image storage")
|
||
return False
|
||
return True
|
||
|
||
elif storage_backing == 'local_lvm':
|
||
if (not block_migrate) and vm_boot_from == 'volume' and not \
|
||
vm_info.has_local_disks():
|
||
return True
|
||
else:
|
||
LOG.warning(
|
||
"Live (block) migration is not supported for local_lvm vm "
|
||
"with localdisk")
|
||
return False
|
||
|
||
else:
|
||
# remote backend
|
||
if block_migrate:
|
||
LOG.warning(
|
||
"Live block migration is not supported for vm with remote "
|
||
"storage")
|
||
return False
|
||
else:
|
||
return True
|
||
|
||
|
||
def get_dest_host_for_live_migrate(vm_id, con_ssh=None):
|
||
"""
|
||
Check whether a destination host exists with following criteria:
|
||
Criteria:
|
||
1) host has same storage backing as the vm
|
||
2) host is unlocked
|
||
3) different than current host
|
||
Args:
|
||
vm_id (str):
|
||
con_ssh (SSHClient):
|
||
|
||
Returns (str): hostname for the first host found. Or '' if no proper host
|
||
found
|
||
"""
|
||
vm_info = VMInfo.get_vm_info(vm_id, con_ssh=con_ssh)
|
||
vm_storage_backing = vm_info.get_storage_type()
|
||
current_host = vm_info.get_host_name()
|
||
if not vm_storage_backing:
|
||
vm_storage_backing = host_helper.get_host_instance_backing(
|
||
host=current_host, con_ssh=con_ssh)
|
||
candidate_hosts = host_helper.get_hosts_in_storage_backing(
|
||
storage_backing=vm_storage_backing, con_ssh=con_ssh)
|
||
|
||
hosts_table_ = table_parser.table(cli.system('host-list')[1])
|
||
for host in candidate_hosts:
|
||
if not host == current_host:
|
||
host_state = table_parser.get_values(hosts_table_, 'administrative',
|
||
hostname=host)[0]
|
||
if host_state == 'unlocked':
|
||
LOG.debug(
|
||
"At least one host - {} is available for live migrating "
|
||
"vm {}".format(
|
||
host, vm_id))
|
||
return host
|
||
|
||
LOG.warning("No valid host found for live migrating vm {}".format(vm_id))
|
||
return ''
|
||
|
||
|
||
def cold_migrate_vm(vm_id, revert=False, con_ssh=None, fail_ok=False,
|
||
auth_info=Tenant.get('admin')):
|
||
"""
|
||
Cold migrate a vm and confirm/revert
|
||
Args:
|
||
vm_id (str): vm to cold migrate
|
||
revert (bool): False to confirm resize, True to revert
|
||
con_ssh (SSHClient):
|
||
fail_ok (bool): True if fail ok. Default to False, ie., throws
|
||
exception upon cold migration fail.
|
||
auth_info (dict):
|
||
|
||
Returns (tuple): (rtn_code, message)
|
||
(0, success_msg) # Cold migration and confirm/revert succeeded. VM is
|
||
back to original state or Active state.
|
||
(1, <stderr>) # cold migration cli rejected
|
||
# (2, <stderr>) # Cold migration cli command rejected. <stderr> is
|
||
the err message returned by cli cmd.
|
||
(3, <stdout>) # Cold migration cli accepted, but not finished.
|
||
<stdout> is the output of cli cmd.
|
||
(4, timeout_message] # Cold migration command ran successfully,
|
||
but timed out waiting for VM to reach
|
||
'Verify Resize' state or Error state.
|
||
(5, err_msg) # Cold migration command ran successfully, but VM is in
|
||
Error state.
|
||
(6, err_msg) # Cold migration command ran successfully, and resize
|
||
confirm/revert performed. But VM is not in
|
||
Active state after confirm/revert.
|
||
(7, err_msg) # Cold migration and resize confirm/revert ran
|
||
successfully and vm in active state. But host for vm
|
||
is not as expected. i.e., still the same host after confirm
|
||
resize, or different host after revert resize.
|
||
(8, <stderr>) # Confirm/Revert resize cli rejected
|
||
|
||
"""
|
||
before_host = get_vm_host(vm_id, con_ssh=con_ssh)
|
||
before_status = \
|
||
get_vm_values(vm_id, 'status', strict=True, con_ssh=con_ssh)[0]
|
||
if not before_status == VMStatus.ACTIVE:
|
||
LOG.warning("Non-active VM status before cold migrate: {}".format(
|
||
before_status))
|
||
|
||
LOG.info("Cold migrating VM {} from {}...".format(vm_id, before_host))
|
||
exitcode, output = cli.nova('migrate --poll', vm_id, ssh_client=con_ssh,
|
||
fail_ok=fail_ok, auth_info=auth_info,
|
||
timeout=VMTimeout.COLD_MIGRATE_CONFIRM)
|
||
|
||
if exitcode == 1:
|
||
return 1, output
|
||
|
||
LOG.info(
|
||
"Waiting for VM status change to {}".format(VMStatus.VERIFY_RESIZE))
|
||
vm_status = wait_for_vm_status(vm_id=vm_id,
|
||
status=[VMStatus.VERIFY_RESIZE,
|
||
VMStatus.ERROR],
|
||
timeout=300,
|
||
fail_ok=fail_ok, con_ssh=con_ssh)
|
||
|
||
if vm_status is None:
|
||
return 4, 'Timed out waiting for Error or Verify_Resize status for ' \
|
||
'VM {}'.format(vm_id)
|
||
|
||
verify_resize_str = 'Revert' if revert else 'Confirm'
|
||
if vm_status == VMStatus.VERIFY_RESIZE:
|
||
LOG.info("{}ing resize..".format(verify_resize_str))
|
||
res, out = _confirm_or_revert_resize(vm=vm_id, revert=revert,
|
||
fail_ok=fail_ok, con_ssh=con_ssh)
|
||
if res > 0:
|
||
return 8, out
|
||
|
||
elif vm_status == VMStatus.ERROR:
|
||
err_msg = "VM {} in Error state after cold migrate. {} resize is not " \
|
||
"reached.".format(vm_id, verify_resize_str)
|
||
if fail_ok:
|
||
return 5, err_msg
|
||
nova_helper.get_migration_list_table(con_ssh=con_ssh,
|
||
auth_info=auth_info)
|
||
raise exceptions.VMPostCheckFailed(err_msg)
|
||
|
||
post_confirm_state = wait_for_vm_status(
|
||
vm_id, status=VMStatus.ACTIVE,
|
||
timeout=VMTimeout.COLD_MIGRATE_CONFIRM, fail_ok=fail_ok,
|
||
con_ssh=con_ssh)
|
||
|
||
if post_confirm_state is None:
|
||
err_msg = "VM {} is not in Active state after {} Resize".format(
|
||
vm_id, verify_resize_str)
|
||
return 6, err_msg
|
||
|
||
# Process results
|
||
after_host = get_vm_host(vm_id, con_ssh=con_ssh)
|
||
host_changed = before_host != after_host
|
||
host_change_str = "changed" if host_changed else "did not change"
|
||
operation_ok = not host_changed if revert else host_changed
|
||
|
||
if not operation_ok:
|
||
err_msg = (
|
||
"VM {} host {} after {} Resize. Before host: {}. After host: {}".
|
||
format(vm_id, host_change_str, verify_resize_str, before_host,
|
||
after_host))
|
||
if fail_ok:
|
||
return 7, err_msg
|
||
nova_helper.get_migration_list_table(con_ssh=con_ssh,
|
||
auth_info=auth_info)
|
||
raise exceptions.VMPostCheckFailed(err_msg)
|
||
|
||
success_msg = "VM {} successfully cold migrated and {}ed Resize.".format(
|
||
vm_id, verify_resize_str)
|
||
LOG.info(success_msg)
|
||
return 0, success_msg
|
||
|
||
|
||
def resize_vm(vm_id, flavor_id, revert=False, con_ssh=None, fail_ok=False,
|
||
auth_info=Tenant.get('admin')):
|
||
"""
|
||
Resize vm to given flavor
|
||
|
||
Args:
|
||
vm_id (str):
|
||
flavor_id (str): flavor to resize to
|
||
revert (bool): True to revert resize, else confirm resize
|
||
con_ssh (SSHClient):
|
||
fail_ok (bool):
|
||
auth_info (dict):
|
||
|
||
Returns (tuple): (rtn_code, msg)
|
||
(0, "VM <vm_id> successfully resized and confirmed/reverted.")
|
||
(1, <std_err>) # resize cli rejected
|
||
(2, "Timed out waiting for Error or Verify_Resize status for VM
|
||
<vm_id>")
|
||
(3, "VM <vm_id> in Error state after resizing. VERIFY_RESIZE is not
|
||
reached.")
|
||
(4, "VM <vm_id> is not in Active state after confirm/revert Resize")
|
||
(5, "Flavor is changed after revert resizing.")
|
||
(6, "VM flavor is not changed to expected after resizing.")
|
||
"""
|
||
before_flavor = get_vm_flavor(vm_id, con_ssh=con_ssh)
|
||
before_status = \
|
||
get_vm_values(vm_id, 'status', strict=True, con_ssh=con_ssh)[0]
|
||
if not before_status == VMStatus.ACTIVE:
|
||
LOG.warning("Non-active VM status before cold migrate: {}".format(
|
||
before_status))
|
||
|
||
LOG.info("Resizing VM {} to flavor {}...".format(vm_id, flavor_id))
|
||
args = '--wait --flavor {} {}'.format(flavor_id, vm_id)
|
||
exitcode, output = cli.openstack('server resize', args, ssh_client=con_ssh,
|
||
fail_ok=fail_ok, auth_info=auth_info,
|
||
timeout=VMTimeout.COLD_MIGRATE_CONFIRM)
|
||
if exitcode > 0:
|
||
return 1, output
|
||
|
||
LOG.info(
|
||
"Waiting for VM status change to {}".format(VMStatus.VERIFY_RESIZE))
|
||
vm_status = wait_for_vm_status(vm_id=vm_id,
|
||
status=[VMStatus.VERIFY_RESIZE,
|
||
VMStatus.ERROR],
|
||
fail_ok=fail_ok,
|
||
timeout=300, con_ssh=con_ssh)
|
||
|
||
if vm_status is None:
|
||
err_msg = 'Timed out waiting for Error or Verify_Resize status for ' \
|
||
'VM {}'.format(vm_id)
|
||
LOG.error(err_msg)
|
||
return 2, err_msg
|
||
|
||
verify_resize_str = 'Revert' if revert else 'Confirm'
|
||
if vm_status == VMStatus.VERIFY_RESIZE:
|
||
LOG.info("{}ing resize..".format(verify_resize_str))
|
||
_confirm_or_revert_resize(vm=vm_id, revert=revert, con_ssh=con_ssh)
|
||
|
||
elif vm_status == VMStatus.ERROR:
|
||
err_msg = "VM {} in Error state after resizing. {} is not " \
|
||
"reached.".format(vm_id, VMStatus.VERIFY_RESIZE)
|
||
if fail_ok:
|
||
LOG.error(err_msg)
|
||
return 3, err_msg
|
||
raise exceptions.VMPostCheckFailed(err_msg)
|
||
|
||
post_confirm_state = wait_for_vm_status(
|
||
vm_id, status=VMStatus.ACTIVE, timeout=VMTimeout.COLD_MIGRATE_CONFIRM,
|
||
fail_ok=fail_ok, con_ssh=con_ssh)
|
||
|
||
if post_confirm_state is None:
|
||
err_msg = "VM {} is not in Active state after {} Resize".format(
|
||
vm_id, verify_resize_str)
|
||
LOG.error(err_msg)
|
||
return 4, err_msg
|
||
|
||
after_flavor = get_vm_flavor(vm_id)
|
||
if revert and after_flavor != before_flavor:
|
||
err_msg = "Flavor is changed after revert resizing. Before flavor: " \
|
||
"{}, after flavor: {}".format(before_flavor, after_flavor)
|
||
if fail_ok:
|
||
LOG.error(err_msg)
|
||
return 5, err_msg
|
||
raise exceptions.VMPostCheckFailed(err_msg)
|
||
|
||
if not revert and after_flavor != flavor_id:
|
||
err_msg = "VM flavor {} is not changed to expected after resizing. " \
|
||
"Before flavor: {}, after flavor: {}".\
|
||
format(flavor_id, before_flavor, after_flavor)
|
||
if fail_ok:
|
||
LOG.error(err_msg)
|
||
return 6, err_msg
|
||
raise exceptions.VMPostCheckFailed(err_msg)
|
||
|
||
success_msg = "VM {} successfully resized and {}ed.".format(
|
||
vm_id, verify_resize_str)
|
||
LOG.info(success_msg)
|
||
return 0, success_msg
|
||
|
||
|
||
def wait_for_vm_values(vm_id, timeout=VMTimeout.STATUS_CHANGE, check_interval=3,
|
||
fail_ok=True, strict=True,
|
||
regex=False, con_ssh=None, auth_info=None, **kwargs):
|
||
"""
|
||
Wait for vm to reach given states.
|
||
|
||
Args:
|
||
vm_id (str): vm id
|
||
timeout (int): in seconds
|
||
check_interval (int): in seconds
|
||
fail_ok (bool): whether to return result or raise exception when vm
|
||
did not reach expected value(s).
|
||
strict (bool): whether to perform strict search(match) for the value(s)
|
||
For regular string: if True, match the whole string; if False,
|
||
find any substring match
|
||
For regex: if True, match from start of the value string; if
|
||
False, search anywhere of the value string
|
||
regex (bool): whether to use regex to find matching value(s)
|
||
con_ssh (SSHClient):
|
||
auth_info (dict):
|
||
**kwargs: field/value pair(s) to identify the waiting criteria.
|
||
|
||
Returns (tuple): (result(bool), actual_vals(dict))
|
||
|
||
"""
|
||
if not kwargs:
|
||
raise ValueError("No field/value pair is passed via kwargs")
|
||
LOG.info("Waiting for vm to reach state(s): {}".format(kwargs))
|
||
|
||
fields_to_check = list(kwargs.keys())
|
||
results = {}
|
||
end_time = time.time() + timeout
|
||
while time.time() < end_time:
|
||
actual_vals = get_vm_values(vm_id=vm_id, con_ssh=con_ssh,
|
||
auth_info=auth_info,
|
||
fields=fields_to_check)
|
||
for i in range(len(fields_to_check)):
|
||
field = fields_to_check[i]
|
||
expt_vals = kwargs[field]
|
||
actual_val = actual_vals[i]
|
||
results[field] = actual_val
|
||
if not isinstance(expt_vals, list):
|
||
expt_vals = [expt_vals]
|
||
for expt_val in expt_vals:
|
||
if regex:
|
||
match_found = re.match(expt_val,
|
||
actual_val) if strict else re.search(
|
||
expt_val, actual_val)
|
||
else:
|
||
match_found = expt_val == actual_val if strict else \
|
||
expt_val in actual_val
|
||
|
||
if match_found:
|
||
fields_to_check.remove(field)
|
||
|
||
if not fields_to_check:
|
||
LOG.info("VM has reached states: {}".format(results))
|
||
return True, results
|
||
|
||
time.sleep(check_interval)
|
||
|
||
msg = "VM {} did not reach expected states within timeout. Actual state(" \
|
||
"s): {}".format(vm_id, results)
|
||
if fail_ok:
|
||
LOG.warning(msg)
|
||
return False, results
|
||
else:
|
||
raise exceptions.VMTimeout(msg)
|
||
|
||
|
||
def wait_for_vm_status(vm_id, status=VMStatus.ACTIVE,
|
||
timeout=VMTimeout.STATUS_CHANGE, check_interval=3,
|
||
fail_ok=False,
|
||
con_ssh=None, auth_info=Tenant.get('admin')):
|
||
"""
|
||
|
||
Args:
|
||
vm_id:
|
||
status (list|str):
|
||
timeout:
|
||
check_interval:
|
||
fail_ok (bool):
|
||
con_ssh:
|
||
auth_info:
|
||
|
||
Returns: The Status of the vm_id depend on what Status it is looking for
|
||
|
||
"""
|
||
end_time = time.time() + timeout
|
||
if isinstance(status, str):
|
||
status = [status]
|
||
|
||
current_status = get_vm_values(vm_id, 'status', strict=True,
|
||
con_ssh=con_ssh, auth_info=auth_info)[0]
|
||
while time.time() < end_time:
|
||
for expected_status in status:
|
||
if current_status == expected_status:
|
||
LOG.info("VM status has reached {}".format(expected_status))
|
||
return expected_status
|
||
|
||
time.sleep(check_interval)
|
||
current_status = get_vm_values(vm_id, 'status', strict=True,
|
||
con_ssh=con_ssh, auth_info=auth_info)[0]
|
||
|
||
err_msg = "Timed out waiting for vm status: {}. Actual vm status: " \
|
||
"{}".format(status, current_status)
|
||
if fail_ok:
|
||
LOG.warning(err_msg)
|
||
return None
|
||
else:
|
||
raise exceptions.VMTimeout(err_msg)
|
||
|
||
|
||
def _confirm_or_revert_resize(vm, revert=False, con_ssh=None, fail_ok=False):
|
||
args = '--revert' if revert else '--confirm'
|
||
args = '{} {}'.format(args, vm)
|
||
return cli.openstack('server resize', args, ssh_client=con_ssh,
|
||
fail_ok=fail_ok, auth_info=Tenant.get('admin'))
|
||
|
||
|
||
def _get_vms_ips(vm_ids, net_types='mgmt', exclude_nets=None, con_ssh=None,
|
||
vshell=False):
|
||
if isinstance(net_types, str):
|
||
net_types = [net_types]
|
||
|
||
if isinstance(vm_ids, str):
|
||
vm_ids = [vm_ids]
|
||
|
||
valid_net_types = ['mgmt', 'data', 'internal', 'external']
|
||
if not set(net_types) <= set(valid_net_types):
|
||
raise ValueError(
|
||
"Invalid net type(s) provided. Valid net_types: {}. net_types "
|
||
"given: {}".
|
||
format(valid_net_types, net_types))
|
||
|
||
vms_ips = []
|
||
vshell_ips_dict = dict(data=[], internal=[])
|
||
if 'mgmt' in net_types:
|
||
mgmt_ips = network_helper.get_mgmt_ips_for_vms(
|
||
vms=vm_ids, con_ssh=con_ssh, exclude_nets=exclude_nets)
|
||
if not mgmt_ips:
|
||
raise exceptions.VMNetworkError(
|
||
"Management net ip is not found for vms {}".format(vm_ids))
|
||
vms_ips += mgmt_ips
|
||
|
||
if 'external' in net_types:
|
||
ext_ips = network_helper.get_external_ips_for_vms(
|
||
vms=vm_ids, con_ssh=con_ssh, exclude_nets=exclude_nets)
|
||
if not ext_ips:
|
||
raise exceptions.VMNetworkError(
|
||
"No external network ip found for vms {}".format(vm_ids))
|
||
vms_ips += ext_ips
|
||
|
||
if 'data' in net_types:
|
||
data_ips = network_helper.get_tenant_ips_for_vms(
|
||
vms=vm_ids, con_ssh=con_ssh, exclude_nets=exclude_nets)
|
||
if not data_ips:
|
||
raise exceptions.VMNetworkError(
|
||
"Data network ip is not found for vms {}".format(vm_ids))
|
||
if vshell:
|
||
vshell_ips_dict['data'] = data_ips
|
||
else:
|
||
vms_ips += data_ips
|
||
|
||
if 'internal' in net_types:
|
||
internal_ips = network_helper.get_internal_ips_for_vms(
|
||
vms=vm_ids, con_ssh=con_ssh, exclude_nets=exclude_nets)
|
||
if not internal_ips:
|
||
raise exceptions.VMNetworkError(
|
||
"Internal net ip is not found for vms {}".format(vm_ids))
|
||
if vshell:
|
||
vshell_ips_dict['internal'] = internal_ips
|
||
else:
|
||
vms_ips += internal_ips
|
||
|
||
return vms_ips, vshell_ips_dict
|
||
|
||
|
||
def _ping_vms(ssh_client, vm_ids=None, con_ssh=None, num_pings=5, timeout=15,
|
||
fail_ok=False, net_types='mgmt', retry=3,
|
||
retry_interval=3, vshell=False, sep_file=None,
|
||
source_net_types=None):
|
||
"""
|
||
|
||
Args:
|
||
vm_ids (list|str): list of vms to ping
|
||
ssh_client (SSHClient): ping from this ssh client. Usually a natbox'
|
||
ssh client or another vm's ssh client
|
||
con_ssh (SSHClient): active controller ssh client to run cli command
|
||
to get all the management ips
|
||
num_pings (int): number of pings to send
|
||
timeout (int): timeout waiting for response of ping messages in seconds
|
||
fail_ok (bool): Whether it's okay to have 100% packet loss rate.
|
||
sep_file (str|None)
|
||
net_types (str|list|tuple)
|
||
source_net_types (str|list|tuple|None):
|
||
vshell specific
|
||
None: use the same net_type s as the target IPs'
|
||
str: use the specified net_type for all target IPs
|
||
tuple: (net_type_data, net_type_internal)
|
||
use net_type_data for data IPs
|
||
use net_type_internal for internal IPs
|
||
list: same as tuple
|
||
|
||
Returns (tuple): (res (bool), packet_loss_dict (dict))
|
||
Packet loss rate dictionary format:
|
||
{
|
||
ip1: packet_loss_percentile1,
|
||
ip2: packet_loss_percentile2,
|
||
...
|
||
}
|
||
|
||
"""
|
||
vms_ips, vshell_ips_dict = _get_vms_ips(vm_ids=vm_ids, net_types=net_types,
|
||
con_ssh=con_ssh, vshell=vshell)
|
||
|
||
res_bool = False
|
||
res_dict = {}
|
||
for i in range(retry + 1):
|
||
for ip in vms_ips:
|
||
packet_loss_rate = network_helper.ping_server(
|
||
server=ip, ssh_client=ssh_client, num_pings=num_pings,
|
||
timeout=timeout, fail_ok=True, vshell=False)[0]
|
||
res_dict[ip] = packet_loss_rate
|
||
|
||
for net_type, vshell_ips in vshell_ips_dict.items():
|
||
|
||
if source_net_types is None:
|
||
pass
|
||
elif isinstance(source_net_types, str):
|
||
net_type = source_net_types
|
||
else:
|
||
net_type_data, net_type_internal = source_net_types
|
||
if net_type == 'data':
|
||
net_type = net_type_data
|
||
elif net_type == 'internal':
|
||
net_type = net_type_internal
|
||
else:
|
||
raise ValueError(net_type)
|
||
|
||
for vshell_ip in vshell_ips:
|
||
packet_loss_rate = network_helper.ping_server(
|
||
server=vshell_ip, ssh_client=ssh_client,
|
||
num_pings=num_pings, timeout=timeout, fail_ok=True,
|
||
vshell=True, net_type=net_type)[0]
|
||
res_dict[vshell_ip] = packet_loss_rate
|
||
|
||
res_bool = not any(loss_rate == 100 for loss_rate in res_dict.values())
|
||
if res_bool:
|
||
LOG.info(
|
||
"Ping successful from {}: {}".format(ssh_client.host, res_dict))
|
||
return res_bool, res_dict
|
||
|
||
if i < retry:
|
||
LOG.info("Retry in {} seconds".format(retry_interval))
|
||
time.sleep(retry_interval)
|
||
|
||
if not res_dict:
|
||
raise ValueError("Ping res dict contains no result.")
|
||
|
||
err_msg = "Ping unsuccessful from vm (logged in via {}): {}".format(
|
||
ssh_client.host, res_dict)
|
||
if fail_ok:
|
||
LOG.info(err_msg)
|
||
return res_bool, res_dict
|
||
else:
|
||
if sep_file:
|
||
msg = "==========================Ping unsuccessful from vm to " \
|
||
"vms===================="
|
||
common.write_to_file(
|
||
sep_file,
|
||
content="{}\nLogged into vm via {}. Result: {}".format(
|
||
msg, ssh_client.host, res_dict))
|
||
raise exceptions.VMNetworkError(err_msg)
|
||
|
||
|
||
def configure_vm_vifs_on_same_net(vm_id, vm_ips=None, ports=None,
|
||
vm_prompt=None, restart_service=True,
|
||
reboot=False):
|
||
"""
|
||
Configure vm routes if the vm has multiple vifs on same network.
|
||
Args:
|
||
vm_id (str):
|
||
vm_ips (str|list): ips for specific vifs. Only works if vifs are up
|
||
with ips assigned
|
||
ports (list of dict): vm ports to configure.
|
||
vm_prompt (None|str)
|
||
restart_service
|
||
reboot
|
||
|
||
Returns:
|
||
|
||
"""
|
||
|
||
if isinstance(vm_ips, str):
|
||
vm_ips = [vm_ips]
|
||
|
||
vnics_info = {}
|
||
if ports:
|
||
LOG.info("Get vm interfaces' mac and ip addressess")
|
||
if isinstance(ports, str):
|
||
ports = [ports]
|
||
vm_interfaces_table = table_parser.table(
|
||
cli.openstack('port list', '--server {}'.format(vm_id))[1])
|
||
vm_interfaces_dict = table_parser.row_dict_table(
|
||
table_=vm_interfaces_table, key_header='ID')
|
||
for i in range(len(ports)):
|
||
port_id = ports[i]
|
||
vif_info = vm_interfaces_dict[port_id]
|
||
vif_ip = vif_info['fixed ip addresses']
|
||
if vif_ip and 'ip_address' in vif_ip:
|
||
vif_ip = \
|
||
re.findall("ip_address='(.*)'", vif_ip.split(sep=',')[0])[0]
|
||
else:
|
||
if not vm_ips:
|
||
raise ValueError(
|
||
"vm_ips for matching vnics has to be provided for "
|
||
"ports without ip address "
|
||
"listed in neutron port-list")
|
||
vif_ip = vm_ips[i]
|
||
cidr = vif_ip.rsplit('.', maxsplit=1)[0] + '.0/24'
|
||
vif_mac = vif_info['mac address']
|
||
vnics_info[vif_mac] = (cidr, vif_ip)
|
||
|
||
LOG.info("Configure vm routes if the vm has multiple vifs on same network.")
|
||
with ssh_to_vm_from_natbox(vm_id=vm_id, prompt=vm_prompt) as vm_ssh:
|
||
vifs_to_conf = {}
|
||
if not ports:
|
||
extra_grep = '| grep --color=never -E "{}"'.format(
|
||
'|'.join(vm_ips)) if vm_ips else ''
|
||
kernel_routes = vm_ssh.exec_cmd(
|
||
'ip route | grep --color=never "proto kernel" {}'.format(
|
||
extra_grep))[1]
|
||
cidr_dict = {}
|
||
for line in kernel_routes.splitlines():
|
||
found = re.findall(
|
||
r'^(.*/\d+)\sdev\s(.*)\sproto kernel.*\ssrc\s(.*)$', line)
|
||
cidr, dev_name, dev_ip = found[0]
|
||
if cidr not in cidr_dict:
|
||
cidr_dict[cidr] = []
|
||
cidr_dict[cidr].append((dev_name, dev_ip))
|
||
|
||
for cidr_, val in cidr_dict.items():
|
||
if not vm_ips:
|
||
val = val[1:]
|
||
for eth_info in val:
|
||
dev_name, dev_ip = eth_info
|
||
vifs_to_conf[dev_name] = \
|
||
(cidr_, dev_ip, 'stxauto_{}'.format(dev_name))
|
||
|
||
if not vifs_to_conf:
|
||
LOG.info(
|
||
"Did not find multiple vifs on same subnet. Do nothing.")
|
||
|
||
else:
|
||
for mac_addr in vnics_info:
|
||
dev_name = network_helper.get_eth_for_mac(vm_ssh,
|
||
mac_addr=mac_addr)
|
||
cidr_, dev_ip = vnics_info[mac_addr]
|
||
vifs_to_conf[dev_name] = (
|
||
cidr_, dev_ip, 'stxauto_{}'.format(dev_name))
|
||
|
||
used_tables = vm_ssh.exec_cmd(
|
||
'grep --color=never -E "^[0-9]" {}'.format(VMPath.RT_TABLES))[1]
|
||
used_tables = [int(re.split(r'[\s\t]', line_)[0].strip()) for line_ in
|
||
used_tables.splitlines()]
|
||
|
||
start_range = 110
|
||
for eth_name, eth_info in vifs_to_conf.items():
|
||
cidr_, vif_ip, table_name = eth_info
|
||
exiting_tab = vm_ssh.exec_cmd(
|
||
'grep --color=never {} {}'.format(table_name,
|
||
VMPath.RT_TABLES))[1]
|
||
if not exiting_tab:
|
||
for i in range(start_range, 250):
|
||
if i not in used_tables:
|
||
LOG.info(
|
||
"Append new routing table {} to rt_tables".
|
||
format(table_name))
|
||
vm_ssh.exec_sudo_cmd(
|
||
'echo "{} {}" >> {}'.format(i, table_name,
|
||
VMPath.RT_TABLES))
|
||
start_range = i + 1
|
||
break
|
||
else:
|
||
raise ValueError(
|
||
"Unable to get a valid table number to create route "
|
||
"for {}".format(eth_name))
|
||
|
||
LOG.info(
|
||
"Update arp_filter, arp_announce, route and rule scripts for "
|
||
"vm {} {}".format(vm_id, eth_name))
|
||
vm_ssh.exec_sudo_cmd(
|
||
'echo 2 > {}'.format(VMPath.ETH_ARP_ANNOUNCE.format(eth_name)))
|
||
vm_ssh.exec_sudo_cmd(
|
||
'echo 1 > {}'.format(VMPath.ETH_ARP_FILTER.format(eth_name)))
|
||
route = '{} dev {} proto kernel scope link src {} table {}'.format(
|
||
cidr_, eth_name, vif_ip, table_name)
|
||
vm_ssh.exec_sudo_cmd('echo "{}" > {}'.format(
|
||
route, VMPath.ETH_RT_SCRIPT.format(eth_name)))
|
||
rule = 'table {} from {}'.format(table_name, vif_ip)
|
||
vm_ssh.exec_sudo_cmd('echo "{}" > {}'.format(
|
||
rule, VMPath.ETH_RULE_SCRIPT.format(eth_name)))
|
||
|
||
if restart_service and not reboot:
|
||
LOG.info("Restart network service after configure vm routes")
|
||
vm_ssh.exec_sudo_cmd('systemctl restart network',
|
||
expect_timeout=120, get_exit_code=False)
|
||
# vm_ssh.exec_cmd('ip addr')
|
||
|
||
if reboot:
|
||
LOG.info("Reboot vm after configure vm routes")
|
||
reboot_vm(vm_id=vm_id)
|
||
|
||
|
||
def cleanup_routes_for_vifs(vm_id, vm_ips, rm_ifcfg=True, restart_service=True,
|
||
reboot=False):
|
||
"""
|
||
Cleanup the configured routes for specified vif(s). This is needed when a
|
||
vif is detached from a vm.
|
||
|
||
Args:
|
||
vm_id:
|
||
vm_ips:
|
||
rm_ifcfg
|
||
restart_service
|
||
reboot
|
||
|
||
Returns:
|
||
|
||
"""
|
||
with ssh_to_vm_from_natbox(vm_id=vm_id) as vm_ssh:
|
||
|
||
if isinstance(vm_ips, str):
|
||
vm_ips = [vm_ips]
|
||
|
||
for vm_ip in vm_ips:
|
||
LOG.info("Clean up route for dev with ip {}".format(vm_ip))
|
||
route = vm_ssh.exec_sudo_cmd(
|
||
'grep --color=never {} {}'.format(
|
||
vm_ip, VMPath.ETH_RT_SCRIPT.format('*')))[1]
|
||
if not route:
|
||
continue
|
||
|
||
pattern = '(.*) dev (.*) proto kernel .* src {} table (.*)'.format(
|
||
vm_ip)
|
||
found = re.findall(pattern, route)
|
||
if found:
|
||
cidr, eth_name, table_name = found[0]
|
||
LOG.info(
|
||
"Update arp_filter, arp_announce, route and rule scripts "
|
||
"for vm {} {}".format(vm_id, eth_name))
|
||
# vm_ssh.exec_sudo_cmd('rm -f {}'.format(
|
||
# VMPath.ETH_ARP_ANNOUNCE.format(eth_name)))
|
||
# vm_ssh.exec_sudo_cmd('rm -f {}'.format(
|
||
# VMPath.ETH_ARP_FILTER.format(eth_name)))
|
||
vm_ssh.exec_sudo_cmd(
|
||
'rm -f {}'.format(VMPath.ETH_RULE_SCRIPT.format(eth_name)))
|
||
vm_ssh.exec_sudo_cmd(
|
||
'rm -f {}'.format(VMPath.ETH_RT_SCRIPT.format(eth_name)))
|
||
vm_ssh.exec_sudo_cmd("sed -n -i '/{}/!p' {}".format(
|
||
table_name, VMPath.RT_TABLES))
|
||
|
||
if rm_ifcfg:
|
||
vm_ssh.exec_sudo_cmd('rm -f {}'.format(
|
||
VMPath.ETH_PATH_CENTOS.format(eth_name)))
|
||
|
||
if restart_service and not reboot:
|
||
LOG.info("Restart network service")
|
||
vm_ssh.exec_sudo_cmd('systemctl restart network',
|
||
get_exit_code=False, expect_timeout=60)
|
||
|
||
if reboot:
|
||
reboot_vm(vm_id=vm_id)
|
||
|
||
|
||
def ping_vms_from_natbox(vm_ids=None, natbox_client=None, con_ssh=None,
|
||
num_pings=5, timeout=30, fail_ok=False,
|
||
use_fip=False, retry=0):
|
||
"""
|
||
|
||
Args:
|
||
vm_ids: vms to ping. If None, all vms will be ping'd.
|
||
con_ssh (SSHClient): active controller client to retrieve the vm info
|
||
natbox_client (NATBoxClient): ping vms from this client
|
||
num_pings (int): number of pings to send
|
||
timeout (int): timeout waiting for response of ping messages in seconds
|
||
fail_ok (bool): When False, test will stop right away if one ping
|
||
failed. When True, test will continue to ping
|
||
the rest of the vms and return results even if pinging one vm
|
||
failed.
|
||
use_fip (bool): Whether to ping floating ip only if a vm has more
|
||
than one management ips
|
||
retry (int): number of times to retry if ping fails
|
||
|
||
Returns (tuple): (res (bool), packet_loss_dict (dict))
|
||
Packet loss rate dictionary format:
|
||
{
|
||
ip1: packet_loss_percentile1,
|
||
ip2: packet_loss_percentile2,
|
||
...
|
||
}
|
||
"""
|
||
if isinstance(vm_ids, str):
|
||
vm_ids = [vm_ids]
|
||
|
||
if not natbox_client:
|
||
natbox_client = NATBoxClient.get_natbox_client()
|
||
|
||
if not con_ssh:
|
||
con_ssh = ControllerClient.get_active_controller()
|
||
|
||
net_type = 'external' if use_fip else 'mgmt'
|
||
res_bool, res_dict = _ping_vms(ssh_client=natbox_client, vm_ids=vm_ids,
|
||
con_ssh=con_ssh, num_pings=num_pings,
|
||
timeout=timeout, fail_ok=True,
|
||
net_types=net_type, retry=retry,
|
||
vshell=False)
|
||
if not res_bool and not fail_ok:
|
||
msg = "==================Ping vm(s) from NatBox failed - Collecting " \
|
||
"extra information==============="
|
||
LOG.error(msg)
|
||
time_stamp = common.get_date_in_format(ssh_client=con_ssh,
|
||
date_format='%Y%m%d_%H-%M')
|
||
f_path = '{}/{}-{}'.format(ProjVar.get_var('PING_FAILURE_DIR'),
|
||
time_stamp, ProjVar.get_var("TEST_NAME"))
|
||
common.write_to_file(file_path=f_path,
|
||
content="\n{}\nResult(s): {}\n".format(msg,
|
||
res_dict))
|
||
ProjVar.set_var(PING_FAILURE=True)
|
||
get_console_logs(vm_ids=vm_ids, sep_file=f_path)
|
||
network_helper.collect_networking_info(vms=vm_ids, sep_file=f_path,
|
||
time_stamp=time_stamp,
|
||
con_ssh=con_ssh)
|
||
raise exceptions.VMNetworkError(
|
||
"Ping failed from NatBox. Details: {}".format(res_dict))
|
||
|
||
return res_bool, res_dict
|
||
|
||
|
||
def get_console_logs(vm_ids, length=None, con_ssh=None, sep_file=None):
|
||
"""
|
||
Get console logs for given vm(s)
|
||
Args:
|
||
vm_ids (str|list):
|
||
length (int|None): how many lines to tail
|
||
con_ssh:
|
||
sep_file (str|None): write vm console logs to given sep_file if
|
||
specified.
|
||
|
||
Returns (dict): {<vm1_id>: <vm1_console>, <vm2_id>: <vm2_console>, ...}
|
||
"""
|
||
if isinstance(vm_ids, str):
|
||
vm_ids = [vm_ids]
|
||
|
||
vm_ids = list(set(vm_ids))
|
||
console_logs = {}
|
||
args = '--lines={} '.format(length) if length else ''
|
||
content = ''
|
||
for vm_id in vm_ids:
|
||
vm_args = '{}{}'.format(args, vm_id)
|
||
output = cli.openstack('console log show', vm_args, ssh_client=con_ssh,
|
||
auth_info=Tenant.get('admin'))[1]
|
||
console_logs[vm_id] = output
|
||
content += "\n#### Console log for vm {} ####\n{}\n".format(vm_id,
|
||
output)
|
||
|
||
if sep_file:
|
||
common.write_to_file(sep_file, content=content)
|
||
|
||
return console_logs
|
||
|
||
|
||
def ping_vms_from_vm(to_vms=None, from_vm=None, user=None, password=None,
|
||
prompt=None, con_ssh=None, natbox_client=None,
|
||
num_pings=5, timeout=120, fail_ok=False, from_vm_ip=None,
|
||
from_fip=False, net_types='mgmt',
|
||
retry=3, retry_interval=5, vshell=False,
|
||
source_net_types=None):
|
||
"""
|
||
|
||
Args:
|
||
from_vm (str):
|
||
to_vms (str|list|None):
|
||
user (str):
|
||
password (str):
|
||
prompt (str):
|
||
con_ssh (SSHClient):
|
||
natbox_client (SSHClient):
|
||
num_pings (int):
|
||
timeout (int): max number of seconds to wait for ssh connection to
|
||
from_vm
|
||
fail_ok (bool): When False, test will stop right away if one ping
|
||
failed. When True, test will continue to ping
|
||
the rest of the vms and return results even if pinging one vm
|
||
failed.
|
||
from_vm_ip (str): vm ip to ssh to if given. from_fip flag will be
|
||
considered only if from_vm_ip=None
|
||
from_fip (bool): whether to ssh to vm's floating ip if it has
|
||
floating ip associated with it
|
||
net_types (list|str|tuple): 'mgmt', 'data', or 'internal'
|
||
retry (int): number of times to retry
|
||
retry_interval (int): seconds to wait between each retries
|
||
vshell (bool): whether to ping vms' data interface through internal
|
||
interface.
|
||
Usage: when set to True, use 'vshell ping --count 3
|
||
<other_vm_data_ip> <internal_if_id>'
|
||
- dpdk vms should be booted from lab_setup scripts
|
||
source_net_types (str|list|tuple|None):
|
||
vshell specific
|
||
None: use the same net_type s as the target IPs'
|
||
str: use the specified net_type for all target IPs
|
||
tuple: (net_type_data, net_type_internal)
|
||
use net_type_data for data IPs
|
||
use net_type_internal for internal IPs
|
||
list: same as tuple
|
||
|
||
Returns (tuple):
|
||
A tuple in form: (res (bool), packet_loss_dict (dict))
|
||
|
||
Packet loss rate dictionary format:
|
||
{
|
||
ip1: packet_loss_percentile1,
|
||
ip2: packet_loss_percentile2,
|
||
...
|
||
}
|
||
|
||
"""
|
||
if isinstance(net_types, str):
|
||
net_types = [net_types]
|
||
|
||
if from_vm is None or to_vms is None:
|
||
vms_ips = network_helper.get_mgmt_ips_for_vms(con_ssh=con_ssh,
|
||
rtn_dict=True)
|
||
if not vms_ips:
|
||
raise exceptions.NeutronError("No management ip found for any vms")
|
||
|
||
vms_ids = list(vms_ips.keys())
|
||
if from_vm is None:
|
||
from_vm = random.choice(vms_ids)
|
||
if to_vms is None:
|
||
to_vms = vms_ids
|
||
|
||
if isinstance(to_vms, str):
|
||
to_vms = [to_vms]
|
||
|
||
if not isinstance(from_vm, str):
|
||
raise ValueError("from_vm is not a string: {}".format(from_vm))
|
||
|
||
assert from_vm and to_vms, "from_vm: {}, to_vms: {}".format(from_vm, to_vms)
|
||
|
||
time_stamp = common.get_date_in_format(ssh_client=con_ssh,
|
||
date_format='%Y%m%d_%H-%M')
|
||
f_path = '{}/{}-{}'.format(ProjVar.get_var('PING_FAILURE_DIR'), time_stamp,
|
||
ProjVar.get_var('TEST_NAME'))
|
||
try:
|
||
with ssh_to_vm_from_natbox(vm_id=from_vm, username=user,
|
||
password=password,
|
||
natbox_client=natbox_client,
|
||
prompt=prompt, con_ssh=con_ssh,
|
||
vm_ip=from_vm_ip, use_fip=from_fip,
|
||
retry_timeout=300) as from_vm_ssh:
|
||
res = _ping_vms(ssh_client=from_vm_ssh, vm_ids=to_vms,
|
||
con_ssh=con_ssh, num_pings=num_pings,
|
||
timeout=timeout, fail_ok=fail_ok,
|
||
net_types=net_types, retry=retry,
|
||
retry_interval=retry_interval, vshell=vshell,
|
||
sep_file=f_path,
|
||
source_net_types=source_net_types)
|
||
return res
|
||
|
||
except (exceptions.TiSError, pexpect.ExceptionPexpect):
|
||
ProjVar.set_var(PING_FAILURE=True)
|
||
collect_to_vms = False if list(to_vms) == [from_vm] else True
|
||
get_console_logs(vm_ids=from_vm, length=20, sep_file=f_path)
|
||
if collect_to_vms:
|
||
get_console_logs(vm_ids=to_vms, sep_file=f_path)
|
||
network_helper.collect_networking_info(vms=to_vms, sep_file=f_path,
|
||
time_stamp=time_stamp)
|
||
try:
|
||
LOG.warning(
|
||
"Ping vm(s) from vm failed - Attempt to ssh to from_vm and "
|
||
"collect vm networking info")
|
||
with ssh_to_vm_from_natbox(vm_id=from_vm, username=user,
|
||
password=password,
|
||
natbox_client=natbox_client,
|
||
prompt=prompt, con_ssh=con_ssh,
|
||
vm_ip=from_vm_ip,
|
||
use_fip=from_fip) as from_vm_ssh:
|
||
_collect_vm_networking_info(vm_ssh=from_vm_ssh, sep_file=f_path,
|
||
vm_id=from_vm)
|
||
|
||
if collect_to_vms:
|
||
LOG.warning(
|
||
"Ping vm(s) from vm failed - Attempt to ssh to to_vms and "
|
||
"collect vm networking info")
|
||
for vm_ in to_vms:
|
||
with ssh_to_vm_from_natbox(vm_, retry=False,
|
||
con_ssh=con_ssh) as to_ssh:
|
||
_collect_vm_networking_info(to_ssh, sep_file=f_path,
|
||
vm_id=vm_)
|
||
except (exceptions.TiSError, pexpect.ExceptionPexpect):
|
||
pass
|
||
|
||
raise
|
||
|
||
|
||
def _collect_vm_networking_info(vm_ssh, sep_file=None, vm_id=None):
|
||
vm = vm_id if vm_id else ''
|
||
content = '#### VM network info collected when logged into vm {}via {} ' \
|
||
'####'.format(vm, vm_ssh.host)
|
||
for cmd in ('ip addr', 'ip neigh', 'ip route'):
|
||
output = vm_ssh.exec_cmd(cmd, get_exit_code=False)[1]
|
||
content += '\nSent: {}\nOutput:\n{}\n'.format(cmd, output)
|
||
|
||
if sep_file:
|
||
common.write_to_file(sep_file, content=content)
|
||
|
||
|
||
def ping_ext_from_vm(from_vm, ext_ip=None, user=None, password=None,
|
||
prompt=None, con_ssh=None, natbox_client=None,
|
||
num_pings=5, timeout=30, fail_ok=False, vm_ip=None,
|
||
use_fip=False):
|
||
if ext_ip is None:
|
||
ext_ip = EXT_IP
|
||
|
||
with ssh_to_vm_from_natbox(vm_id=from_vm, username=user, password=password,
|
||
natbox_client=natbox_client,
|
||
prompt=prompt, con_ssh=con_ssh, vm_ip=vm_ip,
|
||
use_fip=use_fip) as from_vm_ssh:
|
||
from_vm_ssh.exec_cmd('ip addr', get_exit_code=False)
|
||
return network_helper.ping_server(ext_ip, ssh_client=from_vm_ssh,
|
||
num_pings=num_pings,
|
||
timeout=timeout, fail_ok=fail_ok)[0]
|
||
|
||
|
||
def scp_to_vm_from_natbox(vm_id, source_file, dest_file, timeout=60,
|
||
validate=True, natbox_client=None, sha1sum=None):
|
||
"""
|
||
scp a file to a vm from natbox
|
||
the file must be located in the natbox
|
||
the natbox must has connectivity to the VM
|
||
|
||
Args:
|
||
vm_id (str): vm to scp to
|
||
source_file (str): full pathname to the source file
|
||
dest_file (str): destination full pathname in the VM
|
||
timeout (int): scp timeout
|
||
validate (bool): verify src and dest sha1sum
|
||
natbox_client (NATBoxClient|None):
|
||
sha1sum (str|None): validates the source file prior to operation,
|
||
or None, only checked if validate=True
|
||
|
||
Returns (None):
|
||
|
||
"""
|
||
if natbox_client is None:
|
||
natbox_client = NATBoxClient.get_natbox_client()
|
||
|
||
LOG.info("scp-ing from {} to VM {}".format(natbox_client.host, vm_id))
|
||
|
||
tmp_loc = '/tmp'
|
||
fname = os.path.basename(os.path.normpath(source_file))
|
||
|
||
# ensure source file exists
|
||
natbox_client.exec_cmd('test -f {}'.format(source_file), fail_ok=False)
|
||
|
||
# calculate sha1sum
|
||
src_sha1 = None
|
||
if validate:
|
||
src_sha1 = natbox_client.exec_cmd('sha1sum {}'.format(source_file),
|
||
fail_ok=False)[1]
|
||
src_sha1 = src_sha1.split(' ')[0]
|
||
LOG.info("src: {}, sha1sum: {}".format(source_file, src_sha1))
|
||
if sha1sum is not None and src_sha1 != sha1sum:
|
||
raise ValueError(
|
||
"src sha1sum validation failed {} != {}".format(src_sha1,
|
||
sha1sum))
|
||
|
||
with ssh_to_vm_from_natbox(vm_id) as vm_ssh:
|
||
vm_ssh.exec_cmd('mkdir -p {}'.format(tmp_loc))
|
||
vm_ssh.scp_on_dest(natbox_client.user, natbox_client.host, source_file,
|
||
'/'.join([tmp_loc, fname]), natbox_client.password,
|
||
timeout=timeout)
|
||
|
||
# `mv $s $d` fails if $s == $d
|
||
if os.path.normpath(os.path.join(tmp_loc, fname)) != os.path.normpath(
|
||
dest_file):
|
||
vm_ssh.exec_sudo_cmd(
|
||
'mv -f {} {}'.format('/'.join([tmp_loc, fname]), dest_file),
|
||
fail_ok=False)
|
||
|
||
# ensure destination file exists
|
||
vm_ssh.exec_sudo_cmd('test -f {}'.format(dest_file), fail_ok=False)
|
||
|
||
# validation
|
||
if validate:
|
||
dest_sha1 = vm_ssh.exec_sudo_cmd(
|
||
'sha1sum {}'.format(dest_file), fail_ok=False)[1]
|
||
dest_sha1 = dest_sha1.split(' ')[0]
|
||
LOG.info("dst: {}, sha1sum: {}".format(dest_file, dest_sha1))
|
||
if src_sha1 != dest_sha1:
|
||
raise ValueError(
|
||
"dst sha1sum validation failed {} != {}".format(src_sha1,
|
||
dest_sha1))
|
||
LOG.info("scp completed successfully")
|
||
|
||
|
||
def scp_to_vm(vm_id, source_file, dest_file, timeout=60, validate=True,
|
||
source_ssh=None, natbox_client=None):
|
||
"""
|
||
scp a file from any SSHClient to a VM
|
||
since not all SSHClient's has connectivity to the VM, this function scps
|
||
the source file to natbox first
|
||
|
||
Args:
|
||
vm_id (str): vm to scp to
|
||
source_file (str): full pathname to the source file
|
||
dest_file (str): destination path in the VM
|
||
timeout (int): scp timeout
|
||
validate (bool): verify src and dest sha1sum
|
||
source_ssh (SSHClient|None): the source ssh session, or None to use
|
||
'localhost'
|
||
natbox_client (NATBoxClient|None):
|
||
|
||
Returns (None):
|
||
|
||
"""
|
||
if not natbox_client:
|
||
natbox_client = NATBoxClient.get_natbox_client()
|
||
|
||
close_source = False
|
||
if not source_ssh:
|
||
source_ssh = LocalHostClient()
|
||
source_ssh.connect()
|
||
close_source = True
|
||
|
||
try:
|
||
# scp-ing from natbox, forward the call
|
||
if source_ssh.host == natbox_client.host:
|
||
return scp_to_vm_from_natbox(vm_id, source_file, dest_file, timeout,
|
||
validate, natbox_client=natbox_client)
|
||
|
||
LOG.info("scp-ing from {} to natbox {}".format(source_ssh.host,
|
||
natbox_client.host))
|
||
tmp_loc = '~'
|
||
fname = os.path.basename(os.path.normpath(source_file))
|
||
|
||
# ensure source file exists
|
||
source_ssh.exec_cmd('test -f {}'.format(source_file), fail_ok=False)
|
||
|
||
# calculate sha1sum
|
||
if validate:
|
||
src_sha1 = source_ssh.exec_cmd('sha1sum {}'.format(source_file),
|
||
fail_ok=False)[1]
|
||
src_sha1 = src_sha1.split(' ')[0]
|
||
LOG.info("src: {}, sha1sum: {}".format(source_file, src_sha1))
|
||
else:
|
||
src_sha1 = None
|
||
|
||
# scp to natbox
|
||
# natbox_client.exec_cmd('mkdir -p {}'.format(tmp_loc))
|
||
source_ssh.scp_on_source(
|
||
source_file, natbox_client.user, natbox_client.host, tmp_loc,
|
||
natbox_client.password, timeout=timeout)
|
||
|
||
return scp_to_vm_from_natbox(
|
||
vm_id, '/'.join([tmp_loc, fname]), dest_file, timeout, validate,
|
||
natbox_client=natbox_client, sha1sum=src_sha1)
|
||
|
||
finally:
|
||
if close_source:
|
||
source_ssh.close()
|
||
|
||
|
||
@contextmanager
|
||
def ssh_to_vm_from_natbox(vm_id, vm_image_name=None, username=None,
|
||
password=None, prompt=None,
|
||
timeout=VMTimeout.SSH_LOGIN, natbox_client=None,
|
||
con_ssh=None, vm_ip=None,
|
||
vm_ext_port=None, use_fip=False, retry=True,
|
||
retry_timeout=120, close_ssh=True,
|
||
auth_info=Tenant.get('admin')):
|
||
"""
|
||
ssh to a vm from natbox.
|
||
|
||
Args:
|
||
vm_id (str): vm to ssh to
|
||
vm_image_name (str): such as cgcs-guest, tis-centos-guest, ubuntu_14
|
||
username (str):
|
||
password (str):
|
||
prompt (str):
|
||
timeout (int):
|
||
natbox_client (NATBoxClient):
|
||
con_ssh (SSHClient): ssh connection to TiS active controller
|
||
vm_ip (str): ssh to this ip from NatBox if given
|
||
vm_ext_port (str): port forwarding rule external port. If given this
|
||
port will be used. vm_ip must be external
|
||
router ip address.
|
||
use_fip (bool): Whether to ssh to floating ip if a vm has one
|
||
associated. Not applicable if vm_ip is given.
|
||
retry (bool): whether or not to retry if fails to connect
|
||
retry_timeout (int): max time to retry
|
||
close_ssh
|
||
auth_info (dict|None)
|
||
|
||
Yields (VMSSHClient):
|
||
ssh client of the vm
|
||
|
||
Examples:
|
||
with ssh_to_vm_from_natbox(vm_id=<id>) as vm_ssh:
|
||
vm_ssh.exec_cmd(cmd)
|
||
|
||
"""
|
||
if vm_image_name is None:
|
||
vm_image_name = get_vm_image_name(vm_id=vm_id, con_ssh=con_ssh,
|
||
auth_info=auth_info).strip().lower()
|
||
|
||
if vm_ip is None:
|
||
if use_fip:
|
||
vm_ip = network_helper.get_external_ips_for_vms(
|
||
vms=vm_id, con_ssh=con_ssh, auth_info=auth_info)[0]
|
||
else:
|
||
vm_ip = network_helper.get_mgmt_ips_for_vms(
|
||
vms=vm_id, con_ssh=con_ssh, auth_info=auth_info)[0]
|
||
|
||
if not natbox_client:
|
||
natbox_client = NATBoxClient.get_natbox_client()
|
||
|
||
try:
|
||
vm_ssh = VMSSHClient(natbox_client=natbox_client, vm_ip=vm_ip,
|
||
vm_ext_port=vm_ext_port,
|
||
vm_img_name=vm_image_name, user=username,
|
||
password=password, prompt=prompt,
|
||
timeout=timeout, retry=retry,
|
||
retry_timeout=retry_timeout)
|
||
|
||
except (exceptions.TiSError, pexpect.ExceptionPexpect):
|
||
LOG.warning(
|
||
'Failed to ssh to VM {}! Collecting vm console log'.format(vm_id))
|
||
get_console_logs(vm_ids=vm_id)
|
||
raise
|
||
|
||
try:
|
||
yield vm_ssh
|
||
finally:
|
||
if close_ssh:
|
||
vm_ssh.close()
|
||
|
||
|
||
def get_vm_pid(instance_name, host_ssh):
|
||
"""
|
||
Get instance pid on its host.
|
||
|
||
Args:
|
||
instance_name: instance name of a vm
|
||
host_ssh: ssh for the host of the given instance
|
||
|
||
Returns (str): pid of a instance on its host
|
||
|
||
"""
|
||
code, vm_pid = host_ssh.exec_sudo_cmd(
|
||
"ps aux | grep --color='never' {} | grep -v grep | awk '{{print $2}}'".
|
||
format(instance_name))
|
||
if code != 0:
|
||
raise exceptions.SSHExecCommandFailed(
|
||
"Failed to get pid for vm: {}".format(instance_name))
|
||
|
||
if not vm_pid:
|
||
LOG.warning("PID for {} is not found on host!".format(instance_name))
|
||
|
||
return vm_pid
|
||
|
||
|
||
class VMInfo:
|
||
"""
|
||
class for storing and retrieving information for specific VM using
|
||
openstack admin.
|
||
|
||
Notes: Do not use this class for vm actions, such as boot, delete,
|
||
migrate, etc as these actions should be done by
|
||
tenants.
|
||
"""
|
||
__instances = {}
|
||
active_controller_ssh = None
|
||
|
||
def __init__(self, vm_id, con_ssh=None, auth_info=Tenant.get('admin')):
|
||
"""
|
||
|
||
Args:
|
||
vm_id:
|
||
con_ssh: floating controller ssh for the system
|
||
|
||
Returns:
|
||
|
||
"""
|
||
if con_ssh is None:
|
||
con_ssh = ControllerClient.get_active_controller()
|
||
VMInfo.active_controller_ssh = con_ssh
|
||
self.vm_id = vm_id
|
||
self.con_ssh = con_ssh
|
||
self.auth_info = auth_info
|
||
self.initial_table_ = table_parser.table(
|
||
cli.openstack('server show', vm_id, ssh_client=con_ssh,
|
||
auth_info=self.auth_info, timeout=60)[1])
|
||
self.table_ = self.initial_table_
|
||
self.name = table_parser.get_value_two_col_table(self.initial_table_,
|
||
'name', strict=True)
|
||
self.tenant_id = table_parser.get_value_two_col_table(
|
||
self.initial_table_, 'project_id')
|
||
self.user_id = table_parser.get_value_two_col_table(self.initial_table_,
|
||
'user_id')
|
||
self.boot_info = self.__get_boot_info()
|
||
self.flavor_table = None
|
||
VMInfo.__instances[
|
||
vm_id] = self # add instance to class variable for tracking
|
||
|
||
def refresh_table(self):
|
||
self.table_ = table_parser.table(
|
||
cli.openstack('server show', self.vm_id, ssh_client=self.con_ssh,
|
||
auth_info=self.auth_info, timeout=60)[1])
|
||
|
||
def get_host_name(self):
|
||
self.refresh_table()
|
||
return table_parser.get_value_two_col_table(table_=self.table_,
|
||
field=':host', strict=False)
|
||
|
||
def get_flavor_id(self):
|
||
"""
|
||
|
||
Returns: (dict) {'name': flavor_name, 'id': flavor_id}
|
||
|
||
"""
|
||
flavor = table_parser.get_value_two_col_table(self.table_, 'flavor')
|
||
flavor_id = re.findall(r'\((.*)\)', flavor)[0]
|
||
return flavor_id
|
||
|
||
def refresh_flavor_table(self):
|
||
flavor_id = self.get_flavor_id()
|
||
self.flavor_table = table_parser.table(
|
||
cli.openstack('flavor show', flavor_id, ssh_client=self.con_ssh,
|
||
auth_info=Tenant.get('admin'))[1])
|
||
return self.flavor_table
|
||
|
||
def __get_boot_info(self):
|
||
return _get_boot_info(table_=self.table_, vm_id=self.vm_id,
|
||
auth_info=self.auth_info,
|
||
con_ssh=self.con_ssh)
|
||
|
||
def get_storage_type(self):
|
||
table_ = self.flavor_table
|
||
if not table_:
|
||
table_ = self.refresh_flavor_table()
|
||
extra_specs = table_parser.get_value_two_col_table(table_, 'properties',
|
||
merge_lines=True)
|
||
extra_specs = table_parser.convert_value_to_dict(value=extra_specs)
|
||
return extra_specs.get(FlavorSpec.STORAGE_BACKING, None)
|
||
|
||
def has_local_disks(self):
|
||
if self.boot_info['type'] == 'image':
|
||
return True
|
||
|
||
table_ = self.flavor_table
|
||
if not table_:
|
||
table_ = self.refresh_flavor_table()
|
||
swap = table_parser.get_value_two_col_table(table_, 'swap')
|
||
ephemeral = table_parser.get_value_two_col_table(table_, 'ephemeral',
|
||
strict=False)
|
||
return bool(swap or int(ephemeral))
|
||
|
||
@classmethod
|
||
def get_vms_info(cls):
|
||
return tuple(cls.__instances)
|
||
|
||
@classmethod
|
||
def get_vm_info(cls, vm_id, con_ssh=None):
|
||
if vm_id not in cls.__instances:
|
||
if vm_id in get_all_vms(con_ssh=con_ssh):
|
||
return cls(vm_id, con_ssh)
|
||
else:
|
||
raise exceptions.VMError(
|
||
"VM with id {} does not exist!".format(vm_id))
|
||
instance = cls.__instances[vm_id]
|
||
instance.refresh_table()
|
||
return instance
|
||
|
||
@classmethod
|
||
def remove_instance(cls, vm_id):
|
||
cls.__instances.pop(vm_id, default="No instance found")
|
||
|
||
|
||
def delete_vms(vms=None, delete_volumes=True, check_first=True,
|
||
timeout=VMTimeout.DELETE, fail_ok=False,
|
||
stop_first=True, con_ssh=None, auth_info=Tenant.get('admin'),
|
||
remove_cleanup=None):
|
||
"""
|
||
Delete given vm(s) (and attached volume(s)). If None vms given, all vms
|
||
on the system will be deleted.
|
||
|
||
Args:
|
||
vms (list|str): list of vm ids to be deleted. If string input,
|
||
assume only one vm id is provided.
|
||
check_first (bool): Whether to check if given vm(s) exist on system
|
||
before attempt to delete
|
||
timeout (int): Max time to wait for delete cli finish and wait for
|
||
vms actually disappear from system
|
||
delete_volumes (bool): delete attached volume(s) if set to True
|
||
fail_ok (bool):
|
||
stop_first (bool): whether to stop active vm(s) first before
|
||
deleting. Best effort only
|
||
con_ssh (SSHClient):
|
||
auth_info (dict):
|
||
remove_cleanup (None|str): remove from vm cleanup list if deleted
|
||
successfully
|
||
|
||
Returns (tuple): (rtn_code(int), msg(str)) # rtn_code 1,2,3 only returns
|
||
when fail_ok=True
|
||
(-1, 'No vm(s) to delete.') # "Empty vm list/string provided and
|
||
no vm exist on system.
|
||
(-1, 'None of the given vm(s) exists on system.')
|
||
(0, "VM(s) deleted successfully.")
|
||
(1, <stderr>) # delete vm(s) cli returns stderr, some or all vms
|
||
failed to delete.
|
||
(2, "VMs deletion cmd all accepted, but some vms still exist after
|
||
deletion")
|
||
|
||
"""
|
||
existing_vms = None
|
||
if not vms:
|
||
vms = get_vms(con_ssh=con_ssh, auth_info=auth_info, all_projects=True,
|
||
long=False)
|
||
existing_vms = list(vms)
|
||
elif isinstance(vms, str):
|
||
vms = [vms]
|
||
|
||
vms = [vm for vm in vms if vm]
|
||
if not vms:
|
||
LOG.warning(
|
||
"Empty vm list/string provided and no vm exist on system. Do "
|
||
"Nothing")
|
||
return -1, 'No vm(s) to delete.'
|
||
|
||
if check_first:
|
||
if existing_vms is None:
|
||
existing_vms = get_vms(con_ssh=con_ssh, auth_info=auth_info,
|
||
all_projects=True, long=False)
|
||
|
||
vms = list(set(vms) & set(existing_vms))
|
||
if not vms:
|
||
LOG.info("None given vms exist on system. Do nothing")
|
||
return -1, 'None of the given vm(s) exists on system.'
|
||
|
||
if stop_first: # best effort only
|
||
active_vms = get_vms(vms=vms, auth_info=auth_info, con_ssh=con_ssh,
|
||
all_projects=True,
|
||
Status=VMStatus.ACTIVE)
|
||
if active_vms:
|
||
stop_vms(active_vms, fail_ok=True, con_ssh=con_ssh,
|
||
auth_info=auth_info)
|
||
|
||
vols_to_del = []
|
||
if delete_volumes:
|
||
vols_to_del = cinder_helper.get_volumes_attached_to_vms(
|
||
vms=vms, auth_info=auth_info, con_ssh=con_ssh)
|
||
|
||
LOG.info("Deleting vm(s): {}".format(vms))
|
||
vms_accepted = []
|
||
deletion_err = ''
|
||
for vm in vms:
|
||
# Deleting vm one by one due to the cmd will stop if a failure is
|
||
# encountered, causing no attempt to delete
|
||
# other vms
|
||
code, output = cli.openstack('server delete', vm, ssh_client=con_ssh,
|
||
fail_ok=True, auth_info=auth_info,
|
||
timeout=timeout)
|
||
if code > 0:
|
||
deletion_err += '{}\n'.format(output)
|
||
else:
|
||
vms_accepted.append(vm)
|
||
|
||
# check if vms are actually removed from nova list
|
||
all_deleted, vms_undeleted = _wait_for_vms_deleted(vms_accepted,
|
||
fail_ok=True,
|
||
auth_info=auth_info,
|
||
timeout=timeout,
|
||
con_ssh=con_ssh)
|
||
if remove_cleanup:
|
||
vms_deleted = list(set(vms_accepted) - set(vms_undeleted))
|
||
ResourceCleanup.remove('vm', vms_deleted, scope=remove_cleanup,
|
||
del_vm_vols=False)
|
||
|
||
# Delete volumes results will not be returned. Best effort only.
|
||
if delete_volumes:
|
||
res = cinder_helper.delete_volumes(vols_to_del, fail_ok=True,
|
||
auth_info=auth_info,
|
||
con_ssh=con_ssh)[0]
|
||
if res == 0 and remove_cleanup:
|
||
ResourceCleanup.remove('volume', vols_to_del, scope=remove_cleanup)
|
||
|
||
# Process returns
|
||
if deletion_err:
|
||
LOG.warning(deletion_err)
|
||
if fail_ok:
|
||
return 1, deletion_err
|
||
raise exceptions.CLIRejected(deletion_err)
|
||
|
||
if vms_undeleted:
|
||
msg = 'VM(s) still exsit after deletion: {}'.format(vms_undeleted)
|
||
LOG.warning(msg)
|
||
if fail_ok:
|
||
return 2, msg
|
||
raise exceptions.VMPostCheckFailed(msg)
|
||
|
||
LOG.info("VM(s) deleted successfully: {}".format(vms))
|
||
return 0, "VM(s) deleted successfully."
|
||
|
||
|
||
def _wait_for_vms_deleted(vms, timeout=VMTimeout.DELETE, fail_ok=True,
|
||
check_interval=3, con_ssh=None,
|
||
auth_info=Tenant.get('admin')):
|
||
"""
|
||
Wait for specific vm to be removed from nova list
|
||
|
||
Args:
|
||
vms (str|list): list of vms' ids
|
||
timeout (int): in seconds
|
||
fail_ok (bool):
|
||
check_interval (int):
|
||
con_ssh (SSHClient|None):
|
||
auth_info (dict|None):
|
||
|
||
Returns (tuple): (result(bool), vms_failed_to_delete(list))
|
||
|
||
"""
|
||
if isinstance(vms, str):
|
||
vms = [vms]
|
||
|
||
vms_to_check = list(vms)
|
||
end_time = time.time() + timeout
|
||
while time.time() < end_time:
|
||
try:
|
||
vms_to_check = get_vms(vms=vms_to_check, con_ssh=con_ssh,
|
||
auth_info=auth_info)
|
||
except exceptions.CLIRejected:
|
||
pass
|
||
|
||
if not vms_to_check:
|
||
return True, []
|
||
time.sleep(check_interval)
|
||
|
||
if fail_ok:
|
||
return False, vms_to_check
|
||
raise exceptions.VMPostCheckFailed(
|
||
"Some vm(s) are not removed from nova list within {} seconds: {}".
|
||
format(timeout, vms_to_check))
|
||
|
||
|
||
def wait_for_vms_values(vms, header='Status', value=VMStatus.ACTIVE,
|
||
timeout=VMTimeout.STATUS_CHANGE, fail_ok=True,
|
||
check_interval=3, con_ssh=None,
|
||
auth_info=Tenant.get('admin')):
|
||
"""
|
||
Wait for specific vms to reach any of the given state(s) in openstack
|
||
server list
|
||
|
||
Args:
|
||
vms (str|list): id(s) of vms to check
|
||
header (str): target header in nova list
|
||
value (str|list): expected value(s)
|
||
timeout (int): in seconds
|
||
fail_ok (bool):
|
||
check_interval (int):
|
||
con_ssh (SSHClient|None):
|
||
auth_info (dict|None):
|
||
|
||
Returns (list): [result(bool), vms_in_state(dict),
|
||
vms_failed_to_reach_state(dict)]
|
||
|
||
"""
|
||
if isinstance(vms, str):
|
||
vms = [vms]
|
||
|
||
if isinstance(value, str):
|
||
value = [value]
|
||
|
||
res_fail = res_pass = None
|
||
end_time = time.time() + timeout
|
||
while time.time() < end_time:
|
||
res_pass = {}
|
||
res_fail = {}
|
||
vms_values = get_vms(vms=vms, con_ssh=con_ssh, auth_info=auth_info,
|
||
field=header)
|
||
for i in range(len(vms)):
|
||
vm = vms[i]
|
||
vm_value = vms_values[i]
|
||
if vm_value in value:
|
||
res_pass[vm] = vm_value
|
||
else:
|
||
res_fail[vm] = vm_value
|
||
|
||
if not res_fail:
|
||
return True, res_pass, res_fail
|
||
|
||
time.sleep(check_interval)
|
||
|
||
fail_msg = "Some vm(s) did not reach given status from nova list within " \
|
||
"{} seconds: {}".format(timeout, res_fail)
|
||
if fail_ok:
|
||
LOG.warning(fail_msg)
|
||
return False, res_pass, res_fail
|
||
raise exceptions.VMPostCheckFailed(fail_msg)
|
||
|
||
|
||
def set_vm_state(vm_id, check_first=False, error_state=True, fail_ok=False,
|
||
auth_info=Tenant.get('admin'),
|
||
con_ssh=None):
|
||
"""
|
||
Set vm state to error or active via nova reset-state.
|
||
|
||
Args:
|
||
vm_id:
|
||
check_first:
|
||
error_state:
|
||
fail_ok:
|
||
auth_info:
|
||
con_ssh:
|
||
|
||
Returns (tuple):
|
||
|
||
"""
|
||
expt_vm_status = VMStatus.ERROR if error_state else VMStatus.ACTIVE
|
||
LOG.info("Setting vm {} state to: {}".format(vm_id, expt_vm_status))
|
||
|
||
if check_first:
|
||
pre_vm_status = get_vm_values(vm_id, fields='status', con_ssh=con_ssh,
|
||
auth_info=auth_info)[0]
|
||
if pre_vm_status.lower() == expt_vm_status.lower():
|
||
msg = "VM {} already in {} state. Do nothing.".format(vm_id,
|
||
pre_vm_status)
|
||
LOG.info(msg)
|
||
return -1, msg
|
||
|
||
code, out = set_vm(vm_id=vm_id, state=expt_vm_status, con_ssh=con_ssh,
|
||
auth_info=auth_info, fail_ok=fail_ok)
|
||
if code > 0:
|
||
return 1, out
|
||
|
||
result = wait_for_vm_status(vm_id, expt_vm_status, fail_ok=fail_ok)
|
||
if result is None:
|
||
msg = "VM {} did not reach expected state - {} after " \
|
||
"reset-state.".format(vm_id, expt_vm_status)
|
||
LOG.warning(msg)
|
||
return 2, msg
|
||
|
||
msg = "VM state is successfully set to: {}".format(expt_vm_status)
|
||
LOG.info(msg)
|
||
return 0, msg
|
||
|
||
|
||
def reboot_vm(vm_id, hard=False, fail_ok=False, con_ssh=None, auth_info=None,
|
||
cli_timeout=CMDTimeout.REBOOT_VM,
|
||
reboot_timeout=VMTimeout.REBOOT):
|
||
"""
|
||
reboot vm via openstack server reboot
|
||
Args:
|
||
vm_id:
|
||
hard (bool): hard or soft reboot
|
||
fail_ok:
|
||
con_ssh:
|
||
auth_info:
|
||
cli_timeout:
|
||
reboot_timeout:
|
||
|
||
Returns (tuple):
|
||
|
||
"""
|
||
vm_status = get_vm_status(vm_id, con_ssh=con_ssh)
|
||
if not vm_status.lower() == 'active':
|
||
LOG.warning(
|
||
"VM is not in active state before rebooting. VM status: {}".format(
|
||
vm_status))
|
||
|
||
extra_arg = '--hard ' if hard else ''
|
||
arg = "{}{}".format(extra_arg, vm_id)
|
||
|
||
date_format = "%Y%m%d %T"
|
||
start_time = common.get_date_in_format(date_format=date_format)
|
||
code, output = cli.openstack('server reboot', arg, ssh_client=con_ssh,
|
||
fail_ok=fail_ok, auth_info=auth_info,
|
||
timeout=cli_timeout)
|
||
if code > 0:
|
||
return 1, output
|
||
|
||
# expt_reboot = VMStatus.HARD_REBOOT if hard else VMStatus.SOFT_REBOOT
|
||
# _wait_for_vm_status(vm_id, expt_reboot, check_interval=0, fail_ok=False)
|
||
LOG.info("Wait for vm reboot events to appear in fm event-list")
|
||
expt_reason = 'hard-reboot' if hard else 'soft-reboot'
|
||
system_helper.wait_for_events(
|
||
timeout=30, num=10, entity_instance_id=vm_id,
|
||
start=start_time, fail_ok=False, strict=False,
|
||
**{'Event Log ID': EventLogID.REBOOT_VM_ISSUED,
|
||
'Reason Text': expt_reason})
|
||
|
||
system_helper.wait_for_events(
|
||
timeout=reboot_timeout, num=10, entity_instance_id=vm_id,
|
||
start=start_time, fail_ok=False,
|
||
**{'Event Log ID': EventLogID.REBOOT_VM_COMPLETE})
|
||
|
||
LOG.info("Check vm status from nova show")
|
||
actual_status = wait_for_vm_status(vm_id,
|
||
[VMStatus.ACTIVE, VMStatus.ERROR],
|
||
fail_ok=fail_ok, con_ssh=con_ssh,
|
||
timeout=30)
|
||
if not actual_status:
|
||
msg = "VM {} did not reach active state after reboot.".format(vm_id)
|
||
LOG.warning(msg)
|
||
return 2, msg
|
||
|
||
if actual_status.lower() == VMStatus.ERROR.lower():
|
||
msg = "VM is in error state after reboot."
|
||
if fail_ok:
|
||
LOG.warning(msg)
|
||
return 3, msg
|
||
raise exceptions.VMPostCheckFailed(msg)
|
||
|
||
succ_msg = "VM rebooted successfully."
|
||
LOG.info(succ_msg)
|
||
return 0, succ_msg
|
||
|
||
|
||
def __perform_vm_action(vm_id, action, expt_status,
|
||
timeout=VMTimeout.STATUS_CHANGE, fail_ok=False,
|
||
con_ssh=None,
|
||
auth_info=None):
|
||
LOG.info("{} vm {} begins...".format(action, vm_id))
|
||
code, output = cli.nova(action, vm_id, ssh_client=con_ssh, fail_ok=fail_ok,
|
||
auth_info=auth_info, timeout=120)
|
||
|
||
if code == 1:
|
||
return 1, output
|
||
|
||
actual_status = wait_for_vm_status(vm_id, [expt_status, VMStatus.ERROR],
|
||
fail_ok=fail_ok, con_ssh=con_ssh,
|
||
timeout=timeout)
|
||
|
||
if not actual_status:
|
||
msg = "VM {} did not reach expected state {} after {}.".format(
|
||
vm_id, expt_status, action)
|
||
LOG.warning(msg)
|
||
return 2, msg
|
||
|
||
if actual_status.lower() == VMStatus.ERROR.lower():
|
||
msg = "VM is in error state after {}.".format(action)
|
||
if fail_ok:
|
||
LOG.warning(msg)
|
||
return 3, msg
|
||
raise exceptions.VMPostCheckFailed(msg)
|
||
|
||
succ_msg = "{} VM succeeded.".format(action)
|
||
LOG.info(succ_msg)
|
||
return 0, succ_msg
|
||
|
||
|
||
def suspend_vm(vm_id, timeout=VMTimeout.STATUS_CHANGE, fail_ok=False,
|
||
con_ssh=None, auth_info=None):
|
||
return __perform_vm_action(vm_id, 'suspend', VMStatus.SUSPENDED,
|
||
timeout=timeout, fail_ok=fail_ok,
|
||
con_ssh=con_ssh, auth_info=auth_info)
|
||
|
||
|
||
def resume_vm(vm_id, timeout=VMTimeout.STATUS_CHANGE, fail_ok=False,
|
||
con_ssh=None, auth_info=None):
|
||
return __perform_vm_action(vm_id, 'resume', VMStatus.ACTIVE,
|
||
timeout=timeout, fail_ok=fail_ok,
|
||
con_ssh=con_ssh,
|
||
auth_info=auth_info)
|
||
|
||
|
||
def pause_vm(vm_id, timeout=VMTimeout.PAUSE, fail_ok=False, con_ssh=None,
|
||
auth_info=None):
|
||
return __perform_vm_action(vm_id, 'pause', VMStatus.PAUSED, timeout=timeout,
|
||
fail_ok=fail_ok, con_ssh=con_ssh,
|
||
auth_info=auth_info)
|
||
|
||
|
||
def unpause_vm(vm_id, timeout=VMTimeout.STATUS_CHANGE, fail_ok=False,
|
||
con_ssh=None, auth_info=None):
|
||
return __perform_vm_action(vm_id, 'unpause', VMStatus.ACTIVE,
|
||
timeout=timeout, fail_ok=fail_ok,
|
||
con_ssh=con_ssh,
|
||
auth_info=auth_info)
|
||
|
||
|
||
def stop_vms(vms, timeout=VMTimeout.STATUS_CHANGE, fail_ok=False, con_ssh=None,
|
||
auth_info=None):
|
||
return _start_or_stop_vms(vms, 'stop', VMStatus.STOPPED, timeout,
|
||
check_interval=1, fail_ok=fail_ok,
|
||
con_ssh=con_ssh, auth_info=auth_info)
|
||
|
||
|
||
def start_vms(vms, timeout=VMTimeout.STATUS_CHANGE, fail_ok=False, con_ssh=None,
|
||
auth_info=None):
|
||
return _start_or_stop_vms(vms, 'start', VMStatus.ACTIVE, timeout,
|
||
check_interval=1, fail_ok=fail_ok,
|
||
con_ssh=con_ssh, auth_info=auth_info)
|
||
|
||
|
||
def _start_or_stop_vms(vms, action, expt_status,
|
||
timeout=VMTimeout.STATUS_CHANGE, check_interval=3,
|
||
fail_ok=False,
|
||
con_ssh=None, auth_info=None):
|
||
LOG.info("{}ing vms {}...".format(action, vms))
|
||
action = action.lower()
|
||
if isinstance(vms, str):
|
||
vms = [vms]
|
||
|
||
# Not using openstack client due to stop will be aborted at first
|
||
# failure, without continue processing other vms
|
||
code, output = cli.nova(action, ' '.join(vms), ssh_client=con_ssh,
|
||
fail_ok=fail_ok, auth_info=auth_info)
|
||
|
||
vms_to_check = list(vms)
|
||
if code == 1:
|
||
vms_to_check = re.findall(
|
||
NovaCLIOutput.VM_ACTION_ACCEPTED.format(action), output)
|
||
if not vms_to_check:
|
||
return 1, output
|
||
|
||
res_bool, res_pass, res_fail = wait_for_vms_values(
|
||
vms_to_check, 'Status', [expt_status, VMStatus.ERROR],
|
||
fail_ok=fail_ok, check_interval=check_interval, con_ssh=con_ssh,
|
||
timeout=timeout)
|
||
|
||
if not res_bool:
|
||
msg = "Some VM(s) did not reach expected state(s) - {}. Actual " \
|
||
"states: {}".format(expt_status, res_fail)
|
||
LOG.warning(msg)
|
||
return 2, msg
|
||
|
||
error_vms = [vm_id for vm_id in vms_to_check if
|
||
res_pass[vm_id].lower() == VMStatus.ERROR.lower()]
|
||
if error_vms:
|
||
msg = "Some VM(s) in error state after {}: {}".format(action, error_vms)
|
||
if fail_ok:
|
||
LOG.warning(msg)
|
||
return 3, msg
|
||
raise exceptions.VMPostCheckFailed(msg)
|
||
|
||
succ_msg = "Action {} performed successfully on vms.".format(action)
|
||
LOG.info(succ_msg)
|
||
return 0, succ_msg
|
||
|
||
|
||
def rebuild_vm(vm_id, image_id=None, new_name=None, preserve_ephemeral=None,
|
||
fail_ok=False, con_ssh=None,
|
||
auth_info=Tenant.get('admin'), **metadata):
|
||
if image_id is None:
|
||
image_id = glance_helper.get_image_id_from_name(
|
||
GuestImages.DEFAULT['guest'], strict=True)
|
||
|
||
args = '{} {}'.format(vm_id, image_id)
|
||
|
||
if new_name:
|
||
args += ' --name {}'.format(new_name)
|
||
|
||
if preserve_ephemeral:
|
||
args += ' --preserve-ephemeral'
|
||
|
||
for key, value in metadata.items():
|
||
args += ' --meta {}={}'.format(key, value)
|
||
|
||
LOG.info("Rebuilding vm {}".format(vm_id))
|
||
# Some features such as trusted image cert not available with openstack
|
||
# client
|
||
code, output = cli.nova('rebuild', args, ssh_client=con_ssh,
|
||
fail_ok=fail_ok, auth_info=auth_info)
|
||
if code == 1:
|
||
return code, output
|
||
|
||
LOG.info("Check vm status after vm rebuild")
|
||
wait_for_vm_status(vm_id, status=VMStatus.ACTIVE, fail_ok=fail_ok,
|
||
con_ssh=con_ssh)
|
||
actual_status = wait_for_vm_status(vm_id, [VMStatus.ACTIVE, VMStatus.ERROR],
|
||
fail_ok=fail_ok, con_ssh=con_ssh,
|
||
timeout=VMTimeout.REBUILD)
|
||
|
||
if not actual_status:
|
||
msg = "VM {} did not reach active state after rebuild.".format(vm_id)
|
||
LOG.warning(msg)
|
||
return 2, msg
|
||
|
||
if actual_status.lower() == VMStatus.ERROR.lower():
|
||
msg = "VM is in error state after rebuild."
|
||
if fail_ok:
|
||
LOG.warning(msg)
|
||
return 3, msg
|
||
raise exceptions.VMPostCheckFailed(msg)
|
||
|
||
succ_msg = "VM rebuilded successfully."
|
||
LOG.info(succ_msg)
|
||
return 0, succ_msg
|
||
|
||
|
||
def get_vm_numa_nodes_via_ps(vm_id=None, instance_name=None, host=None,
|
||
con_ssh=None, auth_info=Tenant.get('admin'),
|
||
per_vcpu=False):
|
||
"""
|
||
Get numa nodes VM is currently on
|
||
Args:
|
||
vm_id:
|
||
instance_name:
|
||
host:
|
||
con_ssh:
|
||
auth_info:
|
||
per_vcpu (bool): if True, return per vcpu, e.g., if vcpu=0,1,2,
|
||
returned list will have same length [0,1,0]
|
||
|
||
Returns (list): e.g., [0], [0, 1]
|
||
|
||
"""
|
||
if not instance_name or not host:
|
||
if not vm_id:
|
||
raise ValueError('vm_id has to be provided')
|
||
instance_name, host = get_vm_values(vm_id,
|
||
fields=[":instance_name", ":host"],
|
||
strict=False, con_ssh=con_ssh,
|
||
auth_info=auth_info)
|
||
|
||
with host_helper.ssh_to_host(host, con_ssh=con_ssh) as host_ssh:
|
||
vcpu_cpu_map = get_vcpu_cpu_map(instance_names=instance_name,
|
||
host_ssh=host_ssh, con_ssh=con_ssh)[
|
||
instance_name]
|
||
cpus = []
|
||
for i in range(len(vcpu_cpu_map)):
|
||
cpus.append(vcpu_cpu_map[i])
|
||
|
||
cpu_non_dup = sorted(list(set(cpus)))
|
||
grep_str = ' '.join(
|
||
['-e "processor.*: {}$"'.format(cpu) for cpu in cpu_non_dup])
|
||
cmd = 'cat /proc/cpuinfo | grep -A 10 {} | grep --color=never ' \
|
||
'"physical id"'.format(grep_str)
|
||
physical_ids = host_ssh.exec_cmd(cmd, fail_ok=False)[1].splitlines()
|
||
physical_ids = [int(proc.split(sep=':')[-1].strip()) for proc in
|
||
physical_ids if 'physical' in proc]
|
||
if per_vcpu:
|
||
physical_ids = [physical_ids[cpu_non_dup.index(cpu)] for cpu in
|
||
cpus]
|
||
|
||
return physical_ids
|
||
|
||
|
||
def get_vm_host_and_numa_nodes(vm_id, con_ssh=None, per_vcpu=False):
|
||
"""
|
||
Get vm host and numa nodes used for the vm on the host
|
||
Args:
|
||
vm_id (str):
|
||
con_ssh (SSHClient):
|
||
per_vcpu (bool): if True, return numa nodes per vcpu, e.g., vcpu=0,1,
|
||
2, returned list can be: [0,1,0]
|
||
|
||
Returns (tuple): (<vm_hostname> (str), <numa_nodes> (list of integers))
|
||
|
||
"""
|
||
instance_name, host = get_vm_values(vm_id,
|
||
fields=[":instance_name", ":host"],
|
||
strict=False)
|
||
actual_node_vals = get_vm_numa_nodes_via_ps(vm_id=vm_id,
|
||
instance_name=instance_name,
|
||
host=host, con_ssh=con_ssh,
|
||
per_vcpu=per_vcpu)
|
||
|
||
return host, actual_node_vals
|
||
|
||
|
||
def perform_action_on_vm(vm_id, action, auth_info=Tenant.get('admin'),
|
||
con_ssh=None, **kwargs):
|
||
"""
|
||
Perform action on a given vm.
|
||
|
||
Args:
|
||
vm_id (str):
|
||
action (str): action to perform on vm. Valid_actions: 'start',
|
||
'stop', 'suspend', 'resume', 'pause', 'unpause',
|
||
'reboot', 'live_migrate', or 'cold_migrate'
|
||
auth_info (dict):
|
||
con_ssh (SSHClient):
|
||
**kwargs: extra params to pass to action function,
|
||
e.g.destination_host='compute-0' when action is live_migrate
|
||
|
||
Returns (None):
|
||
|
||
"""
|
||
action_function_map = {
|
||
'start': start_vms,
|
||
'stop': stop_vms,
|
||
'suspend': suspend_vm,
|
||
'resume': resume_vm,
|
||
'pause': pause_vm,
|
||
'unpause': unpause_vm,
|
||
'reboot': reboot_vm,
|
||
'rebuild': rebuild_vm,
|
||
'live_migrate': live_migrate_vm,
|
||
'cold_migrate': cold_migrate_vm,
|
||
'cold_mig_revert': cold_migrate_vm,
|
||
}
|
||
if not vm_id:
|
||
raise ValueError("vm id is not provided.")
|
||
|
||
valid_actions = list(action_function_map.keys())
|
||
action = action.lower().replace(' ', '_')
|
||
if action not in valid_actions:
|
||
raise ValueError(
|
||
"Invalid action provided: {}. Valid actions: {}".format(
|
||
action, valid_actions))
|
||
|
||
if action == 'cold_mig_revert':
|
||
kwargs['revert'] = True
|
||
|
||
return action_function_map[action](vm_id, con_ssh=con_ssh,
|
||
auth_info=auth_info, **kwargs)
|
||
|
||
|
||
def get_vm_nics_info(vm_id, network=None, vnic_type=None, rtn_dict=False):
|
||
"""
|
||
Get vm nics info
|
||
Args:
|
||
vm_id:
|
||
network:
|
||
vnic_type:
|
||
rtn_dict:
|
||
|
||
Returns (list of dict|dict of dict):
|
||
list or dict (port as key) of port_info_dict. Each port_info_dict
|
||
contains following info:
|
||
{
|
||
'port_id': <port_id>,
|
||
'network': <net_name>,
|
||
'network_id': <net_id>,
|
||
'vnic_type': <vnic_type>,
|
||
'mac_address': <mac_address>,
|
||
'subnet_id': <subnet_id>,
|
||
'subnet_cidr': <subnet_cidr>
|
||
}
|
||
|
||
"""
|
||
vm_ports, vm_macs, vm_ips_info = network_helper.get_ports(
|
||
server=vm_id, network=network,
|
||
field=('ID', 'MAC Address', 'Fixed IP Addresses'))
|
||
vm_subnets = []
|
||
vm_ips = []
|
||
for ip_info in vm_ips_info:
|
||
ip_info = ip_info[0]
|
||
vm_ips.append(ip_info.get('ip_address'))
|
||
vm_subnets.append(ip_info.get('subnet_id'))
|
||
|
||
indexes = list(range(len(vm_ports)))
|
||
vnic_types = []
|
||
vm_net_ids = []
|
||
for port in vm_ports:
|
||
port_vnic_type, port_net_id = network_helper.get_port_values(
|
||
port=port, fields=('binding_vnic_type', 'network_id'))
|
||
vnic_types.append(port_vnic_type)
|
||
vm_net_ids.append(port_net_id)
|
||
if vnic_type and vnic_type != port_vnic_type:
|
||
indexes.remove(list(vm_ports).index(port))
|
||
|
||
vm_net_names = []
|
||
ids_, names_, = network_helper.get_networks(field=('ID', 'Name'),
|
||
strict=False)
|
||
for net_id in vm_net_ids:
|
||
vm_net_names.append(names_[ids_.index(net_id)])
|
||
|
||
res_dict = {}
|
||
res = []
|
||
for i in indexes:
|
||
port_dict = {
|
||
'port_id': vm_ports[i],
|
||
'network': vm_net_names[i],
|
||
'network_id': vm_net_ids[i],
|
||
'vnic_type': vnic_types[i],
|
||
'mac_address': vm_macs[i],
|
||
'ip_address': vm_ips[i]
|
||
}
|
||
if rtn_dict:
|
||
res_dict[vm_ports[i]] = port_dict
|
||
else:
|
||
res.append(port_dict)
|
||
|
||
return res_dict if rtn_dict else res
|
||
|
||
|
||
def get_vm_interfaces_via_virsh(vm_id, con_ssh=None):
|
||
"""
|
||
|
||
Args:
|
||
vm_id:
|
||
con_ssh:
|
||
|
||
Returns (list of tuple):
|
||
[(mac_0, vif_model_0)...]
|
||
|
||
"""
|
||
vm_host = get_vm_host(vm_id=vm_id, con_ssh=con_ssh)
|
||
inst_name = get_vm_instance_name(vm_id=vm_id, con_ssh=con_ssh)
|
||
|
||
vm_ifs = []
|
||
with host_helper.ssh_to_host(vm_host, con_ssh=con_ssh) as host_ssh:
|
||
output = host_ssh.exec_sudo_cmd('virsh domiflist {}'.format(inst_name),
|
||
fail_ok=False)[1]
|
||
if_lines = output.split('-------------------------------\n', 1)[
|
||
-1].splitlines()
|
||
for line in if_lines:
|
||
if not line.strip():
|
||
continue
|
||
|
||
interface, type_, source, model, mac = line.split()
|
||
vm_ifs.append((mac, model))
|
||
|
||
return vm_ifs
|
||
|
||
|
||
def add_vlan_for_vm_pcipt_interfaces(vm_id, net_seg_id, retry=3,
|
||
init_conf=False):
|
||
"""
|
||
Add vlan for vm pci-passthrough interface and restart networking service.
|
||
Do nothing if expected vlan interface already exists in 'ip addr'.
|
||
|
||
Args:
|
||
vm_id (str):
|
||
net_seg_id (int|str|dict): such as 1792
|
||
retry (int): max number of times to reboot vm to try to recover it
|
||
from non-exit
|
||
init_conf (bool): To workaround upstream bug where mac changes after
|
||
migrate or resize https://bugs.launchpad.net/nova/+bug/1617429
|
||
|
||
Returns: None
|
||
|
||
Raises: VMNetworkError if vlan interface is not found in 'ip addr' after
|
||
adding
|
||
|
||
Notes:
|
||
Sometimes a non-exist 'rename6' interface will be used for
|
||
pci-passthrough nic after vm maintenance
|
||
Sudo reboot from the vm as workaround.
|
||
By default will try to reboot for a maximum of 3 times
|
||
|
||
"""
|
||
if not vm_id or not net_seg_id:
|
||
raise ValueError("vm_id and/or net_seg_id not provided.")
|
||
|
||
net_seg_id_dict = None
|
||
if isinstance(net_seg_id, dict):
|
||
net_seg_id_dict = net_seg_id
|
||
net_seg_id = None
|
||
|
||
for i in range(retry):
|
||
vm_pcipt_nics = get_vm_nics_info(vm_id, vnic_type='direct-physical')
|
||
|
||
if not vm_pcipt_nics:
|
||
LOG.warning("No pci-passthrough device found for vm from nova "
|
||
"show {}".format(vm_id))
|
||
return
|
||
|
||
with ssh_to_vm_from_natbox(vm_id=vm_id) as vm_ssh:
|
||
for pcipt_nic in vm_pcipt_nics:
|
||
|
||
mac_addr = pcipt_nic['mac_address']
|
||
eth_name = network_helper.get_eth_for_mac(mac_addr=mac_addr,
|
||
ssh_client=vm_ssh)
|
||
if not eth_name:
|
||
if not init_conf:
|
||
LOG.warning(
|
||
"Interface with mac {} is not listed in 'ip addr' "
|
||
"in vm {}".format(mac_addr, vm_id))
|
||
LOG.info("Try to get first eth with mac 90:...")
|
||
eth_name = network_helper.get_eth_for_mac(
|
||
mac_addr="link/ether 90:", ssh_client=vm_ssh)
|
||
if not eth_name:
|
||
exceptions.VMNetworkError(
|
||
"No Mac starts with 90: in ip addr for vm "
|
||
"{}".format(vm_id))
|
||
else:
|
||
raise exceptions.VMNetworkError(
|
||
"Interface with mac {} is not listed in 'ip addr' "
|
||
"in vm {}".format(mac_addr, vm_id))
|
||
|
||
if 'rename' in eth_name:
|
||
LOG.warning(
|
||
"Retry {}: non-existing interface {} found on "
|
||
"pci-passthrough nic in vm {}, "
|
||
"reboot vm to try to recover".format(
|
||
i + 1, eth_name, vm_id))
|
||
sudo_reboot_from_vm(vm_id=vm_id, vm_ssh=vm_ssh)
|
||
wait_for_vm_pingable_from_natbox(vm_id)
|
||
break
|
||
|
||
else:
|
||
if net_seg_id_dict:
|
||
net_name = pcipt_nic['network']
|
||
net_seg_id = net_seg_id_dict[net_name]
|
||
LOG.info(
|
||
"Seg id for {}: {}".format(net_name, net_seg_id))
|
||
|
||
vlan_name = "{}.{}".format(eth_name, net_seg_id)
|
||
|
||
output_pre_ipaddr = \
|
||
vm_ssh.exec_cmd('ip addr', fail_ok=False)[1]
|
||
if vlan_name in output_pre_ipaddr:
|
||
LOG.info("{} already in ip addr. Skip.".format(
|
||
vlan_name))
|
||
continue
|
||
|
||
# Bring up pcipt interface and assign IP manually.
|
||
# Upstream bug causes dev name and MAC addr
|
||
# change after reboot,migrate, making it impossible to
|
||
# use DHCP or configure permanant static IP.
|
||
# https://bugs.launchpad.net/nova/+bug/1617429
|
||
wait_for_interfaces_up(vm_ssh, eth_name, set_up=True)
|
||
# 'ip link add' works for all linux guests but it does
|
||
# not persists after network service restart
|
||
vm_ssh.exec_cmd(
|
||
'ip link add link {} name {} type vlan id {}'.format(
|
||
eth_name, vlan_name,
|
||
net_seg_id))
|
||
vm_ssh.exec_cmd('ip link set {} up'.format(vlan_name))
|
||
vnic_ip = pcipt_nic['ip_address']
|
||
vm_ssh.exec_cmd(
|
||
'ip addr add {}/24 dev {}'.format(vnic_ip, vlan_name))
|
||
|
||
LOG.info(
|
||
"Check if vlan is added successfully with IP assigned")
|
||
output_post_ipaddr = \
|
||
vm_ssh.exec_cmd('ip addr', fail_ok=False)[1]
|
||
if vlan_name not in output_post_ipaddr:
|
||
raise exceptions.VMNetworkError(
|
||
"{} is not found in 'ip addr' after adding vlan "
|
||
"interface".
|
||
format(vlan_name))
|
||
time.sleep(5)
|
||
if not is_ip_assigned(vm_ssh, eth_name=vlan_name):
|
||
msg = 'No IP assigned to {} vlan interface for VM ' \
|
||
'{}'.format(vlan_name, vm_id)
|
||
LOG.warning(msg)
|
||
raise exceptions.VMNetworkError(msg)
|
||
else:
|
||
LOG.info(
|
||
"vlan {} is successfully added and an IP is "
|
||
"assigned.".format(vlan_name))
|
||
else:
|
||
# did not break, meaning no 'rename' interface detected,
|
||
# vlan either existed or successfully added
|
||
return
|
||
|
||
# 'for' loop break which means 'rename' interface detected,
|
||
# and vm reboot triggered - known issue with wrl
|
||
LOG.info("Reboot vm completed. Retry started.")
|
||
|
||
else:
|
||
raise exceptions.VMNetworkError(
|
||
"'rename' interface still exists in pci-passthrough vm {} with {} "
|
||
"reboot attempts.".format(vm_id, retry))
|
||
|
||
|
||
def is_ip_assigned(vm_ssh, eth_name):
|
||
output = vm_ssh.exec_cmd('ip addr show {}'.format(eth_name),
|
||
fail_ok=False)[1]
|
||
return re.search('inet {}'.format(Networks.IPV4_IP), output)
|
||
|
||
|
||
def wait_for_interfaces_up(vm_ssh, eth_names, check_interval=10, timeout=180,
|
||
set_up=False):
|
||
LOG.info(
|
||
"Waiting for vm interface(s) to be in UP state: {}".format(eth_names))
|
||
end_time = time.time() + timeout
|
||
if isinstance(eth_names, str):
|
||
eth_names = [eth_names]
|
||
ifs_to_check = list(eth_names)
|
||
while time.time() < end_time:
|
||
for eth in ifs_to_check:
|
||
output = \
|
||
vm_ssh.exec_cmd('ip -d link show {}'.format(eth),
|
||
fail_ok=False)[1]
|
||
if 'state UP' in output:
|
||
ifs_to_check.remove(eth)
|
||
continue
|
||
else:
|
||
if set_up:
|
||
vm_ssh.exec_cmd('ip link set {} up'.format(eth))
|
||
LOG.info(
|
||
"{} is not up - wait for {} seconds and check again".format(
|
||
eth, check_interval))
|
||
break
|
||
|
||
if not ifs_to_check:
|
||
LOG.info('interfaces are up: {}'.format(eth_names))
|
||
return
|
||
|
||
time.sleep(check_interval)
|
||
|
||
raise exceptions.VMNetworkError("Interface(s) not up for given vm")
|
||
|
||
|
||
def sudo_reboot_from_vm(vm_id, vm_ssh=None, check_host_unchanged=True,
|
||
con_ssh=None):
|
||
pre_vm_host = None
|
||
if check_host_unchanged:
|
||
pre_vm_host = get_vm_host(vm_id, con_ssh=con_ssh)
|
||
|
||
LOG.info("Initiate sudo reboot from vm")
|
||
|
||
def _sudo_reboot(vm_ssh_):
|
||
extra_prompt = 'Broken pipe'
|
||
output = vm_ssh_.exec_sudo_cmd('reboot -f', get_exit_code=False,
|
||
extra_prompt=extra_prompt)[1]
|
||
expt_string = 'The system is going down for reboot|Broken pipe'
|
||
if re.search(expt_string, output):
|
||
# Sometimes system rebooting msg will be displayed right after
|
||
# reboot cmd sent
|
||
vm_ssh_.parent.flush()
|
||
return
|
||
|
||
try:
|
||
time.sleep(10)
|
||
vm_ssh_.send('')
|
||
index = vm_ssh_.expect([expt_string, vm_ssh_.prompt], timeout=60)
|
||
if index == 1:
|
||
raise exceptions.VMOperationFailed("Unable to reboot vm {}")
|
||
vm_ssh_.parent.flush()
|
||
except pexpect.TIMEOUT:
|
||
vm_ssh_.send_control('c')
|
||
vm_ssh_.expect()
|
||
raise
|
||
|
||
if not vm_ssh:
|
||
with ssh_to_vm_from_natbox(vm_id) as vm_ssh:
|
||
_sudo_reboot(vm_ssh)
|
||
else:
|
||
_sudo_reboot(vm_ssh)
|
||
|
||
LOG.info(
|
||
"sudo vm reboot initiated - wait for reboot completes and VM reaches "
|
||
"active state")
|
||
system_helper.wait_for_events(VMTimeout.AUTO_RECOVERY, strict=False,
|
||
fail_ok=False, con_ssh=con_ssh,
|
||
**{'Entity Instance ID': vm_id,
|
||
'Event Log ID':
|
||
EventLogID.REBOOT_VM_COMPLETE})
|
||
wait_for_vm_status(vm_id, status=VMStatus.ACTIVE, fail_ok=False,
|
||
con_ssh=con_ssh)
|
||
|
||
if check_host_unchanged:
|
||
post_vm_host = get_vm_host(vm_id, con_ssh=con_ssh)
|
||
if not pre_vm_host == post_vm_host:
|
||
raise exceptions.HostError(
|
||
"VM host changed from {} to {} after sudo reboot vm".format(
|
||
pre_vm_host, post_vm_host))
|
||
|
||
|
||
def get_proc_nums_from_vm(vm_ssh):
|
||
total_cores = common.parse_cpus_list(
|
||
vm_ssh.exec_cmd('cat /sys/devices/system/cpu/present', fail_ok=False)[
|
||
1])
|
||
online_cores = common.parse_cpus_list(
|
||
vm_ssh.exec_cmd('cat /sys/devices/system/cpu/online', fail_ok=False)[1])
|
||
offline_cores = common.parse_cpus_list(
|
||
vm_ssh.exec_cmd('cat /sys/devices/system/cpu/offline', fail_ok=False)[
|
||
1])
|
||
|
||
return total_cores, online_cores, offline_cores
|
||
|
||
|
||
def get_instance_names_via_virsh(host_ssh):
|
||
"""
|
||
Get instance names via virsh list on given host
|
||
Args:
|
||
host_ssh:
|
||
|
||
Returns (list):
|
||
|
||
"""
|
||
inst_names = host_ssh.exec_sudo_cmd(
|
||
"virsh list | grep instance- | awk {{'print $2'}}",
|
||
get_exit_code=False)[1]
|
||
return [name.strip() for name in inst_names.splitlines()]
|
||
|
||
|
||
def get_vcpu_cpu_map(instance_names=None, host_ssh=None, host=None,
|
||
con_ssh=None):
|
||
"""
|
||
Get vm(s) vcpu cpu map on given host
|
||
Args:
|
||
instance_names (str|tuple|list|None):
|
||
host_ssh (SSHClient|None):
|
||
host (str|None):
|
||
con_ssh:
|
||
|
||
Returns (dict): {<instance_name>: {0: <host_log_core0>,
|
||
1: <host_log_core1>, ...}, ...}
|
||
|
||
"""
|
||
if not host and not host_ssh:
|
||
raise ValueError('host or host_ssh has to be specified')
|
||
|
||
extra_grep = ''
|
||
if instance_names:
|
||
if isinstance(instance_names, str):
|
||
instance_names = (instance_names,)
|
||
extra_grep = '|grep -E "{}"'.format('|'.join(instance_names))
|
||
cmd = 'ps-sched.sh|grep qemu{}|grep " CPU" '.format(extra_grep) + \
|
||
"""| awk '{{print $10" "$12" "$15 ;}}'"""
|
||
|
||
if host_ssh:
|
||
output = host_ssh.exec_cmd(cmd)[1]
|
||
else:
|
||
with host_helper.ssh_to_host(host, con_ssh=con_ssh) as host_ssh:
|
||
output = host_ssh.exec_cmd(cmd)[1]
|
||
vcpu_cpu_map = {}
|
||
for line in output.splitlines():
|
||
cpu, vcpu, instance_name = line.split()
|
||
instance_name = instance_name.split(sep=',')[0].split(sep='=')[1]
|
||
if instance_name not in vcpu_cpu_map:
|
||
vcpu_cpu_map[instance_name] = {}
|
||
vcpu_cpu_map[instance_name][int(vcpu.split(sep='/')[0])] = int(cpu)
|
||
return vcpu_cpu_map
|
||
|
||
|
||
def get_affined_cpus_for_vm(vm_id, host_ssh=None, vm_host=None,
|
||
instance_name=None, con_ssh=None):
|
||
"""
|
||
cpu affinity list for vm via taskset -pc
|
||
Args:
|
||
vm_id (str):
|
||
host_ssh
|
||
vm_host
|
||
instance_name
|
||
con_ssh (SSHClient):
|
||
|
||
Returns (list): such as [10, 30]
|
||
|
||
"""
|
||
cmd = "ps-sched.sh|grep qemu|grep {}|grep -v grep|awk '{{print $2;}}'" + \
|
||
'|xargs -i /bin/sh -c "taskset -pc {{}}"'
|
||
|
||
if host_ssh:
|
||
if not vm_host or not instance_name:
|
||
raise ValueError(
|
||
"vm_host and instance_name have to be provided together with "
|
||
"host_ssh")
|
||
|
||
output = host_ssh.exec_cmd(cmd.format(instance_name))[1]
|
||
|
||
else:
|
||
vm_host = get_vm_host(vm_id, con_ssh=con_ssh)
|
||
instance_name = get_vm_instance_name(vm_id, con_ssh=con_ssh)
|
||
|
||
with host_helper.ssh_to_host(vm_host, con_ssh=con_ssh) as host_ssh:
|
||
output = host_ssh.exec_cmd(cmd.format(instance_name))[1]
|
||
|
||
# Sample output:
|
||
# pid 6376's current affinity list: 10
|
||
# pid 6380's current affinity list: 10
|
||
# pid 6439's current affinity list: 10
|
||
# pid 6441's current affinity list: 10
|
||
# pid 6442's current affinity list: 30
|
||
# pid 6445's current affinity list: 10
|
||
# pid 24142's current affinity list: 10
|
||
|
||
all_cpus = []
|
||
lines = output.splitlines()
|
||
for line in lines:
|
||
|
||
# skip line if below output occurs due to timing in executing cmds
|
||
# taskset: failed to get pid 17125's affinity: No such process
|
||
if "No such process" in line:
|
||
continue
|
||
|
||
cpu_str = line.split(sep=': ')[-1].strip()
|
||
cpus = common.parse_cpus_list(cpus=cpu_str)
|
||
all_cpus += cpus
|
||
|
||
all_cpus = sorted(list(set(all_cpus)))
|
||
LOG.info("Affined cpus on host {} for vm {}: {}".format(vm_host, vm_id,
|
||
all_cpus))
|
||
|
||
return all_cpus
|
||
|
||
|
||
def _scp_net_config_cloud_init(guest_os):
|
||
con_ssh = get_cli_client()
|
||
dest_dir = '{}/userdata'.format(ProjVar.get_var('USER_FILE_DIR'))
|
||
|
||
if 'ubuntu' in guest_os:
|
||
dest_name = 'ubuntu_cloud_init_if_conf.sh'
|
||
elif 'centos' in guest_os:
|
||
dest_name = 'centos_cloud_init_if_conf.sh'
|
||
else:
|
||
raise ValueError("Unknown guest_os")
|
||
|
||
dest_path = '{}/{}'.format(dest_dir, dest_name)
|
||
|
||
if con_ssh.file_exists(file_path=dest_path):
|
||
LOG.info('userdata {} already exists. Return existing path'.format(
|
||
dest_path))
|
||
return dest_path
|
||
|
||
LOG.debug('Create userdata directory if not already exists')
|
||
cmd = 'mkdir -p {}'.format(dest_dir)
|
||
con_ssh.exec_cmd(cmd, fail_ok=False)
|
||
|
||
# LOG.info('wget image from {} to {}/{}'.format(img_url, img_dest,
|
||
# new_name))
|
||
# cmd = 'wget {} --no-check-certificate -P {} -O {}'.format(img_url,
|
||
# img_dest, new_name)
|
||
# con_ssh.exec_cmd(cmd, expect_timeout=7200, fail_ok=False)
|
||
|
||
source_path = '{}/userdata/{}'.format(TestFileServer.HOME, dest_name)
|
||
LOG.info('scp image from test server to active controller')
|
||
|
||
scp_cmd = 'scp -oStrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null' \
|
||
' {}@{}:{} {}'.format(TestFileServer.USER, TestFileServer.SERVER,
|
||
source_path, dest_dir)
|
||
|
||
con_ssh.send(scp_cmd)
|
||
index = con_ssh.expect(
|
||
[con_ssh.prompt, Prompt.PASSWORD_PROMPT, Prompt.ADD_HOST], timeout=3600)
|
||
if index == 2:
|
||
con_ssh.send('yes')
|
||
index = con_ssh.expect([con_ssh.prompt, Prompt.PASSWORD_PROMPT],
|
||
timeout=3600)
|
||
if index == 1:
|
||
con_ssh.send(TestFileServer.PASSWORD)
|
||
index = con_ssh.expect()
|
||
if index != 0:
|
||
raise exceptions.SSHException("Failed to scp files")
|
||
|
||
return dest_dir
|
||
|
||
|
||
def _create_cloud_init_if_conf(guest_os, nics_num):
|
||
"""
|
||
|
||
Args:
|
||
guest_os:
|
||
nics_num:
|
||
|
||
Returns (str|None): file path of the cloud init userdata file for given
|
||
guest os and number of nics
|
||
Sample file content for Centos vm:
|
||
#!/bin/bash
|
||
sudo cp /etc/sysconfig/network-scripts/ifcfg-eth0
|
||
/etc/sysconfig/network-scripts/ifcfg-eth1
|
||
sudo sed -i 's/eth0/eth1/g'
|
||
/etc/sysconfig/network-scripts/ifcfg-eth1
|
||
sudo ifup eth1
|
||
|
||
Sample file content for Ubuntu vm:
|
||
|
||
"""
|
||
|
||
file_dir = '{}/userdata'.format(ProjVar.get_var('USER_FILE_DIR'))
|
||
guest_os = guest_os.lower()
|
||
|
||
# default eth_path for non-ubuntu image
|
||
eth_path = VMPath.ETH_PATH_CENTOS
|
||
new_user = None
|
||
|
||
if 'ubuntu' in guest_os or 'trusty_uefi' in guest_os:
|
||
guest_os = 'ubuntu'
|
||
# vm_if_path = VMPath.VM_IF_PATH_UBUNTU
|
||
eth_path = VMPath.ETH_PATH_UBUNTU
|
||
new_user = 'ubuntu'
|
||
elif 'centos' in guest_os:
|
||
# vm_if_path = VMPath.VM_IF_PATH_CENTOS
|
||
new_user = 'centos'
|
||
|
||
file_name = '{}_{}nic_cloud_init_if_conf.sh'.format(guest_os, nics_num)
|
||
|
||
file_path = file_dir + file_name
|
||
con_ssh = get_cli_client()
|
||
if con_ssh.file_exists(file_path=file_path):
|
||
LOG.info('userdata {} already exists. Return existing path'.format(
|
||
file_path))
|
||
return file_path
|
||
|
||
LOG.info('Create userdata directory if not already exists')
|
||
cmd = 'mkdir -p {}'.format(file_dir)
|
||
con_ssh.exec_cmd(cmd, fail_ok=False)
|
||
|
||
tmp_dir = '{}/userdata'.format(ProjVar.get_var('TEMP_DIR'))
|
||
os.makedirs(tmp_dir, exist_ok=True)
|
||
tmp_file = tmp_dir + file_name
|
||
|
||
# No longer need to specify bash using cloud-config
|
||
# if 'centos_7' in guest_os:
|
||
# shell = '/usr/bin/bash'
|
||
# else:
|
||
# shell = '/bin/bash'
|
||
|
||
with open(tmp_file, mode='a') as f:
|
||
f.write("#cloud-config\n")
|
||
|
||
if new_user is not None:
|
||
f.write("user: {}\n"
|
||
"password: {}\n"
|
||
"chpasswd: {{ expire: False}}\n"
|
||
"ssh_pwauth: True\n\n".format(new_user, new_user))
|
||
|
||
if eth_path is not None:
|
||
eth0_path = eth_path.format('eth0')
|
||
f.write("runcmd:\n")
|
||
# f.write(" - echo '#!{}'\n".format(shell))
|
||
for i in range(nics_num - 1):
|
||
ethi_name = 'eth{}'.format(i + 1)
|
||
ethi_path = eth_path.format(ethi_name)
|
||
f.write(' - cp {} {}\n'.format(eth0_path, ethi_path))
|
||
f.write(
|
||
" - sed -i 's/eth0/{}/g' {}\n".format(ethi_name, ethi_path))
|
||
f.write(' - ifup {}\n'.format(ethi_name))
|
||
|
||
if not ProjVar.get_var('REMOTE_CLI'):
|
||
common.scp_from_localhost_to_active_controller(source_path=tmp_file,
|
||
dest_path=file_path,
|
||
is_dir=False)
|
||
|
||
LOG.info("Userdata file created: {}".format(file_path))
|
||
return file_path
|
||
|
||
|
||
def _get_cloud_config_add_user(con_ssh=None):
|
||
"""
|
||
copy the cloud-config userdata to STX server.
|
||
This userdata adds stx/li69nux user to guest
|
||
|
||
Args:
|
||
con_ssh (SSHClient):
|
||
|
||
Returns (str): STX filepath of the userdata
|
||
|
||
"""
|
||
file_dir = ProjVar.get_var('USER_FILE_DIR')
|
||
file_name = UserData.ADDUSER_TO_GUEST
|
||
file_path = file_dir + file_name
|
||
|
||
if con_ssh is None:
|
||
con_ssh = get_cli_client()
|
||
if con_ssh.file_exists(file_path=file_path):
|
||
LOG.info('userdata {} already exists. Return existing path'.format(
|
||
file_path))
|
||
return file_path
|
||
|
||
source_file = TestServerPath.USER_DATA + file_name
|
||
dest_path = common.scp_from_test_server_to_user_file_dir(
|
||
source_path=source_file, dest_dir=file_dir,
|
||
dest_name=file_name, con_ssh=con_ssh)
|
||
if dest_path is None:
|
||
raise exceptions.CommonError(
|
||
"userdata file {} does not exist after download".format(dest_path))
|
||
|
||
return dest_path
|
||
|
||
|
||
def boost_vm_cpu_usage(vm_id, end_event, new_dd_events=None, dd_event=None,
|
||
timeout=1200, con_ssh=None):
|
||
"""
|
||
Boost cpu usage on given number of cpu cores on specified vm using dd cmd
|
||
on a new thread
|
||
|
||
Args:
|
||
vm_id (str):
|
||
end_event (Events): Event for kill the dd processes
|
||
new_dd_events (list|Events): list of Event(s) for adding new dd
|
||
process(es)
|
||
dd_event (Events): Event to set after sending first dd cmd.
|
||
timeout: Max time to wait for the end_event to be set before killing dd.
|
||
con_ssh
|
||
|
||
Returns: thread
|
||
|
||
Examples:
|
||
LOG.tc_step("Boost VM cpu usage")
|
||
|
||
|
||
"""
|
||
if not new_dd_events:
|
||
new_dd_events = []
|
||
elif not isinstance(new_dd_events, list):
|
||
new_dd_events = [new_dd_events]
|
||
|
||
def _boost_cpu_in_vm():
|
||
LOG.info("Boosting cpu usage for vm {} using 'dd'".format(vm_id))
|
||
dd_cmd = 'dd if=/dev/zero of=/dev/null &'
|
||
kill_dd = 'pkill -ex dd'
|
||
|
||
with ssh_to_vm_from_natbox(vm_id, con_ssh=con_ssh, timeout=120,
|
||
auth_info=None) as vm_ssh:
|
||
LOG.info("Start first 2 dd processes in vm")
|
||
vm_ssh.exec_cmd(cmd=dd_cmd)
|
||
vm_ssh.exec_cmd(cmd=dd_cmd)
|
||
if dd_event:
|
||
dd_event.set()
|
||
|
||
end_time = time.time() + timeout
|
||
while time.time() < end_time:
|
||
if end_event.is_set():
|
||
LOG.info("End event set, kill dd processes in vm")
|
||
vm_ssh.flush()
|
||
vm_ssh.exec_cmd(kill_dd, get_exit_code=False)
|
||
return
|
||
|
||
for event in new_dd_events:
|
||
if event.is_set():
|
||
LOG.info(
|
||
"New dd event set, start 2 new dd processes in vm")
|
||
vm_ssh.exec_cmd(cmd=dd_cmd)
|
||
vm_ssh.exec_cmd(cmd=dd_cmd)
|
||
new_dd_events.remove(event)
|
||
break
|
||
|
||
time.sleep(3)
|
||
|
||
LOG.error(
|
||
"End event is not set within timeout - {}s, kill dd "
|
||
"anyways".format(
|
||
timeout))
|
||
vm_ssh.exec_cmd(kill_dd)
|
||
|
||
LOG.info(
|
||
"Creating new thread to spike cpu_usage on vm cores for vm {}".format(
|
||
vm_id))
|
||
thread = multi_thread.MThread(_boost_cpu_in_vm)
|
||
thread.start_thread(timeout=timeout + 10)
|
||
|
||
return thread
|
||
|
||
|
||
def write_in_vm(vm_id, end_event, start_event=None, expect_timeout=120,
|
||
thread_timeout=None, write_interval=5,
|
||
con_ssh=None):
|
||
"""
|
||
Continue to write in vm using dd
|
||
|
||
Args:
|
||
vm_id (str):
|
||
start_event (Events): set this event when write in vm starts
|
||
end_event (Events): if this event is set, end write right away
|
||
expect_timeout (int):
|
||
thread_timeout (int):
|
||
write_interval (int): how frequent to write. Note: 5 seconds seem to
|
||
be a good interval,
|
||
1 second interval might have noticeable impact on the performance
|
||
of pexpect.
|
||
con_ssh (SSHClient): controller ssh client
|
||
|
||
Returns (MThread): new_thread
|
||
|
||
"""
|
||
if not start_event:
|
||
start_event = Events("Write in vm {} start".format(vm_id))
|
||
write_cmd = "while (true) do date; dd if=/dev/urandom of=output.txt " \
|
||
"bs=1k count=1 conv=fsync || break; echo ; " \
|
||
"sleep {}; done 2>&1 | tee trace.txt".format(write_interval)
|
||
|
||
def _keep_writing(vm_id_):
|
||
LOG.info("starting to write to vm using dd...")
|
||
with ssh_to_vm_from_natbox(vm_id_, con_ssh=con_ssh,
|
||
close_ssh=False) as vm_ssh_:
|
||
vm_ssh_.send(cmd=write_cmd)
|
||
|
||
start_event.set()
|
||
LOG.info("Write_in_vm started")
|
||
|
||
LOG.info("Reading the dd output from vm {}".format(vm_id))
|
||
thread.res = True
|
||
try:
|
||
while True:
|
||
expt_output = '1024 bytes'
|
||
index = vm_ssh_.expect([expt_output, vm_ssh_.prompt],
|
||
timeout=expect_timeout, fail_ok=True,
|
||
searchwindowsize=100)
|
||
if index != 0:
|
||
LOG.warning(
|
||
"write has stopped or expected output-'{}' is not "
|
||
"found".format(
|
||
expt_output))
|
||
thread.res = False
|
||
break
|
||
|
||
if end_event.is_set():
|
||
LOG.info("End thread now")
|
||
break
|
||
|
||
LOG.info("Writing in vm continues...")
|
||
time.sleep(write_interval)
|
||
|
||
finally:
|
||
vm_ssh_.send_control('c')
|
||
|
||
return vm_ssh_
|
||
|
||
thread = multi_thread.MThread(_keep_writing, vm_id)
|
||
thread_timeout = expect_timeout + 30 if thread_timeout is None else \
|
||
thread_timeout
|
||
thread.start_thread(timeout=thread_timeout)
|
||
|
||
start_event.wait_for_event(timeout=thread_timeout)
|
||
|
||
return thread
|
||
|
||
|
||
def attach_interface(vm_id, port_id=None, net_id=None, fixed_ip=None,
|
||
fail_ok=False, auth_info=None,
|
||
con_ssh=None):
|
||
"""
|
||
Attach interface to a vm via port_id OR net_id
|
||
Args:
|
||
vm_id (str):
|
||
port_id (str): port to attach to vm
|
||
net_id (str): port from given net to attach to vm
|
||
fixed_ip (str): fixed ip for attached interface. Only works when
|
||
attaching interface via net_id
|
||
fail_ok (bool):
|
||
auth_info (dict):
|
||
con_ssh (SSHClient):
|
||
|
||
Returns (tuple): (<return_code>, <attached_port_id>)
|
||
(0, <port_id_attached>)
|
||
(1, <std_err>) - cli rejected
|
||
(2, "Post interface attach check failed: <reasons>") -
|
||
net_id/port_id, vif_model, or fixed_ip do not match
|
||
with
|
||
given value
|
||
|
||
"""
|
||
LOG.info("Attaching interface to VM {}".format(vm_id))
|
||
if not vm_id:
|
||
raise ValueError('vm_id is not supplied')
|
||
|
||
args = ''
|
||
args_dict = {
|
||
'--port-id': port_id,
|
||
'--net-id': net_id,
|
||
'--fixed-ip': fixed_ip,
|
||
}
|
||
|
||
for key, val in args_dict.items():
|
||
if val is not None:
|
||
args += ' {} {}'.format(key, val)
|
||
|
||
args += ' {}'.format(vm_id)
|
||
|
||
prev_ports = network_helper.get_ports(server=vm_id, auth_info=auth_info,
|
||
con_ssh=con_ssh)
|
||
# Not switching to openstack client due to nova cli makes more sense.
|
||
# openstack client has separate cmds for adding
|
||
# port, network and fixed ip, while fixed ip cmd has to specify the network.
|
||
code, output = cli.nova('interface-attach', args, ssh_client=con_ssh,
|
||
fail_ok=fail_ok, auth_info=auth_info)
|
||
|
||
if code == 1:
|
||
return code, output
|
||
|
||
LOG.info("Post interface-attach checks started...")
|
||
post_ports = network_helper.get_ports(server=vm_id, auth_info=auth_info,
|
||
con_ssh=con_ssh)
|
||
attached_port = list(set(post_ports) - set(prev_ports))
|
||
|
||
err_msgs = []
|
||
if len(attached_port) != 1:
|
||
err_msg = "NICs for vm {} is not incremented by 1".format(vm_id)
|
||
err_msgs.append(err_msg)
|
||
else:
|
||
attached_port = attached_port[0]
|
||
|
||
if net_id:
|
||
net_name = network_helper.get_net_name_from_id(net_id, con_ssh=con_ssh,
|
||
auth_info=auth_info)
|
||
net_ips = get_vm_values(vm_id, fields=net_name, strict=False,
|
||
con_ssh=con_ssh, auth_info=auth_info)[0]
|
||
if fixed_ip and fixed_ip not in net_ips.split(sep=', '):
|
||
err_msg = "specified fixed ip {} is not found in nova show " \
|
||
"{}".format(fixed_ip, vm_id)
|
||
err_msgs.append(err_msg)
|
||
|
||
elif port_id and port_id not in post_ports:
|
||
err_msg = "port {} is not associated to VM".format(port_id)
|
||
err_msgs.append(err_msg)
|
||
|
||
if err_msgs:
|
||
err_msgs_str = "Post interface attach check failed:\n{}".format(
|
||
'\n'.join(err_msgs))
|
||
if fail_ok:
|
||
LOG.warning(err_msgs_str)
|
||
return 2, attached_port
|
||
raise exceptions.NovaError(err_msgs_str)
|
||
|
||
succ_msg = "Port {} successfully attached to VM {}".format(attached_port,
|
||
vm_id)
|
||
LOG.info(succ_msg)
|
||
return 0, attached_port
|
||
|
||
|
||
def add_ifcfg_scripts(vm_id, mac_addrs, static_ips=None, ipv6='no', reboot=True,
|
||
vm_prompt=None,
|
||
**extra_configs):
|
||
"""
|
||
|
||
Args:
|
||
vm_id:
|
||
mac_addrs (list of str):
|
||
static_ips (None|str|list):
|
||
ipv6:
|
||
reboot:
|
||
vm_prompt
|
||
**extra_configs:
|
||
|
||
Returns:
|
||
|
||
"""
|
||
LOG.info('Add ifcfg script(s) to VM {}'.format(vm_id))
|
||
with ssh_to_vm_from_natbox(vm_id, prompt=vm_prompt) as vm_ssh:
|
||
vm_eths = []
|
||
for mac_addr in mac_addrs:
|
||
eth_name = network_helper.get_eth_for_mac(mac_addr=mac_addr,
|
||
ssh_client=vm_ssh)
|
||
assert eth_name, "vif not found for expected mac_address {} in vm" \
|
||
" {}".format(mac_addr, vm_id)
|
||
vm_eths.append(eth_name)
|
||
|
||
if static_ips:
|
||
if isinstance(static_ips, str):
|
||
static_ips = [static_ips]
|
||
if len(static_ips) != len(vm_eths):
|
||
raise ValueError(
|
||
"static_ips count has to be the same as vm devs to be "
|
||
"configured")
|
||
|
||
for i in range(len(vm_eths)):
|
||
eth = vm_eths[i]
|
||
if static_ips:
|
||
static_ip = static_ips[i]
|
||
script_content = VMNetwork.IFCFG_STATIC.format(eth, ipv6,
|
||
static_ip)
|
||
else:
|
||
script_content = VMNetwork.IFCFG_DHCP.format(eth, ipv6)
|
||
|
||
if extra_configs:
|
||
extra_str = '\n'.join(
|
||
['{}={}'.format(k, v) for k, v in extra_configs.items()])
|
||
script_content += '\n{}'.format(extra_str)
|
||
|
||
script_path = VMPath.ETH_PATH_CENTOS.format(eth)
|
||
vm_ssh.exec_sudo_cmd('touch {}'.format(script_path))
|
||
vm_ssh.exec_sudo_cmd(
|
||
"cat > {} << 'EOT'\n{}\nEOT".format(script_path,
|
||
script_content),
|
||
fail_ok=False)
|
||
|
||
if reboot:
|
||
reboot_vm(vm_id=vm_id)
|
||
|
||
|
||
def detach_interface(vm_id, port_id, cleanup_route=False, fail_ok=False,
|
||
auth_info=None, con_ssh=None,
|
||
verify_virsh=True):
|
||
"""
|
||
Detach a port from vm
|
||
Args:
|
||
vm_id (str):
|
||
port_id (str): existing port that is attached to given vm
|
||
fail_ok (bool):
|
||
auth_info (dict):
|
||
con_ssh (SSHClient):
|
||
cleanup_route (bool)
|
||
verify_virsh (bool): Whether to verify in virsh xmldump for detached
|
||
port
|
||
|
||
Returns (tuple): (<return_code>, <msg>)
|
||
(0, Port <port_id> is successfully detached from VM <vm_id>)
|
||
(1, <stderr>) - cli rejected
|
||
(2, "Port <port_id> is not detached from VM <vm_id>") - detached
|
||
port is still shown in nova show
|
||
|
||
"""
|
||
target_ips = None
|
||
if cleanup_route:
|
||
fixed_ips = \
|
||
network_helper.get_ports(field='Fixed IP Addresses',
|
||
port_id=port_id,
|
||
con_ssh=con_ssh, auth_info=auth_info)[0]
|
||
target_ips = [fixed_ip['ip_address'] for fixed_ip in fixed_ips]
|
||
|
||
mac_to_check = None
|
||
if verify_virsh:
|
||
prev_ports, prev_macs = network_helper.get_ports(
|
||
server=vm_id, auth_info=auth_info, con_ssh=con_ssh,
|
||
field=('ID', 'MAC Address'))
|
||
for prev_port in prev_ports:
|
||
if port_id == prev_port:
|
||
mac_to_check = prev_macs[list(prev_ports).index(prev_port)]
|
||
break
|
||
|
||
LOG.info("Detaching port {} from vm {}".format(port_id, vm_id))
|
||
args = '{} {}'.format(vm_id, port_id)
|
||
code, output = cli.nova('interface-detach', args, ssh_client=con_ssh,
|
||
fail_ok=fail_ok, auth_info=auth_info)
|
||
if code == 1:
|
||
return code, output
|
||
|
||
post_ports = network_helper.get_ports(server=vm_id, auth_info=auth_info,
|
||
con_ssh=con_ssh)
|
||
if port_id in post_ports:
|
||
err_msg = "Port {} is not detached from VM {}".format(port_id, vm_id)
|
||
if fail_ok:
|
||
return 2, err_msg
|
||
else:
|
||
raise exceptions.NeutronError(
|
||
'port {} is still listed for vm {} after detaching'.format(
|
||
port_id, vm_id))
|
||
|
||
succ_msg = "Port {} is successfully detached from VM {}".format(port_id,
|
||
vm_id)
|
||
LOG.info(succ_msg)
|
||
|
||
if cleanup_route and target_ips:
|
||
cleanup_routes_for_vifs(vm_id=vm_id, vm_ips=target_ips, reboot=True)
|
||
|
||
if verify_virsh and mac_to_check:
|
||
if not (cleanup_route and target_ips):
|
||
reboot_vm(vm_id=vm_id, auth_info=auth_info, con_ssh=con_ssh)
|
||
|
||
check_devs_detached(vm_id=vm_id, mac_addrs=mac_to_check,
|
||
con_ssh=con_ssh)
|
||
|
||
return 0, succ_msg
|
||
|
||
|
||
def check_devs_detached(vm_id, mac_addrs, con_ssh=None):
|
||
if isinstance(mac_addrs, str):
|
||
mac_addrs = [mac_addrs]
|
||
|
||
wait_for_vm_pingable_from_natbox(vm_id, con_ssh=con_ssh)
|
||
|
||
LOG.info("Check dev detached from vm")
|
||
vm_err = ''
|
||
with ssh_to_vm_from_natbox(vm_id=vm_id, con_ssh=con_ssh,
|
||
retry_timeout=180) as vm_ssh:
|
||
for mac_addr in mac_addrs:
|
||
if vm_ssh.exec_cmd('ip addr | grep -B 1 "{}"'.format(mac_addr))[0] \
|
||
== 0:
|
||
vm_err += 'Interface with mac address {} still exists in ' \
|
||
'vm\n'.format(mac_addr)
|
||
|
||
LOG.info("Check virsh xmldump on compute host")
|
||
inst_name, vm_host = get_vm_values(vm_id,
|
||
fields=[":instance_name", ":host"],
|
||
strict=False)
|
||
host_err = ''
|
||
with host_helper.ssh_to_host(vm_host, con_ssh=con_ssh) as host_ssh:
|
||
for mac_addr in mac_addrs:
|
||
if host_ssh.exec_sudo_cmd(
|
||
'virsh dumpxml {} | grep -B 1 -A 1 "{}"'.format(
|
||
inst_name, mac_addr))[0] == 0:
|
||
host_err += 'VM interface with mac address {} still exists in' \
|
||
' virsh\n'.format(mac_addr)
|
||
assert not host_err, host_err
|
||
|
||
assert not vm_err, vm_err
|
||
|
||
|
||
def evacuate_vms(host, vms_to_check, con_ssh=None, timeout=600,
|
||
wait_for_host_up=False, fail_ok=False, post_host=None,
|
||
force=True, ping_vms=False):
|
||
"""
|
||
Evacuate given vms by rebooting their host. VMs should be on specified
|
||
host already when this keyword called.
|
||
Args:
|
||
host (str): host to reboot
|
||
vms_to_check (list): vms to check status for after host reboot
|
||
con_ssh (SSHClient):
|
||
timeout (int): Max time to wait for vms to reach active state after
|
||
reboot -f initiated on host
|
||
wait_for_host_up (bool): whether to wait for host reboot completes
|
||
before checking vm status
|
||
fail_ok (bool): whether to return or to fail test when vm(s) failed
|
||
to evacuate
|
||
post_host (str): expected host for vms to be evacuated to
|
||
force (bool): whether to use 'reboot -f'. This param is only used if
|
||
vlm=False.
|
||
ping_vms (bool): whether to ping vms after evacuation
|
||
|
||
Returns (tuple): (<code> (int), <vms_failed_to_evac> (list))
|
||
- (0, []) all vms evacuated successfully. i.e., active state,
|
||
host changed, pingable from NatBox
|
||
- (1, <inactive_vms>) some vms did not reach active state after
|
||
host reboot
|
||
- (2, <vms_host_err>) some vms' host did not change after host reboot
|
||
|
||
"""
|
||
if isinstance(vms_to_check, str):
|
||
vms_to_check = [vms_to_check]
|
||
|
||
HostsToRecover.add(host)
|
||
is_swacted = False
|
||
standby = None
|
||
if wait_for_host_up:
|
||
active, standby = system_helper.get_active_standby_controllers(
|
||
con_ssh=con_ssh)
|
||
if standby and active == host:
|
||
is_swacted = True
|
||
|
||
is_sx = system_helper.is_aio_simplex()
|
||
|
||
LOG.tc_step("'sudo reboot -f' from {}".format(host))
|
||
host_helper.reboot_hosts(host, wait_for_offline=True,
|
||
wait_for_reboot_finish=False, force_reboot=force,
|
||
con_ssh=con_ssh)
|
||
|
||
if is_sx:
|
||
host_helper.wait_for_hosts_ready(hosts=host, con_ssh=con_ssh)
|
||
|
||
try:
|
||
LOG.tc_step(
|
||
"Wait for vms to reach ERROR or REBUILD state with best effort")
|
||
if not is_sx:
|
||
wait_for_vms_values(vms_to_check,
|
||
value=[VMStatus.ERROR, VMStatus.REBUILD],
|
||
fail_ok=True, timeout=120,
|
||
con_ssh=con_ssh)
|
||
|
||
LOG.tc_step(
|
||
"Check vms are in Active state and moved to other host(s) ("
|
||
"non-sx) after host failure")
|
||
res, active_vms, inactive_vms = wait_for_vms_values(
|
||
vms=vms_to_check, value=VMStatus.ACTIVE, timeout=timeout,
|
||
con_ssh=con_ssh)
|
||
|
||
if not is_sx:
|
||
vms_host_err = []
|
||
for vm in vms_to_check:
|
||
if post_host:
|
||
if get_vm_host(vm) != post_host:
|
||
vms_host_err.append(vm)
|
||
else:
|
||
if get_vm_host(vm) == host:
|
||
vms_host_err.append(vm)
|
||
|
||
if vms_host_err:
|
||
if post_host:
|
||
err_msg = "Following VMs is not moved to expected host " \
|
||
"{} from {}: {}\nVMs did not reach Active " \
|
||
"state: {}".format(post_host, host, vms_host_err,
|
||
inactive_vms)
|
||
else:
|
||
err_msg = "Following VMs stayed on the same host {}: " \
|
||
"{}\nVMs did not reach Active state: {}".\
|
||
format(host, vms_host_err, inactive_vms)
|
||
|
||
if fail_ok:
|
||
LOG.warning(err_msg)
|
||
return 1, vms_host_err
|
||
raise exceptions.VMError(err_msg)
|
||
|
||
if inactive_vms:
|
||
err_msg = "VMs did not reach Active state after vm host rebooted:" \
|
||
" {}".format(inactive_vms)
|
||
if fail_ok:
|
||
LOG.warning(err_msg)
|
||
return 2, inactive_vms
|
||
raise exceptions.VMError(err_msg)
|
||
|
||
if ping_vms:
|
||
LOG.tc_step("Ping vms after evacuated")
|
||
for vm_ in vms_to_check:
|
||
wait_for_vm_pingable_from_natbox(vm_id=vm_,
|
||
timeout=VMTimeout.DHCP_RETRY)
|
||
|
||
LOG.info("All vms are successfully evacuated to other host")
|
||
return 0, []
|
||
|
||
finally:
|
||
if wait_for_host_up:
|
||
LOG.tc_step("Waiting for {} to recover".format(host))
|
||
host_helper.wait_for_hosts_ready(host, con_ssh=con_ssh)
|
||
# Do not fail the test due to task affining incomplete for now to
|
||
# unblock test case.
|
||
host_helper.wait_for_tasks_affined(host=host, con_ssh=con_ssh,
|
||
fail_ok=True)
|
||
if is_swacted:
|
||
host_helper.wait_for_tasks_affined(standby, con_ssh=con_ssh,
|
||
fail_ok=True)
|
||
time.sleep(60) # Give some idle time before continue.
|
||
if system_helper.is_aio_duplex(con_ssh=con_ssh):
|
||
system_helper.wait_for_alarm_gone(
|
||
alarm_id=EventLogID.CPU_USAGE_HIGH, fail_ok=True,
|
||
check_interval=30)
|
||
|
||
|
||
def boot_vms_various_types(storage_backing=None, target_host=None,
|
||
cleanup='function', avail_zone='nova', vms_num=5):
|
||
"""
|
||
Boot following 5 vms and ensure they are pingable from NatBox:
|
||
- vm1: ephemeral=0, swap=0, boot_from_volume
|
||
- vm2: ephemeral=1, swap=1, boot_from_volume
|
||
- vm3: ephemeral=0, swap=0, boot_from_image
|
||
- vm4: ephemeral=0, swap=0, boot_from_image, attach_volume
|
||
- vm5: ephemeral=1, swap=1, boot_from_image
|
||
Args:
|
||
storage_backing (str|None): storage backing to set in flavor spec.
|
||
When None, storage backing which used by
|
||
most up hypervisors will be used.
|
||
target_host (str|None): Boot vm on target_host when specified. (admin
|
||
role has to be added to tenant under test)
|
||
cleanup (str|None): Scope for resource cleanup, valid values:
|
||
'function', 'class', 'module', None.
|
||
When None, vms/volumes/flavors will be kept on system
|
||
avail_zone (str): availability zone to boot the vms
|
||
vms_num
|
||
|
||
Returns (list): list of vm ids
|
||
|
||
"""
|
||
LOG.info("Create a flavor without ephemeral or swap disks")
|
||
flavor_1 = \
|
||
nova_helper.create_flavor('flv_rootdisk',
|
||
storage_backing=storage_backing,
|
||
cleanup=cleanup)[1]
|
||
|
||
LOG.info("Create another flavor with ephemeral and swap disks")
|
||
flavor_2 = nova_helper.create_flavor('flv_ephemswap', ephemeral=1, swap=512,
|
||
storage_backing=storage_backing,
|
||
cleanup=cleanup)[1]
|
||
|
||
launched_vms = []
|
||
for i in range(int(math.ceil(vms_num / 5.0))):
|
||
LOG.info(
|
||
"Boot vm1 from volume with flavor flv_rootdisk and wait for it "
|
||
"pingable from NatBox")
|
||
vm1_name = "vol_root"
|
||
vm1 = boot_vm(vm1_name, flavor=flavor_1, source='volume',
|
||
avail_zone=avail_zone, vm_host=target_host,
|
||
cleanup=cleanup)[1]
|
||
|
||
wait_for_vm_pingable_from_natbox(vm1)
|
||
launched_vms.append(vm1)
|
||
if len(launched_vms) == vms_num:
|
||
break
|
||
|
||
LOG.info(
|
||
"Boot vm2 from volume with flavor flv_localdisk and wait for it "
|
||
"pingable from NatBox")
|
||
vm2_name = "vol_ephemswap"
|
||
vm2 = boot_vm(vm2_name, flavor=flavor_2, source='volume',
|
||
avail_zone=avail_zone, vm_host=target_host,
|
||
cleanup=cleanup)[1]
|
||
|
||
wait_for_vm_pingable_from_natbox(vm2)
|
||
launched_vms.append(vm2)
|
||
if len(launched_vms) == vms_num:
|
||
break
|
||
|
||
LOG.info(
|
||
"Boot vm3 from image with flavor flv_rootdisk and wait for it "
|
||
"pingable from NatBox")
|
||
vm3_name = "image_root"
|
||
vm3 = boot_vm(vm3_name, flavor=flavor_1, source='image',
|
||
avail_zone=avail_zone, vm_host=target_host,
|
||
cleanup=cleanup)[1]
|
||
|
||
wait_for_vm_pingable_from_natbox(vm3)
|
||
launched_vms.append(vm3)
|
||
if len(launched_vms) == vms_num:
|
||
break
|
||
|
||
LOG.info(
|
||
"Boot vm4 from image with flavor flv_rootdisk, attach a volume to "
|
||
"it and wait for it "
|
||
"pingable from NatBox")
|
||
vm4_name = 'image_root_attachvol'
|
||
vm4 = boot_vm(vm4_name, flavor_1, source='image', avail_zone=avail_zone,
|
||
vm_host=target_host,
|
||
cleanup=cleanup)[1]
|
||
|
||
vol = cinder_helper.create_volume(bootable=False, cleanup=cleanup)[1]
|
||
attach_vol_to_vm(vm4, vol_id=vol, cleanup=cleanup)
|
||
|
||
wait_for_vm_pingable_from_natbox(vm4)
|
||
launched_vms.append(vm4)
|
||
if len(launched_vms) == vms_num:
|
||
break
|
||
|
||
LOG.info(
|
||
"Boot vm5 from image with flavor flv_localdisk and wait for it "
|
||
"pingable from NatBox")
|
||
vm5_name = 'image_ephemswap'
|
||
vm5 = boot_vm(vm5_name, flavor_2, source='image', avail_zone=avail_zone,
|
||
vm_host=target_host,
|
||
cleanup=cleanup)[1]
|
||
|
||
wait_for_vm_pingable_from_natbox(vm5)
|
||
launched_vms.append(vm5)
|
||
if len(launched_vms) == vms_num:
|
||
break
|
||
|
||
assert len(launched_vms) == vms_num
|
||
return launched_vms
|
||
|
||
|
||
def get_vcpu_model(vm_id, guest_os=None, con_ssh=None):
|
||
"""
|
||
Get vcpu model of given vm. e.g., Intel(R) Xeon(R) CPU E5-2680 v2 @ 2.80GHz
|
||
Args:
|
||
vm_id (str):
|
||
guest_os (str):
|
||
con_ssh (SSHClient):
|
||
|
||
Returns (str):
|
||
|
||
"""
|
||
with ssh_to_vm_from_natbox(vm_id, vm_image_name=guest_os,
|
||
con_ssh=con_ssh) as vm_ssh:
|
||
out = vm_ssh.exec_cmd("cat /proc/cpuinfo | grep --color=never "
|
||
"'model name'", fail_ok=False)[1]
|
||
vcpu_model = out.strip().splitlines()[0].split(sep=': ')[1].strip()
|
||
|
||
LOG.info("VM {} cpu model: {}".format(vm_id, vcpu_model))
|
||
return vcpu_model
|
||
|
||
|
||
def get_quotas(quotas, default=False, tenant=None, auth_info=None,
|
||
con_ssh=None):
|
||
"""
|
||
Get openstack quotas
|
||
Args:
|
||
quotas (str|list|tuple):
|
||
default (bool)
|
||
tenant (str|None): Only used if admin user is used in auth_info
|
||
auth_info (dict):
|
||
con_ssh:
|
||
|
||
Returns (list):
|
||
|
||
"""
|
||
if auth_info is None:
|
||
auth_info = Tenant.get_primary()
|
||
|
||
args = ''
|
||
if default:
|
||
args += '--default'
|
||
if tenant and auth_info['user'] == 'admin':
|
||
args += ' {}'.format(tenant)
|
||
|
||
if isinstance(quotas, str):
|
||
quotas = [quotas]
|
||
|
||
table_ = table_parser.table(
|
||
cli.openstack('quota show', args, ssh_client=con_ssh,
|
||
auth_info=auth_info)[1])
|
||
|
||
values = []
|
||
for item in quotas:
|
||
val = table_parser.get_value_two_col_table(table_, item)
|
||
try:
|
||
val = eval(val)
|
||
except (NameError, SyntaxError):
|
||
pass
|
||
values.append(val)
|
||
|
||
return values
|
||
|
||
|
||
def get_quota_details_info(component='compute', tenant=None, detail=True,
|
||
resources=None,
|
||
auth_info=Tenant.get('admin'), con_ssh=None):
|
||
"""
|
||
Get quota details table from openstack quota list --detail
|
||
Args:
|
||
component (str): compute, network or volume
|
||
tenant:
|
||
detail (bool)
|
||
resources (str|list|tuple|None): filter out table. Used only if
|
||
detail is True and component is not volume
|
||
auth_info:
|
||
con_ssh:
|
||
|
||
Returns (dict): All keys are converted to lower case.
|
||
e.g.,
|
||
{'server_groups': {'in use': 0, 'reserved': 1, 'limit': 10},
|
||
...}
|
||
|
||
"""
|
||
valid_components = ('compute', 'network', 'volume')
|
||
if component not in valid_components:
|
||
raise ValueError(
|
||
"Please specify a valid component: {}".format(valid_components))
|
||
|
||
if not tenant:
|
||
tenant = Tenant.get_primary()['tenant']
|
||
|
||
detail_str = ' --detail' if detail and component != 'volume' else ''
|
||
args = '--project={} --{}{}'.format(tenant, component, detail_str)
|
||
|
||
table_ = table_parser.table(
|
||
cli.openstack('quota list', args, ssh_client=con_ssh,
|
||
auth_info=auth_info)[1])
|
||
key_header = 'Project ID'
|
||
if detail_str:
|
||
if resources:
|
||
table_ = table_parser.filter_table(table_, Resource=resources)
|
||
key_header = 'resource'
|
||
|
||
table_ = table_parser.row_dict_table(table_, key_header=key_header,
|
||
lower_case=True,
|
||
eliminate_keys=key_header)
|
||
return {k: int(v) for k, v in table_.items()}
|
||
|
||
|
||
def set_quotas(tenant=None, auth_info=Tenant.get('admin'), con_ssh=None,
|
||
sys_con_for_dc=True, fail_ok=False,
|
||
**kwargs):
|
||
"""
|
||
Set openstack quotas
|
||
Args:
|
||
tenant (str):
|
||
auth_info (dict):
|
||
con_ssh:
|
||
sys_con_for_dc (bool):
|
||
fail_ok (bool):
|
||
**kwargs: quotas to set. e.g., **{'instances': 10, 'volumes': 20}
|
||
|
||
Returns (tuple):
|
||
|
||
"""
|
||
if not tenant:
|
||
tenant = Tenant.get_primary()['tenant']
|
||
if not auth_info:
|
||
auth_info = Tenant.get_primary()
|
||
if ProjVar.get_var('IS_DC') and sys_con_for_dc and auth_info['region'] \
|
||
!= 'SystemController':
|
||
auth_info = Tenant.get(auth_info['user'], dc_region='SystemController')
|
||
|
||
args = common.parse_args(
|
||
args_dict={k.replace('_', '-'): v for k, v in kwargs.items()})
|
||
args = '{} {}'.format(args, tenant)
|
||
code, output = cli.openstack('quota set', args, ssh_client=con_ssh,
|
||
fail_ok=fail_ok, auth_info=auth_info)
|
||
if code > 0:
|
||
return 1, output
|
||
|
||
msg = '{} quotas set successfully'.format(tenant)
|
||
LOG.info(msg)
|
||
return 0, msg
|
||
|
||
|
||
def ensure_vms_quotas(vms_num=10, cores_num=None, vols_num=None, ram=None,
|
||
tenant=None, auth_info=Tenant.get('admin'),
|
||
con_ssh=None):
|
||
"""
|
||
Update instances, cores, volumes quotas to given numbers
|
||
Args:
|
||
vms_num (int): max number of instances allowed for given tenant
|
||
cores_num (int|None): twice of the vms quota when None
|
||
vols_num (int|None): twice of the vms quota when None
|
||
ram (int|None)
|
||
tenant (None|str):
|
||
auth_info (dict): auth info for admin user
|
||
con_ssh (SSHClient):
|
||
|
||
"""
|
||
if not vols_num:
|
||
vols_num = 2 * vms_num
|
||
if not cores_num:
|
||
cores_num = 2 * vms_num
|
||
if not ram:
|
||
ram = 2048 * vms_num
|
||
|
||
if not tenant:
|
||
tenant = Tenant.get_primary()['tenant']
|
||
|
||
volumes_quota, vms_quota, cores_quota, ram_quota = get_quotas(
|
||
quotas=['volumes', 'instances', 'cores', 'ram'],
|
||
con_ssh=con_ssh, tenant=tenant, auth_info=auth_info)
|
||
kwargs = {}
|
||
if vms_num > vms_quota:
|
||
kwargs['instances'] = vms_num
|
||
if cores_num > cores_quota:
|
||
kwargs['cores'] = cores_num
|
||
if vols_num > volumes_quota:
|
||
kwargs['volumes'] = vols_num
|
||
if ram > ram_quota:
|
||
kwargs['ram'] = ram
|
||
|
||
if kwargs:
|
||
set_quotas(con_ssh=con_ssh, tenant=tenant, auth_info=auth_info,
|
||
**kwargs)
|
||
|
||
|
||
def launch_vms(vm_type, count=1, nics=None, flavor=None, storage_backing=None,
|
||
image=None, boot_source=None,
|
||
guest_os=None, avail_zone=None, target_host=None, ping_vms=False,
|
||
con_ssh=None, auth_info=None,
|
||
cleanup='function', **boot_vm_kwargs):
|
||
"""
|
||
|
||
Args:
|
||
vm_type:
|
||
count:
|
||
nics:
|
||
flavor:
|
||
storage_backing (str):
|
||
storage backend for flavor to be created
|
||
only used if flavor is None
|
||
image:
|
||
boot_source:
|
||
guest_os
|
||
avail_zone:
|
||
target_host:
|
||
ping_vms
|
||
con_ssh:
|
||
auth_info:
|
||
cleanup:
|
||
boot_vm_kwargs (dict):
|
||
additional kwargs to pass to boot_vm
|
||
|
||
Returns:
|
||
|
||
"""
|
||
|
||
if not flavor:
|
||
flavor = nova_helper.create_flavor(name=vm_type, vcpus=2,
|
||
storage_backing=storage_backing,
|
||
cleanup=cleanup)[1]
|
||
extra_specs = {FlavorSpec.CPU_POLICY: 'dedicated'}
|
||
|
||
if vm_type in ['vswitch', 'dpdk', 'vhost']:
|
||
extra_specs.update({FlavorSpec.VCPU_MODEL: 'SandyBridge',
|
||
FlavorSpec.MEM_PAGE_SIZE: '2048'})
|
||
|
||
nova_helper.set_flavor(flavor=flavor, **extra_specs)
|
||
|
||
resource_id = None
|
||
boot_source = boot_source if boot_source else 'volume'
|
||
if image:
|
||
if boot_source == 'volume':
|
||
resource_id = \
|
||
cinder_helper.create_volume(name=vm_type, source_id=image,
|
||
auth_info=auth_info,
|
||
guest_image=guest_os)[1]
|
||
if cleanup:
|
||
ResourceCleanup.add('volume', resource_id, scope=cleanup)
|
||
else:
|
||
resource_id = image
|
||
|
||
if not nics:
|
||
if vm_type in ['pci-sriov', 'pci-passthrough']:
|
||
raise NotImplementedError("nics has to be provided for pci-sriov and "
|
||
"pci-passthrough")
|
||
|
||
if vm_type in ['vswitch', 'dpdk', 'vhost']:
|
||
vif_model = 'avp'
|
||
else:
|
||
vif_model = vm_type
|
||
|
||
mgmt_net_id = network_helper.get_mgmt_net_id(auth_info=auth_info)
|
||
tenant_net_id = network_helper.get_tenant_net_id(auth_info=auth_info)
|
||
internal_net_id = network_helper.get_internal_net_id(
|
||
auth_info=auth_info)
|
||
|
||
nics = [{'net-id': mgmt_net_id},
|
||
{'net-id': tenant_net_id, 'vif-model': vif_model},
|
||
{'net-id': internal_net_id, 'vif-model': vif_model}]
|
||
|
||
user_data = None
|
||
if vm_type in ['vswitch', 'dpdk', 'vhost']:
|
||
user_data = network_helper.get_dpdk_user_data(con_ssh=con_ssh)
|
||
|
||
vms = []
|
||
for i in range(count):
|
||
vm_id = boot_vm(name="{}-{}".format(vm_type, i), flavor=flavor,
|
||
source=boot_source, source_id=resource_id,
|
||
nics=nics, guest_os=guest_os, avail_zone=avail_zone,
|
||
vm_host=target_host, user_data=user_data,
|
||
auth_info=auth_info, con_ssh=con_ssh, cleanup=cleanup,
|
||
**boot_vm_kwargs)[1]
|
||
vms.append(vm_id)
|
||
|
||
if ping_vms:
|
||
wait_for_vm_pingable_from_natbox(vm_id=vm_id, con_ssh=con_ssh)
|
||
return vms, nics
|
||
|
||
|
||
def get_ping_loss_duration_between_vms(from_vm, to_vm, net_type='data',
|
||
timeout=600, ipv6=False,
|
||
start_event=None,
|
||
end_event=None, con_ssh=None,
|
||
ping_interval=1):
|
||
"""
|
||
Get ping loss duration in milliseconds from one vm to another
|
||
Args:
|
||
from_vm (str): id of the ping source vm
|
||
to_vm (str): id of the ping destination vm
|
||
net_type (str): e.g., data, internal, mgmt
|
||
timeout (int): max time to wait for ping loss before force end it
|
||
ipv6 (bool): whether to use ping -6 for ipv6 address
|
||
start_event (Event): set given event to signal ping has started
|
||
end_event (Event): stop ping loss detection if given event is set
|
||
con_ssh (SSHClient):
|
||
ping_interval (int|float): timeout of ping cmd in seconds
|
||
|
||
Returns (int): milliseconds of ping loss duration
|
||
|
||
"""
|
||
|
||
to_vm_ip = _get_vms_ips(vm_ids=to_vm, net_types=net_type,
|
||
con_ssh=con_ssh)[0][0]
|
||
with ssh_to_vm_from_natbox(vm_id=from_vm, con_ssh=con_ssh) as from_vm_ssh:
|
||
duration = network_helper.get_ping_failure_duration(
|
||
server=to_vm_ip, ssh_client=from_vm_ssh, timeout=timeout,
|
||
ipv6=ipv6, start_event=start_event, end_event=end_event,
|
||
ping_interval=ping_interval)
|
||
return duration
|
||
|
||
|
||
def get_ping_loss_duration_from_natbox(vm_id, timeout=900, start_event=None,
|
||
end_event=None, con_ssh=None,
|
||
ping_interval=0.5):
|
||
vm_ip = _get_vms_ips(vm_ids=vm_id, net_types='mgmt', con_ssh=con_ssh)[0][0]
|
||
natbox_client = NATBoxClient.get_natbox_client()
|
||
duration = network_helper.get_ping_failure_duration(
|
||
server=vm_ip, ssh_client=natbox_client, timeout=timeout,
|
||
start_event=start_event, end_event=end_event,
|
||
ping_interval=ping_interval)
|
||
return duration
|
||
|
||
|
||
def get_ping_loss_duration_on_operation(vm_id, timeout, ping_interval,
|
||
oper_func, *func_args, **func_kwargs):
|
||
LOG.tc_step("Start pinging vm {} from NatBox on a new thread".format(vm_id))
|
||
start_event = Events("Ping started")
|
||
end_event = Events("Operation completed")
|
||
ping_thread = MThread(get_ping_loss_duration_from_natbox, vm_id=vm_id,
|
||
timeout=timeout,
|
||
start_event=start_event, end_event=end_event,
|
||
ping_interval=ping_interval)
|
||
ping_thread.start_thread(timeout=timeout + 30)
|
||
|
||
try:
|
||
if start_event.wait_for_event(timeout=60):
|
||
LOG.tc_step(
|
||
"Perform operation on vm and ensure it's reachable after that")
|
||
oper_func(*func_args, **func_kwargs)
|
||
# Operation completed. Set end flag so ping thread can end properly
|
||
time.sleep(3)
|
||
end_event.set()
|
||
# Expect ping thread to end in less than 1 minute after
|
||
# live-migration complete
|
||
duration = ping_thread.get_output(timeout=60)
|
||
# assert duration, "No ping loss detected"
|
||
if duration == 0:
|
||
LOG.warning("No ping loss detected")
|
||
return duration
|
||
|
||
assert False, "Ping failed since start"
|
||
finally:
|
||
ping_thread.wait_for_thread_end(timeout=5)
|
||
|
||
|
||
def collect_guest_logs(vm_id):
|
||
LOG.info("Attempt to collect guest logs with best effort")
|
||
log_names = ['messages', 'user.log']
|
||
try:
|
||
res = _recover_vm(vm_id=vm_id)
|
||
if not res:
|
||
LOG.info(
|
||
"VM {} in unrecoverable state, skip collect guest logs.".format(
|
||
vm_id))
|
||
return
|
||
|
||
with ssh_to_vm_from_natbox(vm_id) as vm_ssh:
|
||
for log_name in log_names:
|
||
log_path = '/var/log/{}'.format(log_name)
|
||
if not vm_ssh.file_exists(log_path):
|
||
continue
|
||
|
||
local_log_path = '{}/{}_{}'.format(
|
||
ProjVar.get_var('GUEST_LOGS_DIR'), log_name, vm_id)
|
||
current_user = local_host.get_user()
|
||
if current_user == TestFileServer.USER:
|
||
vm_ssh.exec_sudo_cmd('chmod -R 755 {}'.format(log_path),
|
||
fail_ok=True)
|
||
vm_ssh.scp_on_source_to_localhost(
|
||
source_file=log_path,
|
||
dest_user=current_user,
|
||
dest_password=TestFileServer.PASSWORD,
|
||
dest_path=local_log_path)
|
||
else:
|
||
output = vm_ssh.exec_cmd('tail -n 200 {}'.format(log_path),
|
||
fail_ok=False)[1]
|
||
with open(local_log_path, mode='w') as f:
|
||
f.write(output)
|
||
return
|
||
|
||
except Exception as e:
|
||
LOG.warning("Failed to collect guest logs: {}".format(e))
|
||
|
||
|
||
def _recover_vm(vm_id, con_ssh=None):
|
||
status = get_vm_status(vm_id=vm_id, con_ssh=con_ssh)
|
||
if status == VMStatus.ACTIVE:
|
||
return True
|
||
elif status == VMStatus.STOPPED:
|
||
code, msg = start_vms(vms=vm_id, fail_ok=True)
|
||
return code == 0
|
||
elif status == VMStatus.PAUSED:
|
||
code, msg = unpause_vm(vm_id=vm_id, fail_ok=True, con_ssh=con_ssh)
|
||
if code > 0:
|
||
code, msg = resume_vm(vm_id, fail_ok=True, con_ssh=con_ssh)
|
||
if code > 0:
|
||
return False
|
||
return True
|
||
else:
|
||
return False
|
||
|
||
|
||
def get_vim_events(vm_id, event_ids=None, controller=None, con_ssh=None):
|
||
"""
|
||
Get vim events from nfv-vim-events.log
|
||
Args:
|
||
vm_id (str):
|
||
event_ids (None|str|list|tuple): return only given vim events when
|
||
specified
|
||
controller (None|str): controller where vim log is on. Use current
|
||
active controller if None.
|
||
con_ssh (SSHClient):
|
||
|
||
Returns (list): list of dictionaries, each dictionary is one event. e.g.,:
|
||
[{'log-id': '47', 'event-id': 'instance-live-migrate-begin', ... ,
|
||
'timestamp': '2018-03-04 01:34:28.915008'},
|
||
{'log-id': '49', 'event-id': 'instance-live-migrated', ... ,
|
||
'timestamp': '2018-03-04 01:35:34.043094'}]
|
||
|
||
"""
|
||
if not controller:
|
||
controller = system_helper.get_active_controller_name()
|
||
|
||
if isinstance(event_ids, str):
|
||
event_ids = [event_ids]
|
||
|
||
with host_helper.ssh_to_host(controller, con_ssh=con_ssh) as controller_ssh:
|
||
vm_logs = controller_ssh.exec_cmd(
|
||
'grep --color=never -A 4 -B 6 -E "entity .*{}" '
|
||
'/var/log/nfv-vim-events.log'.
|
||
format(vm_id))[1]
|
||
|
||
log_lines = vm_logs.splitlines()
|
||
vm_events = []
|
||
vm_event = {}
|
||
for line in log_lines:
|
||
if re.search(' = ', line):
|
||
if line.startswith('log-id') and vm_event:
|
||
if not event_ids or vm_event['event-id'] in event_ids:
|
||
vm_events.append(vm_event)
|
||
|
||
vm_event = {}
|
||
key, val = re.findall('(.*)= (.*)', line)[0]
|
||
vm_event[key.strip()] = val.strip()
|
||
|
||
if vm_event and (not event_ids or vm_event['event-id'] in event_ids):
|
||
vm_events.append(vm_event)
|
||
|
||
LOG.info("VM events: {}".format(vm_events))
|
||
return vm_events
|
||
|
||
|
||
def get_live_migrate_duration(vm_id, con_ssh=None):
|
||
LOG.info(
|
||
"Get live migration duration from nfv-vim-events.log for vm {}".format(
|
||
vm_id))
|
||
events = (VimEventID.LIVE_MIG_BEGIN, VimEventID.LIVE_MIG_END)
|
||
live_mig_begin, live_mig_end = get_vim_events(vm_id=vm_id, event_ids=events,
|
||
con_ssh=con_ssh)
|
||
|
||
start_time = live_mig_begin['timestamp']
|
||
end_time = live_mig_end['timestamp']
|
||
duration = common.get_timedelta_for_isotimes(time1=start_time,
|
||
time2=end_time).total_seconds()
|
||
LOG.info("Live migration for vm {} took {} seconds".format(vm_id, duration))
|
||
|
||
return duration
|
||
|
||
|
||
def get_cold_migrate_duration(vm_id, con_ssh=None):
|
||
LOG.info("Get cold migration duration from vim-event-log for vm {}".format(
|
||
vm_id))
|
||
events = (VimEventID.COLD_MIG_BEGIN, VimEventID.COLD_MIG_END,
|
||
VimEventID.COLD_MIG_CONFIRM_BEGIN, VimEventID.COLD_MIG_CONFIRMED)
|
||
cold_mig_begin, cold_mig_end, cold_mig_confirm_begin, \
|
||
cold_mig_confirm_end = get_vim_events(vm_id=vm_id, event_ids=events,
|
||
con_ssh=con_ssh)
|
||
|
||
duration_cold_mig = common.get_timedelta_for_isotimes(
|
||
time1=cold_mig_begin['timestamp'],
|
||
time2=cold_mig_end['timestamp']).total_seconds()
|
||
|
||
duration_confirm = common.get_timedelta_for_isotimes(
|
||
time1=cold_mig_confirm_begin['timestamp'],
|
||
time2=cold_mig_confirm_end['timestamp']).total_seconds()
|
||
|
||
duration = duration_cold_mig + duration_confirm
|
||
LOG.info("Cold migrate and confirm for vm {} took {} seconds".format(
|
||
vm_id, duration))
|
||
|
||
return duration
|
||
|
||
|
||
def live_migrate_force_complete(vm_id, migration_id=None, timeout=300,
|
||
fail_ok=False, con_ssh=None):
|
||
"""
|
||
Run nova live-migration-force-complete against given vm and migration
|
||
session.
|
||
Args:
|
||
vm_id (str):
|
||
migration_id (str|int):
|
||
timeout:
|
||
fail_ok:
|
||
con_ssh:
|
||
|
||
Returns (tuple):
|
||
(0, 'VM is successfully live-migrated after
|
||
live-migration-force-complete')
|
||
(1, <err_msg>) # nova live-migration-force-complete cmd
|
||
rejected. Only returns if fail_ok=True.
|
||
|
||
"""
|
||
if not migration_id:
|
||
migration_id = get_vm_migration_values(vm_id=vm_id, fail_ok=False,
|
||
con_ssh=con_ssh)[0]
|
||
|
||
# No replacement in openstack client
|
||
code, output = cli.nova('live-migration-force-complete',
|
||
'{} {}'.format(vm_id, migration_id),
|
||
ssh_client=con_ssh,
|
||
fail_ok=fail_ok)
|
||
|
||
if code > 0:
|
||
return 1, output
|
||
|
||
wait_for_vm_migration_status(vm_id=vm_id, migration_id=migration_id,
|
||
fail_ok=False, timeout=timeout,
|
||
con_ssh=con_ssh)
|
||
msg = "VM is successfully live-migrated after live-migration-force-complete"
|
||
LOG.info(msg)
|
||
return 0, msg
|
||
|
||
|
||
def get_vm_migration_values(vm_id, field='Id', migration_type='live-migration',
|
||
fail_ok=True, con_ssh=None, **kwargs):
|
||
"""
|
||
Get values for given vm via nova migration-list
|
||
Args:
|
||
vm_id (str):
|
||
field (str):
|
||
migration_type(str): valid types: live-migration, migration
|
||
fail_ok:
|
||
con_ssh:
|
||
**kwargs:
|
||
|
||
Returns (list):
|
||
|
||
"""
|
||
migration_tab = nova_helper.get_migration_list_table(con_ssh=con_ssh)
|
||
filters = {'Instance UUID': vm_id, 'Type': migration_type}
|
||
if kwargs:
|
||
filters.update(kwargs)
|
||
mig_ids = table_parser.get_values(migration_tab, target_header=field,
|
||
**filters)
|
||
if not mig_ids and not fail_ok:
|
||
raise exceptions.VMError(
|
||
"{} has no {} session with filters: {}".format(vm_id,
|
||
migration_type,
|
||
kwargs))
|
||
|
||
return mig_ids
|
||
|
||
|
||
def wait_for_vm_migration_status(vm_id, migration_id=None, migration_type=None,
|
||
expt_status='completed',
|
||
fail_ok=False, timeout=300, check_interval=5,
|
||
con_ssh=None):
|
||
"""
|
||
Wait for a migration session to reach given status in nova mgiration-list
|
||
Args:
|
||
vm_id (str):
|
||
migration_id (str|int):
|
||
migration_type (str): valid types: live-migration, migration
|
||
expt_status (str): migration status to wait for. such as completed,
|
||
running, etc
|
||
fail_ok (bool):
|
||
timeout (int): max time to wait for the state
|
||
check_interval (int):
|
||
con_ssh:
|
||
|
||
Returns (tuple):
|
||
(0, <expt_status>) # migration status reached as expected
|
||
(1, <current_status>) # did not reach given status. This only
|
||
returns if fail_ok=True
|
||
|
||
"""
|
||
if not migration_id:
|
||
migration_id = get_vm_migration_values(
|
||
vm_id=vm_id, migration_type=migration_type, fail_ok=False,
|
||
con_ssh=con_ssh)[0]
|
||
|
||
LOG.info("Waiting for migration {} for vm {} to reach {} status".format(
|
||
migration_id, vm_id, expt_status))
|
||
end_time = time.time() + timeout
|
||
prev_state = None
|
||
while time.time() < end_time:
|
||
mig_status = get_vm_migration_values(vm_id=vm_id, field='Status',
|
||
**{'Id': migration_id})[0]
|
||
if mig_status == expt_status:
|
||
LOG.info(
|
||
"Migration {} for vm {} reached status: {}".format(migration_id,
|
||
vm_id,
|
||
expt_status))
|
||
return True, expt_status
|
||
|
||
if mig_status != prev_state:
|
||
LOG.info(
|
||
"Migration {} for vm {} is in status - {}".format(migration_id,
|
||
vm_id,
|
||
mig_status))
|
||
prev_state = mig_status
|
||
|
||
time.sleep(check_interval)
|
||
|
||
msg = 'Migration {} for vm {} did not reach {} status within {} seconds. ' \
|
||
'It is in {} status.'.format(migration_id, vm_id, expt_status,
|
||
timeout, prev_state)
|
||
if fail_ok:
|
||
LOG.warning(msg)
|
||
return False, prev_state
|
||
else:
|
||
raise exceptions.VMError(msg)
|
||
|
||
|
||
def get_vms_ports_info(vms, rtn_subnet_id=False):
|
||
"""
|
||
Get VMs' ports' (ip_addr, subnet_cidr_or_id, mac_addr).
|
||
|
||
Args:
|
||
vms (str|list):
|
||
vm_id, or a list of vm_ids
|
||
rtn_subnet_id (bool):
|
||
replaces cidr with subnet_id in result
|
||
|
||
Returns (dict):
|
||
{vms[0]: [(ip_addr, subnet, ...], vms[1]: [...], ...}
|
||
"""
|
||
if not issubclass(type(vms), (list, tuple)):
|
||
vms = [vms]
|
||
|
||
info = {}
|
||
subnet_tab_ = table_parser.table(
|
||
cli.openstack('subnet list', auth_info=Tenant.get('admin'))[1])
|
||
for vm in vms:
|
||
info[vm] = []
|
||
vm_ports, vm_macs, vm_fixed_ips = network_helper.get_ports(
|
||
server=vm, field=('ID', 'MAC Address', 'Fixed IP Addresses'))
|
||
for i in range(len(vm_ports)):
|
||
port = vm_ports[i]
|
||
mac = vm_macs[i]
|
||
fixed_ips = vm_fixed_ips[i]
|
||
if not isinstance(fixed_ips, list):
|
||
fixed_ips = [fixed_ips]
|
||
|
||
for fixed_ip in fixed_ips:
|
||
subnet_id = fixed_ip['subnet_id']
|
||
ip_addr = fixed_ip['ip_address']
|
||
subnet = subnet_id if rtn_subnet_id else \
|
||
table_parser.get_values(subnet_tab_, 'Subnet',
|
||
id=subnet_id)[0]
|
||
net_id = table_parser.get_values(subnet_tab_, 'Network',
|
||
id=subnet_id)[0]
|
||
|
||
LOG.info(
|
||
"VM {} port {}: mac={} ip={} subnet={} net_id={}".format(
|
||
vm, port, mac, ip_addr, subnet, net_id))
|
||
info[vm].append((port, ip_addr, subnet, mac, net_id))
|
||
|
||
return info
|
||
|
||
|
||
def _set_vm_route(vm_id, target_subnet, via_ip, dev_or_mac, persist=True):
|
||
# returns True if the targeted VM is vswitch-enabled
|
||
# for vswitch-enabled VMs, it must be setup with TisInitServiceScript if
|
||
# persist=True
|
||
with ssh_to_vm_from_natbox(vm_id) as ssh_client:
|
||
vshell, msg = ssh_client.exec_cmd("vshell port-list", fail_ok=True)
|
||
vshell = not vshell
|
||
if ':' in dev_or_mac:
|
||
dev = network_helper.get_eth_for_mac(ssh_client, dev_or_mac,
|
||
vshell=vshell)
|
||
else:
|
||
dev = dev_or_mac
|
||
if not vshell: # not avs managed
|
||
param = target_subnet, via_ip, dev
|
||
LOG.info("Routing {} via {} on interface {}".format(*param))
|
||
ssh_client.exec_sudo_cmd(
|
||
"route add -net {} gw {} {}".format(*param), fail_ok=False)
|
||
if persist:
|
||
LOG.info("Setting persistent route")
|
||
ssh_client.exec_sudo_cmd(
|
||
"echo -e \"{} via {}\" > "
|
||
"/etc/sysconfig/network-scripts/route-{}".format(
|
||
*param), fail_ok=False)
|
||
return False
|
||
else:
|
||
param = target_subnet, via_ip, dev
|
||
LOG.info(
|
||
"Routing {} via {} on interface {}, AVS-enabled".format(*param))
|
||
ssh_client.exec_sudo_cmd(
|
||
"sed -i $'s,quit,route add {} {} {} 1\\\\nquit,"
|
||
"g' /etc/vswitch/vswitch.cmds.default".format(
|
||
target_subnet, dev, via_ip), fail_ok=False)
|
||
# reload vswitch
|
||
ssh_client.exec_sudo_cmd("/etc/init.d/vswitch restart",
|
||
fail_ok=False)
|
||
if persist:
|
||
LOG.info("Setting persistent route")
|
||
ssh_client.exec_sudo_cmd(
|
||
# ROUTING_STUB
|
||
# "192.168.1.0/24,192.168.111.1,eth0"
|
||
"sed -i $'s@#ROUTING_STUB@\"{},{},"
|
||
"{}\"\\\\n#ROUTING_STUB@g' {}".format(
|
||
target_subnet, via_ip, dev,
|
||
TisInitServiceScript.configuration_path
|
||
), fail_ok=False)
|
||
return True
|
||
|
||
|
||
def route_vm_pair(vm1, vm2, bidirectional=True, validate=True):
|
||
"""
|
||
Route the pair of VMs' data interfaces through internal interfaces
|
||
If multiple interfaces available on either of the VMs, the last one is used
|
||
If no interfaces available for data/internal network for either VM,
|
||
raises IndexError
|
||
The internal interfaces for the pair VM must be on the same gateway
|
||
no fail_ok option, since if failed, the vm's state is undefined
|
||
|
||
Args:
|
||
vm1 (str):
|
||
vm_id, src if bidirectional=False
|
||
vm2 (str):
|
||
vm_id, dest if bidirectional=False
|
||
bidirectional (bool):
|
||
if True, also routes from vm2 to vm1
|
||
validate (bool):
|
||
validate pings between the pair over the data network
|
||
|
||
Returns (dict):
|
||
the interfaces used for routing,
|
||
{vm_id: {'data': {'ip', 'cidr', 'mac'},
|
||
'internal':{'ip', 'cidr', 'mac'}}}
|
||
"""
|
||
if vm1 == vm2:
|
||
raise ValueError("cannot route to a VM itself")
|
||
|
||
auth_info = Tenant.get('admin')
|
||
LOG.info("Collecting VMs' networks")
|
||
interfaces = {
|
||
vm1: {"data": network_helper.get_tenant_ips_for_vms(
|
||
vm1, auth_info=auth_info),
|
||
"internal": network_helper.get_internal_ips_for_vms(vm1)},
|
||
vm2: {"data": network_helper.get_tenant_ips_for_vms(
|
||
vm2, auth_info=auth_info),
|
||
"internal": network_helper.get_internal_ips_for_vms(vm2)},
|
||
}
|
||
|
||
for vm, info in get_vms_ports_info([vm1, vm2]).items():
|
||
for port, ip, cidr, mac, net_id in info:
|
||
# expect one data and one internal
|
||
if ip in interfaces[vm]['data']:
|
||
interfaces[vm]['data'] = {'ip': ip, 'cidr': cidr, 'mac': mac,
|
||
'port': port}
|
||
elif ip in interfaces[vm]['internal']:
|
||
interfaces[vm]['internal'] = {'ip': ip, 'cidr': cidr,
|
||
'mac': mac, 'port': port}
|
||
|
||
if interfaces[vm1]['internal']['cidr'] != \
|
||
interfaces[vm2]['internal']['cidr']:
|
||
raise ValueError(
|
||
"the internal interfaces for the VM pair is not on the same "
|
||
"gateway")
|
||
|
||
vshell_ = _set_vm_route(
|
||
vm1,
|
||
interfaces[vm2]['data']['cidr'], interfaces[vm2]['internal']['ip'],
|
||
interfaces[vm1]['internal']['mac'])
|
||
|
||
if bidirectional:
|
||
_set_vm_route(vm2, interfaces[vm1]['data']['cidr'],
|
||
interfaces[vm1]['internal']['ip'],
|
||
interfaces[vm2]['internal']['mac'])
|
||
|
||
for vm in (vm1, vm2):
|
||
LOG.info("Add vms' data network ip as allowed address for internal "
|
||
"network port")
|
||
network_helper.set_port(
|
||
port_id=interfaces[vm]['internal']['port'],
|
||
auth_info=auth_info,
|
||
allowed_addr_pairs={'ip-address': interfaces[vm]['data']['ip']})
|
||
|
||
if validate:
|
||
LOG.info("Validating route(s) across data")
|
||
ping_between_routed_vms(to_vm=vm2, from_vm=vm1, vshell=vshell_,
|
||
bidirectional=bidirectional)
|
||
|
||
return interfaces
|
||
|
||
|
||
def ping_between_routed_vms(to_vm, from_vm, vshell=True, bidirectional=True,
|
||
timeout=120):
|
||
"""
|
||
Ping between routed vm pair
|
||
Args:
|
||
to_vm:
|
||
from_vm:
|
||
vshell:
|
||
bidirectional:
|
||
timeout:
|
||
|
||
Returns:
|
||
|
||
"""
|
||
ping_vms_from_vm(to_vms=to_vm, from_vm=from_vm, timeout=timeout,
|
||
net_types='data', vshell=vshell,
|
||
source_net_types='internal')
|
||
if bidirectional:
|
||
ping_vms_from_vm(to_vms=from_vm, from_vm=to_vm, timeout=timeout,
|
||
net_types='data', vshell=vshell,
|
||
source_net_types='internal')
|
||
|
||
|
||
def setup_kernel_routing(vm_id, **kwargs):
|
||
"""
|
||
Setup kernel routing function for the specified VM
|
||
replicates the operation as in wrs_guest_setup.sh (and comes
|
||
with the same assumptions)
|
||
in order to persist kernel routing after reboots, the operation has to be
|
||
stored in /etc/init.d
|
||
see TisInitServiceScript for script details
|
||
no fail_ok option, since if failed, the vm's state is undefined
|
||
|
||
Args:
|
||
vm_id (str):
|
||
the VM to be configured
|
||
kwargs (dict):
|
||
kwargs for TisInitServiceScript.configure
|
||
|
||
"""
|
||
LOG.info(
|
||
"Setting up kernel routing for VM {}, kwargs={}".format(vm_id, kwargs))
|
||
|
||
scp_to_vm(vm_id, TisInitServiceScript.src(), TisInitServiceScript.dst())
|
||
with ssh_to_vm_from_natbox(vm_id) as ssh_client:
|
||
r, msg = ssh_client.exec_cmd("cat /proc/sys/net/ipv4/ip_forward",
|
||
fail_ok=False)
|
||
if msg == "1":
|
||
LOG.warn(
|
||
"VM {} has ip_forward enabled already, skipping".format(vm_id))
|
||
return
|
||
TisInitServiceScript.configure(ssh_client, **kwargs)
|
||
TisInitServiceScript.enable(ssh_client)
|
||
TisInitServiceScript.start(ssh_client)
|
||
|
||
|
||
def setup_avr_routing(vm_id, mtu=1500, vm_type='vswitch', **kwargs):
|
||
"""
|
||
Setup avr routing (vswitch L3) function for the specified VM
|
||
replciates the operation as in wrs_guest_setup.sh (and comes with
|
||
the same assumptions)
|
||
in order to persist kernel routing after reboots, the operation has to be
|
||
stored in /etc/init.d
|
||
see TisInitServiceScript for script details
|
||
no fail_ok option, since if failed, the vm's state is undefined
|
||
|
||
Args:
|
||
vm_id (str):
|
||
the VM to be configured
|
||
mtu (int):
|
||
1500 by default
|
||
for jumbo frames (9000), tenant net support is required
|
||
vm_type (str):
|
||
PCI NIC_DEVICE
|
||
vhost: "${PCI_VENDOR_VIRTIO}:${PCI_DEVICE_VIRTIO}:
|
||
${PCI_SUBDEVICE_NET}"
|
||
any other: "${PCI_VENDOR_VIRTIO}:${PCI_DEVICE_MEMORY}:
|
||
${PCI_SUBDEVICE_AVP}" (default)
|
||
kwargs (dict):
|
||
kwargs for TisInitServiceScript.configure
|
||
|
||
"""
|
||
LOG.info(
|
||
"Setting up avr routing for VM {}, kwargs={}".format(vm_id, kwargs))
|
||
datas = network_helper.get_tenant_ips_for_vms(vm_id)
|
||
data_dict = dict()
|
||
try:
|
||
internals = network_helper.get_internal_ips_for_vms(vm_id)
|
||
except ValueError:
|
||
internals = list()
|
||
internal_dict = dict()
|
||
for vm, info in get_vms_ports_info([vm_id]).items():
|
||
for port, ip, cidr, mac, net_id in info:
|
||
if ip in datas:
|
||
data_dict[ip] = ipaddress.ip_network(cidr).netmask
|
||
elif ip in internals:
|
||
internal_dict[ip] = ipaddress.ip_network(cidr).netmask
|
||
|
||
interfaces = list()
|
||
items = list(data_dict.items()) + list(internal_dict.items())
|
||
|
||
if len(items) > 2:
|
||
LOG.warn(
|
||
"wrs_guest_setup/tis_automation_init does not support more than "
|
||
"two DPDK NICs")
|
||
LOG.warn("stripping {} from interfaces".format(items[2:]))
|
||
items = items[:2]
|
||
|
||
for (ip, netmask), ct in zip(items, range(len(items))):
|
||
interfaces.append(
|
||
"""\"{},{},eth{},{}\"""".format(ip, netmask, ct, str(mtu)))
|
||
|
||
nic_device = ""
|
||
if vm_type == 'vhost':
|
||
nic_device = "\"${PCI_VENDOR_VIRTIO}:${PCI_DEVICE_VIRTIO}:" \
|
||
"${PCI_SUBDEVICE_NET}\""
|
||
|
||
scp_to_vm(vm_id, TisInitServiceScript.src(), TisInitServiceScript.dst())
|
||
with ssh_to_vm_from_natbox(vm_id) as ssh_client:
|
||
TisInitServiceScript.configure(
|
||
ssh_client, NIC_DEVICE=nic_device,
|
||
NIC_COUNT=str(len(items)), FUNCTIONS="avr,",
|
||
ROUTES="""(
|
||
#ROUTING_STUB
|
||
)""",
|
||
ADDRESSES="""(
|
||
{}
|
||
)
|
||
""".format("\n ".join(interfaces)), **kwargs)
|
||
TisInitServiceScript.enable(ssh_client)
|
||
TisInitServiceScript.start(ssh_client)
|
||
|
||
|
||
def launch_vm_pair(vm_type='virtio', primary_kwargs=None, secondary_kwargs=None,
|
||
**launch_vms_kwargs):
|
||
"""
|
||
Launch a pair of routed VMs
|
||
one on the primary tenant, and the other on the secondary tenant
|
||
|
||
Args:
|
||
vm_type (str):
|
||
one of 'virtio', 'avp', 'dpdk'
|
||
primary_kwargs (dict):
|
||
launch_vms_kwargs for the VM launched under the primary tenant
|
||
secondary_kwargs (dict):
|
||
launch_vms_kwargs for the VM launched under the secondary tenant
|
||
**launch_vms_kwargs:
|
||
additional keyword arguments for launch_vms for both tenants
|
||
overlapping keys will be overridden by primary_kwargs and
|
||
secondary_kwargs
|
||
shall not specify count, ping_vms, auth_info
|
||
|
||
Returns (tuple):
|
||
(vm_id_on_primary_tenant, vm_id_on_secondary_tenant)
|
||
"""
|
||
LOG.info("Launch a {} test-observer pair of VMs".format(vm_type))
|
||
for invalid_key in ('count', 'ping_vms'):
|
||
if invalid_key in launch_vms_kwargs:
|
||
launch_vms_kwargs.pop(invalid_key)
|
||
|
||
primary_kwargs = dict() if not primary_kwargs else primary_kwargs
|
||
secondary_kwargs = dict() if not secondary_kwargs else secondary_kwargs
|
||
if 'auth_info' not in primary_kwargs:
|
||
primary_kwargs['auth_info'] = Tenant.get_primary()
|
||
if 'auth_info' not in secondary_kwargs:
|
||
secondary_kwargs['auth_info'] = Tenant.get_secondary()
|
||
|
||
if 'nics' not in primary_kwargs or 'nics' not in secondary_kwargs:
|
||
if vm_type in ['pci-sriov', 'pci-passthrough']:
|
||
raise NotImplementedError(
|
||
"nics has to be provided for pci-sriov and pci-passthrough")
|
||
|
||
if vm_type in ['vswitch', 'dpdk', 'vhost']:
|
||
vif_model = 'avp'
|
||
else:
|
||
vif_model = vm_type
|
||
|
||
internal_net_id = network_helper.get_internal_net_id()
|
||
for tenant_info in (primary_kwargs, secondary_kwargs):
|
||
auth_info_ = tenant_info['auth_info']
|
||
mgmt_net_id = network_helper.get_mgmt_net_id(auth_info=auth_info_)
|
||
tenant_net_id = network_helper.get_tenant_net_id(
|
||
auth_info=auth_info_)
|
||
nics = [{'net-id': mgmt_net_id},
|
||
{'net-id': tenant_net_id, 'vif-model': vif_model},
|
||
{'net-id': internal_net_id, 'vif-model': vif_model}]
|
||
tenant_info['nics'] = nics
|
||
|
||
vm_test = launch_vms(vm_type=vm_type, count=1, ping_vms=True,
|
||
**__merge_dict(launch_vms_kwargs, primary_kwargs)
|
||
)[0][0]
|
||
vm_observer = launch_vms(vm_type=vm_type, count=1, ping_vms=True,
|
||
**__merge_dict(launch_vms_kwargs,
|
||
secondary_kwargs))[0][0]
|
||
|
||
LOG.info("Route the {} test-observer VM pair".format(vm_type))
|
||
if vm_type in ('dpdk', 'vhost', 'vswitch'):
|
||
setup_avr_routing(vm_test, vm_type=vm_type)
|
||
setup_avr_routing(vm_observer, vm_type=vm_type)
|
||
else:
|
||
# vm_type in ('virtio', 'avp'):
|
||
setup_kernel_routing(vm_test)
|
||
setup_kernel_routing(vm_observer)
|
||
|
||
route_vm_pair(vm_test, vm_observer)
|
||
|
||
return vm_test, vm_observer
|
||
|
||
|
||
def get_all_vms(field='ID', con_ssh=None, auth_info=Tenant.get('admin')):
|
||
"""
|
||
Get VMs for all tenants in the systems
|
||
|
||
Args:
|
||
field:
|
||
con_ssh:
|
||
auth_info
|
||
|
||
Returns (list): list of all vms on the system
|
||
|
||
"""
|
||
return get_vms(field=field, all_projects=True, long=False, con_ssh=con_ssh,
|
||
auth_info=auth_info)
|
||
|
||
|
||
def get_vms_info(fields, vms=None, con_ssh=None, long=True, all_projects=True,
|
||
host=None,
|
||
auth_info=Tenant.get('admin')):
|
||
"""
|
||
Get vms values for given fields
|
||
Args:
|
||
fields (str|list|tuple):
|
||
vms:
|
||
con_ssh:
|
||
long:
|
||
all_projects:
|
||
host
|
||
auth_info:
|
||
|
||
Returns (dict): vm as key, values for given fields as value
|
||
Examples:
|
||
input: fields = [field1, field2]
|
||
output: {vm_1: [vm1_field1_value, vm1_field2_value],
|
||
vm_2: [vm2_field1_value, vm2_field2_value]}
|
||
|
||
"""
|
||
if isinstance(fields, str):
|
||
fields = (fields,)
|
||
fields = ['ID'] + list(fields)
|
||
|
||
values = get_vms(vms=vms, field=fields, con_ssh=con_ssh, long=long,
|
||
all_projects=all_projects, host=host,
|
||
auth_info=auth_info)
|
||
vm_ids = values.pop(0)
|
||
values = list(zip(*values))
|
||
results = {vm_ids[i]: values[i] for i in range(len(vm_ids))}
|
||
|
||
return results
|
||
|
||
|
||
def get_vms(vms=None, field='ID', long=False, all_projects=True, host=None,
|
||
project=None, project_domain=None,
|
||
strict=True, regex=False, con_ssh=None, auth_info=None, **kwargs):
|
||
"""
|
||
get a list of VM IDs or Names for given tenant in auth_info param.
|
||
|
||
Args:
|
||
vms (list): filter vms from this list if not None
|
||
field (str|tuple|list): 'ID' or 'Name'
|
||
con_ssh (SSHClient): controller SSHClient.
|
||
auth_info (dict): such as ones in auth.py: auth.ADMIN, auth.TENANT1
|
||
long (bool): whether to use --long in cmd
|
||
project (str)
|
||
project_domain (str)
|
||
all_projects (bool): whether to use --a in cmd
|
||
host (str): value for --host arg in cmd
|
||
strict (bool): applies to search for value(s) specified in kwargs
|
||
regex (bool): whether to use regular expression to search for the
|
||
kwargs value(s)
|
||
**kwargs: header/value pair to filter out the vms
|
||
|
||
Returns (list): list of VMs for tenant(s).
|
||
|
||
"""
|
||
args_dict = {'--long': long,
|
||
'--a': all_projects if auth_info and auth_info[
|
||
'user'] == 'admin' else None,
|
||
'--host': host,
|
||
'--project': project,
|
||
'--project-domain': project_domain}
|
||
args = common.parse_args(args_dict)
|
||
table_ = table_parser.table(
|
||
cli.openstack('server list', args, ssh_client=con_ssh,
|
||
auth_info=auth_info)[1])
|
||
if vms:
|
||
table_ = table_parser.filter_table(table_, ID=vms)
|
||
|
||
return table_parser.get_multi_values(table_, field, strict=strict,
|
||
regex=regex, **kwargs)
|
||
|
||
|
||
def get_vm_status(vm_id, con_ssh=None, auth_info=Tenant.get('admin')):
|
||
return get_vm_values(vm_id, 'status', strict=True, con_ssh=con_ssh,
|
||
auth_info=auth_info)[0]
|
||
|
||
|
||
def get_vm_id_from_name(vm_name, con_ssh=None, strict=True, regex=False,
|
||
fail_ok=False, auth_info=Tenant.get('admin')):
|
||
if not auth_info:
|
||
auth_info = Tenant.get_primary()
|
||
vm_ids = get_vms(name=vm_name, strict=strict, regex=regex, con_ssh=con_ssh,
|
||
auth_info=auth_info)
|
||
if not vm_ids:
|
||
err_msg = "No vm found with name: {}".format(vm_name)
|
||
LOG.info(err_msg)
|
||
if fail_ok:
|
||
return ''
|
||
raise exceptions.VMError(err_msg)
|
||
|
||
return vm_ids[0]
|
||
|
||
|
||
def get_vm_name_from_id(vm_id, con_ssh=None, auth_info=None):
|
||
return get_vm_values(vm_id, fields='name', con_ssh=con_ssh,
|
||
auth_info=auth_info)[0]
|
||
|
||
|
||
def get_vm_volumes(vm_id, con_ssh=None, auth_info=None):
|
||
"""
|
||
Get volume ids attached to given vm.
|
||
|
||
Args:
|
||
vm_id (str):
|
||
con_ssh (SSHClient):
|
||
auth_info (dict):
|
||
|
||
Returns (tuple): list of volume ids attached to specific vm
|
||
|
||
"""
|
||
table_ = table_parser.table(
|
||
cli.openstack('server show', vm_id, ssh_client=con_ssh,
|
||
auth_info=auth_info)[1])
|
||
return _get_vm_volumes(table_)
|
||
|
||
|
||
def get_vm_values(vm_id, fields, strict=True, con_ssh=None,
|
||
auth_info=Tenant.get('admin')):
|
||
"""
|
||
Get vm values via openstack server show
|
||
Args:
|
||
vm_id (str):
|
||
fields (str|list|tuple): fields in openstack server show table
|
||
strict (bool): whether to perform a strict search on given field name
|
||
con_ssh (SSHClient):
|
||
auth_info (dict|None):
|
||
|
||
Returns (list): values for given fields
|
||
|
||
"""
|
||
if isinstance(fields, str):
|
||
fields = [fields]
|
||
|
||
table_ = table_parser.table(
|
||
cli.openstack('server show', vm_id, ssh_client=con_ssh,
|
||
auth_info=auth_info)[1])
|
||
|
||
values = []
|
||
for field in fields:
|
||
merge = False
|
||
if field in ('fault',):
|
||
merge = True
|
||
value = table_parser.get_value_two_col_table(table_, field, strict,
|
||
merge_lines=merge)
|
||
if field in ('properties',):
|
||
value = table_parser.convert_value_to_dict(value)
|
||
elif field in ('security_groups',):
|
||
if isinstance(value, str):
|
||
value = [value]
|
||
value = [re.findall("name='(.*)'", v)[0] for v in value]
|
||
values.append(value)
|
||
return values
|
||
|
||
|
||
def get_vm_fault_message(vm_id, con_ssh=None, auth_info=None):
|
||
return get_vm_values(vm_id=vm_id, fields='fault', con_ssh=con_ssh,
|
||
auth_info=auth_info)[0]
|
||
|
||
|
||
def get_vm_flavor(vm_id, field='id', con_ssh=None, auth_info=None):
|
||
"""
|
||
Get flavor id of given vm
|
||
|
||
Args:
|
||
vm_id (str):
|
||
field (str): id or name
|
||
con_ssh (SSHClient):
|
||
auth_info (dict):
|
||
|
||
Returns (str):
|
||
|
||
"""
|
||
flavor = get_vm_values(vm_id, fields='flavor', strict=True, con_ssh=con_ssh,
|
||
auth_info=auth_info)[0]
|
||
flavor_name, flavor_id = flavor.split('(')
|
||
if field == 'id':
|
||
flavor = flavor_id.strip().split(')')[0]
|
||
else:
|
||
flavor = flavor_name.strip()
|
||
return flavor
|
||
|
||
|
||
def get_vm_host(vm_id, con_ssh=None, auth_info=Tenant.get('admin')):
|
||
"""
|
||
Get host of given vm via openstack server show
|
||
Args:
|
||
vm_id:
|
||
con_ssh:
|
||
auth_info
|
||
|
||
Returns (str):
|
||
|
||
"""
|
||
return get_vm_values(vm_id, ':host', strict=False, con_ssh=con_ssh,
|
||
auth_info=auth_info)[0]
|
||
|
||
|
||
def get_vms_hosts(vm_ids, con_ssh=None, auth_info=Tenant.get('admin')):
|
||
"""
|
||
Get vms' hosts via openstack server list
|
||
Args:
|
||
vm_ids:
|
||
con_ssh:
|
||
auth_info
|
||
|
||
Returns:
|
||
|
||
"""
|
||
vms_hosts = get_vms_info(vms=vm_ids, fields='host', auth_info=auth_info,
|
||
con_ssh=con_ssh)
|
||
vms_hosts = [vms_hosts[vm][0] for vm in vm_ids]
|
||
|
||
return vms_hosts
|
||
|
||
|
||
def get_vms_on_host(hostname, field='ID', con_ssh=None,
|
||
auth_info=Tenant.get('admin')):
|
||
"""
|
||
Get vms on given host
|
||
Args:
|
||
field: ID or Name
|
||
hostname (str):Name of a compute node
|
||
con_ssh:
|
||
auth_info
|
||
|
||
Returns (list): A list of VMs' ID under a hypervisor
|
||
|
||
"""
|
||
vms = get_vms(host=hostname, all_projects=True, long=False, con_ssh=con_ssh,
|
||
auth_info=auth_info, field=field)
|
||
return vms
|
||
|
||
|
||
def get_vms_per_host(vms=None, con_ssh=None, auth_info=Tenant.get('admin')):
|
||
"""
|
||
Get vms per host
|
||
Args:
|
||
vms
|
||
con_ssh (SSHClient):
|
||
auth_info (dict)
|
||
|
||
Returns (dict):return a dictionary where the host(hypervisor) is the key
|
||
and value are a list of VMs under the host
|
||
|
||
"""
|
||
vms_hosts = get_vms_info(vms=vms, fields='host', auth_info=auth_info,
|
||
con_ssh=con_ssh, long=True, all_projects=True)
|
||
vms_per_host = {}
|
||
for vm in vms_hosts:
|
||
host = vms_hosts[vm][0]
|
||
if host in vms_per_host:
|
||
vms_per_host[host].append(vm)
|
||
else:
|
||
vms_per_host[host] = [vm]
|
||
|
||
return vms_per_host
|
||
|
||
|
||
def _get_boot_info(table_, vm_id, auth_info=None, con_ssh=None):
|
||
image = table_parser.get_value_two_col_table(table_, 'image')
|
||
if not image:
|
||
volumes = _get_vm_volumes(table_)
|
||
if len(volumes) == 0:
|
||
raise exceptions.VMError(
|
||
"Booted from volume, but no volume id found.")
|
||
|
||
from keywords import cinder_helper
|
||
if len(volumes) == 1:
|
||
vol_id = volumes[0]
|
||
vol_name, image_info = cinder_helper.get_volume_show_values(
|
||
vol_id, auth_info=auth_info, con_ssh=con_ssh,
|
||
fields=('name', 'volume_image_metadata'))
|
||
LOG.info("VM booted from volume.")
|
||
return {'type': 'volume', 'id': vol_id, 'volume_name': vol_name,
|
||
'image_name': image_info['image_name']}
|
||
else:
|
||
LOG.info(
|
||
"VM booted from volume. Multiple volumes found, taking the "
|
||
"first boot-able volume.")
|
||
for volume in volumes:
|
||
bootable, vol_name, image_info = \
|
||
cinder_helper.get_volume_show_values(
|
||
volume,
|
||
fields=('bootable', 'name', 'volume_image_metadata'),
|
||
auth_info=auth_info, con_ssh=con_ssh)
|
||
if str(bootable).lower() == 'true':
|
||
return {'type': 'volume', 'id': volume,
|
||
'volume_name': vol_name,
|
||
'image_name': image_info['image_name']}
|
||
|
||
raise exceptions.VMError(
|
||
"VM {} has no bootable volume attached.".format(vm_id))
|
||
|
||
else:
|
||
name, img_uuid = image.strip().split(sep='(')
|
||
return {'type': 'image', 'id': img_uuid.split(sep=')')[0],
|
||
'image_name': name.strip()}
|
||
|
||
|
||
def get_vm_boot_info(vm_id, auth_info=None, con_ssh=None):
|
||
"""
|
||
Get vm boot source and id.
|
||
|
||
Args:
|
||
vm_id (str):
|
||
auth_info (dict|None):
|
||
con_ssh (SSHClient):
|
||
|
||
Returns (dict): VM boot info dict.
|
||
Format: {'type': <boot_source>, 'id': <source_id>}.
|
||
<boot_source> is either 'volume' or 'image'
|
||
|
||
"""
|
||
table_ = table_parser.table(
|
||
cli.openstack('server show', vm_id, ssh_client=con_ssh,
|
||
auth_info=auth_info)[1])
|
||
return _get_boot_info(table_, vm_id=vm_id, auth_info=auth_info,
|
||
con_ssh=con_ssh)
|
||
|
||
|
||
def get_vm_image_name(vm_id, auth_info=None, con_ssh=None):
|
||
"""
|
||
|
||
Args:
|
||
vm_id (str):
|
||
auth_info (dict):
|
||
con_ssh (SSHClient):
|
||
|
||
Returns (str): image name for the vm. If vm booted from volume,
|
||
then image name in volume image metadata will be returned.
|
||
|
||
"""
|
||
boot_info = get_vm_boot_info(vm_id, auth_info=auth_info, con_ssh=con_ssh)
|
||
|
||
return boot_info['image_name']
|
||
|
||
|
||
def _get_vm_volumes(table_):
|
||
"""
|
||
Args:
|
||
table_ (dict):
|
||
|
||
Returns (list: A list of volume ids from the novashow_table.
|
||
|
||
"""
|
||
volumes = table_parser.get_value_two_col_table(table_, 'volumes_attached',
|
||
merge_lines=False)
|
||
if not volumes:
|
||
return []
|
||
|
||
if isinstance(volumes, str):
|
||
volumes = [volumes]
|
||
|
||
return [re.findall("id='(.*)'", volume)[0] for volume in volumes]
|
||
|
||
|
||
def get_vm_instance_name(vm_id, con_ssh=None):
|
||
return get_vm_values(vm_id, ":instance_name", strict=False,
|
||
con_ssh=con_ssh)[0]
|