579 lines
25 KiB
Python
579 lines
25 KiB
Python
# Copyright (c) 2014 NetApp, Inc.
|
|
# All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
"""Module for managing nova instances for share drivers."""
|
|
|
|
import netaddr
|
|
import os
|
|
import socket
|
|
import threading
|
|
import time
|
|
|
|
from oslo.config import cfg
|
|
|
|
from manila import compute
|
|
from manila import context
|
|
from manila import exception
|
|
from manila.network.linux import ip_lib
|
|
from manila.network.neutron import api as neutron
|
|
from manila.openstack.common import importutils
|
|
from manila.openstack.common import log as logging
|
|
from manila import utils
|
|
|
|
|
|
LOG = logging.getLogger(__name__)
|
|
|
|
server_opts = [
|
|
cfg.StrOpt('service_image_name',
|
|
default='manila-service-image',
|
|
help="Name of image in glance, that will be used to create "
|
|
"service instance."),
|
|
cfg.StrOpt('service_instance_name_template',
|
|
default='manila_service_instance-%s',
|
|
help="Name of service instance."),
|
|
cfg.StrOpt('service_instance_user',
|
|
help="User in service instance."),
|
|
cfg.StrOpt('service_instance_password',
|
|
default=None,
|
|
help="Password to service instance user."),
|
|
cfg.StrOpt('manila_service_keypair_name',
|
|
default='manila-service',
|
|
help="Name of keypair that will be created and used "
|
|
"for service instance."),
|
|
cfg.StrOpt('path_to_public_key',
|
|
default='~/.ssh/id_rsa.pub',
|
|
help="Path to hosts public key."),
|
|
cfg.StrOpt('path_to_private_key',
|
|
default='~/.ssh/id_rsa',
|
|
help="Path to hosts private key."),
|
|
cfg.IntOpt('max_time_to_build_instance',
|
|
default=300,
|
|
help="Maximum time to wait for creating service instance."),
|
|
cfg.IntOpt('service_instance_flavor_id',
|
|
default=100,
|
|
help="ID of flavor, that will be used for service instance "
|
|
"creation."),
|
|
cfg.StrOpt('service_network_name',
|
|
default='manila_service_network',
|
|
help="Name of manila service network."),
|
|
cfg.StrOpt('service_network_cidr',
|
|
default='10.254.0.0/16',
|
|
help="CIDR of manila service network."),
|
|
cfg.StrOpt('interface_driver',
|
|
default='manila.network.linux.interface.OVSInterfaceDriver',
|
|
help="Vif driver."),
|
|
]
|
|
|
|
CONF = cfg.CONF
|
|
CONF.register_opts(server_opts)
|
|
|
|
|
|
def synchronized(f):
|
|
"""Decorates function with unique locks for each share network.
|
|
|
|
Share network id must be provided either as value/attribute
|
|
of one of args or as named argument.
|
|
"""
|
|
def wrapped_func(self, *args, **kwargs):
|
|
share_network_id = kwargs.get('share_network_id', None)
|
|
if not share_network_id:
|
|
for arg in args:
|
|
share_network_id = getattr(arg, 'share_network_id', None)
|
|
if isinstance(arg, dict):
|
|
share_network_id = arg.get('share_network_id', None)
|
|
if share_network_id:
|
|
break
|
|
else:
|
|
raise exception.ServiceInstanceException(_('Could not get '
|
|
'share network id'))
|
|
with self.share_networks_locks.setdefault(share_network_id,
|
|
threading.RLock()):
|
|
return f(self, *args, **kwargs)
|
|
return wrapped_func
|
|
|
|
|
|
def _ssh_exec(server, command):
|
|
"""Executes ssh commands and checks/restores ssh connection."""
|
|
if not server['ssh'].get_transport().is_active():
|
|
server['ssh_pool'].remove(server['ssh'])
|
|
server['ssh'] = server['ssh_pool'].create()
|
|
return utils.ssh_execute(server['ssh'], ' '.join(command))
|
|
|
|
|
|
class ServiceInstanceManager(object):
|
|
"""Manages nova instances for various share drivers."""
|
|
|
|
def __init__(self, db, _helpers, *args, **kwargs):
|
|
"""Do initialization."""
|
|
super(ServiceInstanceManager, self).__init__(*args, **kwargs)
|
|
if not CONF.service_instance_user:
|
|
raise exception.ServiceInstanceException(_('Service instance user '
|
|
'is not specified'))
|
|
self.admin_context = context.get_admin_context()
|
|
self._execute = utils.execute
|
|
self.compute_api = compute.API()
|
|
self.neutron_api = neutron.API()
|
|
self._helpers = _helpers
|
|
self.db = db
|
|
attempts = 5
|
|
while attempts:
|
|
try:
|
|
self.service_tenant_id = self.neutron_api.admin_tenant_id
|
|
break
|
|
except exception.NetworkException:
|
|
LOG.debug(_('Connection to neutron failed.'))
|
|
attempts -= 1
|
|
time.sleep(3)
|
|
else:
|
|
raise exception.ServiceInstanceException(_('Can not receive '
|
|
'service tenant id.'))
|
|
self.share_networks_locks = {}
|
|
self.share_networks_servers = {}
|
|
self.service_network_id = self._get_service_network()
|
|
self.vif_driver = importutils.import_class(CONF.interface_driver)()
|
|
self._setup_connectivity_with_service_instances()
|
|
|
|
def _get_service_network(self):
|
|
"""Finds existing or creates new service network."""
|
|
service_network_name = CONF.service_network_name
|
|
networks = [network for network in self.neutron_api.
|
|
get_all_tenant_networks(self.service_tenant_id)
|
|
if network['name'] == service_network_name]
|
|
if len(networks) > 1:
|
|
raise exception.ServiceInstanceException(_('Ambiguous service '
|
|
'networks.'))
|
|
elif not networks:
|
|
return self.neutron_api.network_create(self.service_tenant_id,
|
|
service_network_name)['id']
|
|
else:
|
|
return networks[0]['id']
|
|
|
|
def _get_service_instance_name(self, share_network_id):
|
|
"""Returns service vms name."""
|
|
return CONF.service_instance_name_template % share_network_id
|
|
|
|
def _get_server_ip(self, server):
|
|
"""Returns service vms ip address."""
|
|
net = server['networks']
|
|
try:
|
|
net_ips = net[CONF.service_network_name]
|
|
return net_ips[0]
|
|
except KeyError:
|
|
msg = _('Service vm is not attached to %s network.')
|
|
except IndexError:
|
|
msg = _('Service vm has no ips on %s network.')
|
|
msg = msg % CONF.service_network_name
|
|
LOG.error(msg)
|
|
raise exception.ServiceInstanceException(msg)
|
|
|
|
def _ensure_server(self, context, server, update=False):
|
|
"""Ensures that server exists and active, otherwise deletes it."""
|
|
if not server:
|
|
return False
|
|
if update:
|
|
try:
|
|
server.update(self.compute_api.server_get(context,
|
|
server['id']))
|
|
except exception.InstanceNotFound as e:
|
|
LOG.debug(e)
|
|
return False
|
|
if server['status'] == 'ACTIVE':
|
|
if self._check_server_availability(server):
|
|
return True
|
|
|
|
return False
|
|
|
|
def _delete_server(self, context, server):
|
|
"""Deletes the server."""
|
|
if not server:
|
|
return
|
|
self.compute_api.server_delete(context, server['id'])
|
|
t = time.time()
|
|
while time.time() - t < CONF.max_time_to_build_instance:
|
|
try:
|
|
server = self.compute_api.server_get(context, server['id'])
|
|
except exception.InstanceNotFound:
|
|
LOG.debug(_('Service instance was deleted succesfully.'))
|
|
break
|
|
time.sleep(1)
|
|
else:
|
|
raise exception.ServiceInstanceException(_('Instance have not '
|
|
'been deleted in %ss. Giving up.') %
|
|
CONF.max_time_to_build_instance)
|
|
|
|
@synchronized
|
|
def get_service_instance(self, context, share_network_id, create=True):
|
|
"""Finds or creates and sets up service vm."""
|
|
server = self.share_networks_servers.get(share_network_id, {})
|
|
old_server_ip = server.get('ip', None)
|
|
if self._ensure_server(context, server, update=True):
|
|
return server
|
|
else:
|
|
self._delete_server(context, server)
|
|
server = {}
|
|
service_instance_name = self._get_service_instance_name(
|
|
share_network_id)
|
|
search_opts = {'name': service_instance_name}
|
|
servers = self.compute_api.server_list(context, search_opts, True)
|
|
if len(servers) == 1:
|
|
server = servers[0]
|
|
server['ip'] = self._get_server_ip(server)
|
|
old_server_ip = server['ip']
|
|
if not self._ensure_server(context, server):
|
|
self._delete_server(context, server)
|
|
server.clear()
|
|
elif len(servers) > 1:
|
|
raise exception.ServiceInstanceException(
|
|
_('Error. Ambiguous service instances.'))
|
|
if not server and create:
|
|
server = self._create_service_instance(context,
|
|
service_instance_name,
|
|
share_network_id,
|
|
old_server_ip)
|
|
if server:
|
|
server['share_network_id'] = share_network_id
|
|
server['ip'] = self._get_server_ip(server)
|
|
server['ssh_pool'] = self._get_ssh_pool(server)
|
|
server['ssh'] = server['ssh_pool'].create()
|
|
for helper in self._helpers.values():
|
|
helper.init_helper(server)
|
|
|
|
self.share_networks_servers[share_network_id] = server
|
|
return server
|
|
|
|
def _get_ssh_pool(self, server):
|
|
"""Returns ssh connection pool for service vm."""
|
|
ssh_pool = utils.SSHPool(server['ip'], 22, None,
|
|
CONF.service_instance_user,
|
|
password=CONF.service_instance_password,
|
|
privatekey=CONF.path_to_private_key,
|
|
max_size=1)
|
|
return ssh_pool
|
|
|
|
def _get_key(self, context):
|
|
"""Returns name of key, that will be injected to service vm."""
|
|
if not CONF.path_to_public_key or not CONF.path_to_private_key:
|
|
return
|
|
path_to_public_key = os.path.expanduser(CONF.path_to_public_key)
|
|
path_to_private_key = os.path.expanduser(CONF.path_to_private_key)
|
|
if (not os.path.exists(path_to_public_key) or
|
|
not os.path.exists(path_to_private_key)):
|
|
return
|
|
keypair_name = CONF.manila_service_keypair_name
|
|
keypairs = [k for k in self.compute_api.keypair_list(context)
|
|
if k.name == keypair_name]
|
|
if len(keypairs) > 1:
|
|
raise exception.ServiceInstanceException(_('Ambiguous keypairs.'))
|
|
|
|
public_key, __ = self._execute('cat', path_to_public_key)
|
|
if not keypairs:
|
|
keypair = self.compute_api.keypair_import(context,
|
|
keypair_name,
|
|
public_key)
|
|
else:
|
|
keypair = keypairs[0]
|
|
if keypair.public_key != public_key:
|
|
LOG.debug(_('Public key differs from existing keypair. '
|
|
'Creating new keypair.'))
|
|
self.compute_api.keypair_delete(context, keypair.id)
|
|
keypair = self.compute_api.keypair_import(context,
|
|
keypair_name,
|
|
public_key)
|
|
return keypair.name
|
|
|
|
def _get_service_image(self, context):
|
|
"""Returns ID of service image for service vm creating."""
|
|
images = [image.id for image in self.compute_api.image_list(context)
|
|
if image.name == CONF.service_image_name]
|
|
if len(images) == 1:
|
|
return images[0]
|
|
elif not images:
|
|
raise exception.ServiceInstanceException(_('No appropriate '
|
|
'image was found.'))
|
|
else:
|
|
raise exception.ServiceInstanceException(
|
|
_('Ambiguous image name.'))
|
|
|
|
def _create_service_instance(self, context, instance_name,
|
|
share_network_id, old_server_ip):
|
|
"""Creates service vm and sets up networking for it."""
|
|
service_image_id = self._get_service_image(context)
|
|
key_name = self._get_key(context)
|
|
if not CONF.service_instance_password and not key_name:
|
|
raise exception.ServiceInstanceException(
|
|
_('Neither service instance password nor key are available.'))
|
|
|
|
port = self._setup_network_for_instance(context,
|
|
share_network_id,
|
|
old_server_ip)
|
|
try:
|
|
self._setup_connectivity_with_service_instances()
|
|
except Exception as e:
|
|
LOG.debug(e)
|
|
self.neutron_api.delete_port(port['id'])
|
|
raise
|
|
service_instance = self.compute_api.server_create(context,
|
|
instance_name,
|
|
service_image_id,
|
|
CONF.service_instance_flavor_id,
|
|
key_name,
|
|
None,
|
|
None,
|
|
nics=[{'port-id': port['id']}])
|
|
|
|
t = time.time()
|
|
while time.time() - t < CONF.max_time_to_build_instance:
|
|
if service_instance['status'] == 'ACTIVE':
|
|
break
|
|
if service_instance['status'] == 'ERROR':
|
|
raise exception.ServiceInstanceException(
|
|
_('Failed to build service instance.'))
|
|
time.sleep(1)
|
|
try:
|
|
service_instance = self.compute_api.server_get(context,
|
|
service_instance['id'])
|
|
except exception.InstanceNotFound as e:
|
|
LOG.debug(e)
|
|
else:
|
|
raise exception.ServiceInstanceException(
|
|
_('Instance have not been spawned in %ss. Giving up.') %
|
|
CONF.max_time_to_build_instance)
|
|
|
|
service_instance['ip'] = self._get_server_ip(service_instance)
|
|
if not self._check_server_availability(service_instance):
|
|
raise exception.ServiceInstanceException(
|
|
_('SSH connection have not been '
|
|
'established in %ss. Giving up.') %
|
|
CONF.max_time_to_build_instance)
|
|
return service_instance
|
|
|
|
def _check_server_availability(self, server):
|
|
t = time.time()
|
|
while time.time() - t < CONF.max_time_to_build_instance:
|
|
LOG.debug(_('Checking service vm availablity.'))
|
|
try:
|
|
socket.socket().connect((server['ip'], 22))
|
|
LOG.debug(_('Service vm is available via ssh.'))
|
|
return True
|
|
except socket.error as e:
|
|
LOG.debug(e)
|
|
LOG.debug(_('Server is not available through ssh. Waiting...'))
|
|
time.sleep(5)
|
|
return False
|
|
|
|
def _setup_network_for_instance(self, context, share_network_id,
|
|
old_server_ip):
|
|
"""Sets up network for service vm."""
|
|
service_subnet = self._get_service_subnet(share_network_id)
|
|
if not service_subnet:
|
|
service_subnet = self.neutron_api.subnet_create(
|
|
self.service_tenant_id,
|
|
self.service_network_id,
|
|
share_network_id,
|
|
self._get_cidr_for_subnet())
|
|
|
|
private_router = self._get_private_router(share_network_id)
|
|
try:
|
|
self.neutron_api.router_add_interface(private_router['id'],
|
|
service_subnet['id'])
|
|
except exception.NetworkException as e:
|
|
if e.kwargs['code'] != 400:
|
|
raise
|
|
LOG.debug(_('Subnet %(subnet_id)s is already attached to the '
|
|
'router %(router_id)s.') %
|
|
{'subnet_id': service_subnet['id'],
|
|
'router_id': private_router['id']})
|
|
|
|
return self.neutron_api.create_port(self.service_tenant_id,
|
|
self.service_network_id,
|
|
subnet_id=service_subnet['id'],
|
|
fixed_ip=old_server_ip,
|
|
device_owner='manila')
|
|
|
|
def _get_private_router(self, share_network_id):
|
|
"""Returns router attached to private subnet gateway."""
|
|
share_network = self.db.share_network_get(self.admin_context,
|
|
share_network_id)
|
|
private_subnet = self.neutron_api.get_subnet(
|
|
share_network['neutron_subnet_id'])
|
|
if not private_subnet['gateway_ip']:
|
|
raise exception.ServiceInstanceException(
|
|
_('Subnet must have gateway.'))
|
|
private_network_ports = [p for p in self.neutron_api.list_ports(
|
|
network_id=share_network['neutron_net_id'])]
|
|
for p in private_network_ports:
|
|
fixed_ip = p['fixed_ips'][0]
|
|
if (fixed_ip['subnet_id'] == private_subnet['id'] and
|
|
fixed_ip['ip_address'] == private_subnet['gateway_ip']):
|
|
private_subnet_gateway_port = p
|
|
break
|
|
else:
|
|
raise exception.ServiceInstanceException(
|
|
_('Subnet gateway is not attached the router.'))
|
|
private_subnet_router = self.neutron_api.show_router(
|
|
private_subnet_gateway_port['device_id'])
|
|
return private_subnet_router
|
|
|
|
def _setup_connectivity_with_service_instances(self):
|
|
"""Sets up connectivity with service instances.
|
|
|
|
Creates creating port in service network, creating and setting up
|
|
required network devices.
|
|
"""
|
|
port = self._get_service_port()
|
|
port = self._add_fixed_ips_to_service_port(port)
|
|
interface_name = self.vif_driver.get_device_name(port)
|
|
self.vif_driver.plug(port['id'], interface_name, port['mac_address'])
|
|
ip_cidrs = []
|
|
for fixed_ip in port['fixed_ips']:
|
|
subnet = self.neutron_api.get_subnet(fixed_ip['subnet_id'])
|
|
net = netaddr.IPNetwork(subnet['cidr'])
|
|
ip_cidr = '%s/%s' % (fixed_ip['ip_address'], net.prefixlen)
|
|
ip_cidrs.append(ip_cidr)
|
|
|
|
self.vif_driver.init_l3(interface_name, ip_cidrs)
|
|
|
|
# ensure that interface is first in the list
|
|
device = ip_lib.IPDevice(interface_name)
|
|
device.route.pullup_route(interface_name)
|
|
|
|
# here we are checking for garbage devices from removed service port
|
|
self._remove_outdated_interfaces(device)
|
|
|
|
def _remove_outdated_interfaces(self, device):
|
|
"""Finds and removes unused network device."""
|
|
list_dev = []
|
|
for dev in ip_lib.IPWrapper().get_devices():
|
|
if dev.name != device.name and dev.name[:3] == device.name[:3]:
|
|
cidr_set = set()
|
|
for a in dev.addr.list():
|
|
if a['ip_version'] == 4:
|
|
cidr_set.add(str(netaddr.IPNetwork(a['cidr']).cidr))
|
|
list_dev.append((dev.name, cidr_set))
|
|
device_cidr_set = set(str(netaddr.IPNetwork(a['cidr']).cidr)
|
|
for a in device.addr.list()
|
|
if a['ip_version'] == 4)
|
|
|
|
for dev_name, cidr_set in list_dev:
|
|
if device_cidr_set & cidr_set:
|
|
self.vif_driver.unplug(dev_name)
|
|
|
|
def _get_service_port(self):
|
|
"""Find or creates service neutron port.
|
|
|
|
This port will be used for connectivity with service instances.
|
|
"""
|
|
ports = [port for port in self.neutron_api.
|
|
list_ports(device_id='manila-share')]
|
|
if len(ports) > 1:
|
|
raise exception.ServiceInstanceException(
|
|
_('Error. Ambiguous service ports.'))
|
|
elif not ports:
|
|
try:
|
|
stdout, stderr = self._execute('hostname')
|
|
host = stdout.strip()
|
|
except exception.ProcessExecutionError as e:
|
|
msg = _('Unable to get host. %s') % e.stderr
|
|
raise exception.ManilaException(msg)
|
|
port = self.neutron_api.create_port(self.service_tenant_id,
|
|
self.service_network_id,
|
|
device_id='manila-share',
|
|
device_owner='manila:share',
|
|
host_id=host)
|
|
else:
|
|
port = ports[0]
|
|
return port
|
|
|
|
def _add_fixed_ips_to_service_port(self, port):
|
|
network = self.neutron_api.get_network(self.service_network_id)
|
|
subnets = set(network['subnets'])
|
|
port_fixed_ips = []
|
|
for fixed_ip in port['fixed_ips']:
|
|
port_fixed_ips.append({'subnet_id': fixed_ip['subnet_id'],
|
|
'ip_address': fixed_ip['ip_address']})
|
|
if fixed_ip['subnet_id'] in subnets:
|
|
subnets.remove(fixed_ip['subnet_id'])
|
|
|
|
# If there are subnets here that means that
|
|
# we need to add those to the port and call update.
|
|
if subnets:
|
|
port_fixed_ips.extend([dict(subnet_id=s) for s in subnets])
|
|
port = self.neutron_api.update_port_fixed_ips(
|
|
port['id'], {'fixed_ips': port_fixed_ips})
|
|
|
|
return port
|
|
|
|
def _remove_fixed_ip_from_service_port(self, port, subnet_id):
|
|
port_fixed_ips = []
|
|
for fixed_ip in port['fixed_ips']:
|
|
if fixed_ip['subnet_id'] == subnet_id:
|
|
continue
|
|
port_fixed_ips.append({'subnet_id': fixed_ip['subnet_id'],
|
|
'ip_address': fixed_ip['ip_address']})
|
|
|
|
if port_fixed_ips != port['fixed_ips']:
|
|
port = self.neutron_api.update_port_fixed_ips(
|
|
port['id'], {'fixed_ips': port_fixed_ips})
|
|
|
|
return port
|
|
|
|
def _get_cidr_for_subnet(self):
|
|
"""Returns not used cidr for service subnet creating."""
|
|
subnets = self._get_all_service_subnets()
|
|
used_cidrs = set(subnet['cidr'] for subnet in subnets)
|
|
serv_cidr = netaddr.IPNetwork(CONF.service_network_cidr)
|
|
for subnet in serv_cidr.subnet(29):
|
|
cidr = str(subnet.cidr)
|
|
if cidr not in used_cidrs:
|
|
return cidr
|
|
else:
|
|
raise exception.ServiceInstanceException(_('No available cidrs.'))
|
|
|
|
def delete_share_infrastructure(self, context, share_network_id):
|
|
"""Removes share infrastructure.
|
|
|
|
Deletes service vm and subnet, associated to share network.
|
|
"""
|
|
server = self.get_service_instance(context,
|
|
share_network_id=share_network_id,
|
|
create=False)
|
|
if server:
|
|
self._delete_server(context, server)
|
|
subnet_id = self._get_service_subnet(share_network_id)
|
|
if subnet_id:
|
|
router = self._get_private_router(share_network_id)
|
|
port = self._get_service_port()
|
|
self.neutron_api.router_remove_interface(router['id'], subnet_id)
|
|
self._remove_fixed_ip_from_service_port(port, subnet_id)
|
|
self.neutron_api.delete_subnet(subnet_id)
|
|
self._setup_connectivity_with_service_instances()
|
|
|
|
def _get_all_service_subnets(self):
|
|
service_network = self.neutron_api.get_network(self.service_network_id)
|
|
return [self.neutron_api.get_subnet(subnet_id)
|
|
for subnet_id in service_network['subnets']]
|
|
|
|
def _get_service_subnet(self, share_network_id):
|
|
all_service_subnets = self._get_all_service_subnets()
|
|
service_subnets = [subnet for subnet in all_service_subnets
|
|
if subnet['name'] == share_network_id]
|
|
if len(service_subnets) == 1:
|
|
return service_subnets[0]
|
|
elif not service_subnets:
|
|
return None
|
|
else:
|
|
raise exception.ServiceInstanceException(_('Ambiguous service '
|
|
'subnets.'))
|