295 lines
12 KiB
Python
295 lines
12 KiB
Python
# All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
from time import sleep
|
|
|
|
from kuryr.lib import constants as kl_const
|
|
from kuryr.lib import exceptions as kl_exc
|
|
from kuryr.lib import segmentation_type_drivers as seg_driver
|
|
from openstack import exceptions as os_exc
|
|
from oslo_config import cfg
|
|
from oslo_log import log as logging
|
|
|
|
from kuryr_kubernetes import clients
|
|
from kuryr_kubernetes import config
|
|
from kuryr_kubernetes import constants
|
|
from kuryr_kubernetes.controller.drivers import nested_vif
|
|
from kuryr_kubernetes.controller.drivers import utils
|
|
from kuryr_kubernetes import exceptions as k_exc
|
|
from kuryr_kubernetes import os_vif_util as ovu
|
|
|
|
|
|
LOG = logging.getLogger(__name__)
|
|
|
|
DEFAULT_MAX_RETRY_COUNT = 3
|
|
DEFAULT_RETRY_INTERVAL = 1
|
|
ACTIVE_TIMEOUT = 90
|
|
|
|
CONF = cfg.CONF
|
|
|
|
|
|
class NestedVlanPodVIFDriver(nested_vif.NestedPodVIFDriver):
|
|
"""Manages ports for nested-containers using VLANs to provide VIFs."""
|
|
|
|
def request_vif(self, pod, project_id, subnets, security_groups):
|
|
os_net = clients.get_network_client()
|
|
parent_port = self._get_parent_port(pod)
|
|
trunk_id = self._get_trunk_id(parent_port)
|
|
|
|
rq = self._get_port_request(pod, project_id, subnets, security_groups)
|
|
port = os_net.create_port(**rq)
|
|
self._check_port_binding([port])
|
|
if not self._tag_on_creation:
|
|
utils.tag_neutron_resources([port])
|
|
vlan_id = self._add_subport(trunk_id, port.id)
|
|
|
|
return ovu.neutron_to_osvif_vif_nested_vlan(port, subnets, vlan_id)
|
|
|
|
def request_vifs(self, pod, project_id, subnets, security_groups,
|
|
num_ports, trunk_ip=None):
|
|
"""This method creates subports and returns a list with their vifs.
|
|
|
|
It creates up to num_ports subports and attaches them to the trunk
|
|
port.
|
|
|
|
If not enough vlan ids are available for all the subports to create,
|
|
it creates as much as available vlan ids.
|
|
|
|
Note the os_net add_trunk_subports is an atomic operation that will
|
|
either attach all or none of the subports. Therefore, if there is a
|
|
vlan id collision, all the created ports will be deleted and the
|
|
exception is raised.
|
|
"""
|
|
os_net = clients.get_network_client()
|
|
if trunk_ip:
|
|
parent_port = self._get_parent_port_by_host_ip(trunk_ip)
|
|
else:
|
|
parent_port = self._get_parent_port(pod)
|
|
trunk_id = self._get_trunk_id(parent_port)
|
|
|
|
port_rq, subports_info = self._create_subports_info(
|
|
pod, project_id, subnets, security_groups,
|
|
trunk_id, num_ports, unbound=True)
|
|
|
|
if not subports_info:
|
|
LOG.error("There are no vlan ids available to create subports")
|
|
return []
|
|
|
|
bulk_port_rq = {'ports': [port_rq] * len(subports_info)}
|
|
try:
|
|
ports = list(os_net.create_ports(bulk_port_rq))
|
|
except os_exc.SDKException:
|
|
LOG.exception("Error creating bulk ports: %s", bulk_port_rq)
|
|
raise
|
|
self._check_port_binding(ports)
|
|
if not self._tag_on_creation:
|
|
utils.tag_neutron_resources(ports)
|
|
|
|
for index, port in enumerate(ports):
|
|
subports_info[index]['port_id'] = port['id']
|
|
|
|
try:
|
|
try:
|
|
os_net.add_trunk_subports(trunk_id, subports_info)
|
|
except os_exc.ConflictException:
|
|
LOG.error("vlan ids already in use on trunk")
|
|
for port in ports:
|
|
os_net.delete_port(port.id)
|
|
return []
|
|
except os_exc.SDKException:
|
|
LOG.exception("Error happened during subport addition to trunk")
|
|
for port in ports:
|
|
os_net.delete_port(port.id)
|
|
return []
|
|
|
|
vifs = []
|
|
for index, port in enumerate(ports):
|
|
vlan_id = subports_info[index]['segmentation_id']
|
|
vif = ovu.neutron_to_osvif_vif_nested_vlan(port, subnets, vlan_id)
|
|
vifs.append(vif)
|
|
return vifs
|
|
|
|
def activate_vif(self, vif, pod=None, retry_info=None):
|
|
try:
|
|
super().activate_vif(vif)
|
|
except k_exc.ResourceNotReady:
|
|
if retry_info and retry_info.get('elapsed', 0) > ACTIVE_TIMEOUT:
|
|
parent_port = self._get_parent_port(pod)
|
|
trunk_id = self._get_trunk_id(parent_port)
|
|
# NOTE(dulek): We don't need a lock to prevent VLAN ID from
|
|
# being taken over because the SegmentationDriver
|
|
# will keep it reserved in memory unless we
|
|
# release it. And we won't.
|
|
LOG.warning('Subport %s is in DOWN status for more than %d '
|
|
'seconds. This is a Neutron issue. Attempting to '
|
|
'reattach the subport to trunk %s using VLAN ID %s'
|
|
' to fix it.', vif.id, retry_info['elapsed'],
|
|
trunk_id, vif.vlan_id)
|
|
try:
|
|
self._remove_subport(trunk_id, vif.id)
|
|
except os_exc.NotFoundException:
|
|
# NOTE(dulek): This may happen when _add_subport() failed
|
|
# or Kuryr crashed between the calls. Let's
|
|
# try to fix it hoping that VLAN ID is still
|
|
# free.
|
|
LOG.warning('Subport %s was not attached to the trunk. '
|
|
'Trying to attach it anyway.', vif.id)
|
|
self._add_subport(trunk_id, vif.id,
|
|
requested_vlan_id=vif.vlan_id)
|
|
LOG.warning("Reattached subport %s, its state will be "
|
|
"rechecked when event will be retried.", vif.id)
|
|
raise
|
|
|
|
def release_vif(self, pod, vif, project_id=None, security_groups=None):
|
|
os_net = clients.get_network_client()
|
|
parent_port = self._get_parent_port(pod)
|
|
trunk_id = self._get_trunk_id(parent_port)
|
|
self._remove_subport(trunk_id, vif.id)
|
|
self._release_vlan_id(vif.vlan_id)
|
|
os_net.delete_port(vif.id)
|
|
|
|
def _get_port_request(self, pod, project_id, subnets, security_groups,
|
|
unbound=False):
|
|
port_req_body = {'project_id': project_id,
|
|
'network_id': utils.get_network_id(subnets),
|
|
'fixed_ips': ovu.osvif_to_neutron_fixed_ips(subnets),
|
|
'device_owner': kl_const.DEVICE_OWNER,
|
|
'admin_state_up': True}
|
|
|
|
# only set name if port_debug is enabled
|
|
if config.CONF.kubernetes.port_debug:
|
|
if unbound:
|
|
port_req_body['name'] = constants.KURYR_PORT_NAME
|
|
else:
|
|
port_req_body['name'] = utils.get_port_name(pod)
|
|
|
|
if security_groups:
|
|
port_req_body['security_groups'] = security_groups
|
|
|
|
if self._tag_on_creation:
|
|
tags = CONF.neutron_defaults.resource_tags
|
|
if tags:
|
|
port_req_body['tags'] = tags
|
|
|
|
return port_req_body
|
|
|
|
def _create_subports_info(self, pod, project_id, subnets,
|
|
security_groups, trunk_id, num_ports,
|
|
unbound=False):
|
|
subports_info = []
|
|
|
|
in_use_vlan_ids = self._get_in_use_vlan_ids_set(trunk_id)
|
|
port_rq = self._get_port_request(pod, project_id, subnets,
|
|
security_groups, unbound)
|
|
for _ in range(num_ports):
|
|
try:
|
|
vlan_id = seg_driver.allocate_segmentation_id(in_use_vlan_ids)
|
|
except kl_exc.SegmentationIdAllocationFailure:
|
|
LOG.warning("There is not enough vlan ids available to "
|
|
"create a batch of %d subports.", num_ports)
|
|
break
|
|
in_use_vlan_ids.add(vlan_id)
|
|
|
|
subports_info.append({'segmentation_id': vlan_id,
|
|
'port_id': '',
|
|
'segmentation_type': 'vlan'})
|
|
return port_rq, subports_info
|
|
|
|
def _get_trunk_id(self, port):
|
|
try:
|
|
return port['trunk_details']['trunk_id']
|
|
except (KeyError, TypeError):
|
|
LOG.error("Neutron port is missing trunk details. "
|
|
"Please ensure that k8s node port is associated "
|
|
"with a Neutron vlan trunk")
|
|
raise k_exc.K8sNodeTrunkPortFailure
|
|
|
|
def _add_subport(self, trunk_id, subport, requested_vlan_id=None):
|
|
"""Adds subport port to Neutron trunk
|
|
|
|
This method gets vlanid allocated from kuryr segmentation driver.
|
|
In active/active HA type deployment, possibility of vlanid conflict
|
|
is there. In such a case, vlanid will be requested again and subport
|
|
addition is re-tried. This is tried DEFAULT_MAX_RETRY_COUNT times in
|
|
case of vlanid conflict.
|
|
"""
|
|
# TODO(vikasc): Better approach for retrying in case of
|
|
# vlan-id conflict.
|
|
os_net = clients.get_network_client()
|
|
retry_count = 1
|
|
while True:
|
|
if requested_vlan_id:
|
|
vlan_id = requested_vlan_id
|
|
else:
|
|
try:
|
|
vlan_id = self._get_vlan_id(trunk_id)
|
|
except os_exc.SDKException:
|
|
LOG.error("Getting VLAN ID for subport on "
|
|
"trunk %s failed!!", trunk_id)
|
|
raise
|
|
|
|
subport = [{'segmentation_id': vlan_id,
|
|
'port_id': subport,
|
|
'segmentation_type': 'vlan'}]
|
|
try:
|
|
os_net.add_trunk_subports(trunk_id, subport)
|
|
except os_exc.ConflictException:
|
|
if (retry_count < DEFAULT_MAX_RETRY_COUNT and
|
|
not requested_vlan_id):
|
|
LOG.error("VLAN ID already in use on trunk %s. "
|
|
"Retrying.", trunk_id)
|
|
retry_count += 1
|
|
sleep(DEFAULT_RETRY_INTERVAL)
|
|
continue
|
|
else:
|
|
LOG.error("Failed to add subport %s to trunk %s due to "
|
|
"VLAN ID %d conflict.", subport, trunk_id,
|
|
vlan_id)
|
|
raise
|
|
except os_exc.SDKException:
|
|
LOG.exception("Error happened during subport "
|
|
"addition to trunk %s", trunk_id)
|
|
raise
|
|
return vlan_id
|
|
|
|
def _remove_subports(self, trunk_id, subports_id):
|
|
os_net = clients.get_network_client()
|
|
subports_body = []
|
|
for subport_id in set(subports_id):
|
|
subports_body.append({'port_id': subport_id})
|
|
try:
|
|
os_net.delete_trunk_subports(trunk_id, subports_body)
|
|
except os_exc.SDKException:
|
|
LOG.exception("Error happened during subport removal from "
|
|
"trunk %s", trunk_id)
|
|
raise
|
|
|
|
def _remove_subport(self, trunk_id, subport_id):
|
|
self._remove_subports(trunk_id, [subport_id])
|
|
|
|
def _get_vlan_id(self, trunk_id):
|
|
vlan_ids = self._get_in_use_vlan_ids_set(trunk_id)
|
|
return seg_driver.allocate_segmentation_id(vlan_ids)
|
|
|
|
def _release_vlan_id(self, id):
|
|
return seg_driver.release_segmentation_id(id)
|
|
|
|
def _get_in_use_vlan_ids_set(self, trunk_id):
|
|
vlan_ids = set()
|
|
os_net = clients.get_network_client()
|
|
trunk = os_net.get_trunk(trunk_id)
|
|
for port in trunk.sub_ports:
|
|
vlan_ids.add(port['segmentation_id'])
|
|
|
|
return vlan_ids
|