Merge "Add basical functionalities for metadata path extension"

This commit is contained in:
Zuul
2025-02-13 05:15:39 +00:00
committed by Gerrit Code Review
8 changed files with 569 additions and 1 deletions

View File

@ -44,7 +44,7 @@ global
daemon
frontend public
bind *:80 name clear
bind *:{{ bind_port }} name clear
mode http
log global
option httplog
@ -142,6 +142,7 @@ class HostMedataHAProxyDaemonMonitor:
user=username,
group=groupname,
maxconn=1024,
bind_port=cfg.CONF.METADATA.host_proxy_listen_port,
instance_list=instance_infos,
meta_api=meta_api))

View File

@ -0,0 +1,318 @@
# Copyright (c) 2023 China Unicom Cloud Data Co.,Ltd.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
import secrets
import time
import netaddr
from neutron_lib.agent import l2_extension as l2_agent_extension
from neutron_lib import constants
from neutron_lib import exceptions as n_exc
from neutron_lib.plugins.ml2 import ovs_constants as p_const
from neutron_lib.plugins import utils as p_utils
from neutron_lib.utils import net as net_lib
from oslo_concurrency import lockutils
from oslo_config import cfg
from oslo_log import log as logging
from neutron._i18n import _
from neutron.agent.common import ip_lib
from neutron.agent.l2.extensions.metadata import host_metadata_proxy
from neutron.agent.linux import external_process
from neutron.api.rpc.callbacks import resources
LOG = logging.getLogger(__name__)
DEFAULT_META_GATEWAY_MAC = "fa:16:ee:00:00:01"
class InvalidProviderCIDR(n_exc.NeutronException):
message = _("Not enough Metadata IPs in /32 CIDR")
class NoMoreProviderRes(n_exc.NeutronException):
message = _("No more %(res)s")
class FailedToInitMetadataPathExtension(n_exc.NeutronException):
message = _("Could not initialize agent extension "
"metadata path, error: %(msg)s")
class MetadataPathExtensionPortInfoAPI():
def __init__(self, cache_api):
self.cache_api = cache_api
self.allocated_ips = netaddr.IPSet()
self.allocated_macs = set()
def get_port_fixed_ip(self, port):
for ip in port.fixed_ips:
ip_addr = netaddr.IPAddress(str(ip.ip_address))
if ip_addr.version == constants.IP_VERSION_4:
return str(ip.ip_address)
def remove_allocated_ip(self, ip):
self.allocated_ips.remove(ip)
def remove_allocated_mac(self, mac):
self.allocated_macs.remove(mac)
def _get_one_ip(self):
def generate_local_ip(cidr):
network = netaddr.IPNetwork(cidr)
if network.prefixlen == 32:
raise InvalidProviderCIDR()
# https://docs.python.org/3/library/secrets.html#module-secrets
# secrets.randbelow(exclusive_upper_bound)
# Return a random int in the range [0, exclusive_upper_bound).
# Here we remove the first and last IPs here.
index = secrets.randbelow(network.size - 1)
return str(network[index + 1])
for _i in range(1, 100):
ip = generate_local_ip(cfg.CONF.METADATA.provider_cidr)
if ip not in self.allocated_ips:
return ip
raise NoMoreProviderRes(res="provider IP addresses")
def _get_one_mac(self):
for _i in range(1, 1000):
base_mac = cfg.CONF.METADATA.provider_base_mac
mac = net_lib.get_random_mac(base_mac.split(':'))
if mac not in self.allocated_macs:
return mac
raise NoMoreProviderRes(res="provider MAC addresses")
def get_provider_ip_info(self, port_id,
provider_ip=None,
provider_mac=None):
port_obj = self.cache_api.get_resource_by_id(
resources.PORT, port_id)
if not port_obj or not port_obj.device_id:
return
info = {"instance_id": port_obj.device_id,
"project_id": port_obj.project_id}
if (not provider_ip or netaddr.IPNetwork(provider_ip) not in
netaddr.IPNetwork(cfg.CONF.METADATA.provider_cidr)):
provider_ip = self._get_one_ip()
self.allocated_ips.add(provider_ip)
info["provider_ip"] = provider_ip
if not provider_mac:
provider_mac = self._get_one_mac()
self.allocated_macs.add(provider_mac)
info["provider_port_mac"] = provider_mac
return info
class MetadataPathAgentExtension(l2_agent_extension.L2AgentExtension):
PORT_INFO_CACHE = {}
META_DEV_NAME = "tap-meta"
@lockutils.synchronized('networking-path-ofport-cache')
def set_port_info_cache(self, port_id, port_info):
self.PORT_INFO_CACHE[port_id] = port_info
@lockutils.synchronized('networking-path-ofport-cache')
def get_port_info_from_cache(self, port_id):
return self.PORT_INFO_CACHE.pop(port_id, None)
def consume_api(self, agent_api):
if not all([agent_api.br_phys.get('meta'), agent_api.phys_ofports,
agent_api.bridge_mappings.get('meta')]):
raise FailedToInitMetadataPathExtension(
msg="The metadata bridge device may not exist.")
self.agent_api = agent_api
self.rcache_api = agent_api.plugin_rpc.remote_resource_cache
def initialize(self, connection, driver_type):
"""Initialize agent extension."""
self.ext_api = MetadataPathExtensionPortInfoAPI(self.rcache_api)
self.int_br = self.agent_api.request_int_br()
self.meta_br = self.agent_api.request_physical_br('meta')
self.instance_infos = {}
bridge = self.agent_api.bridge_mappings.get('meta')
port_name = p_utils.get_interface_name(
bridge, prefix=p_const.PEER_INTEGRATION_PREFIX)
self.ofport_int_to_meta = self.int_br.get_port_ofport(port_name)
self.ofport_meta_to_int = self.agent_api.phys_ofports['meta']
if (not cfg.CONF.METADATA.nova_metadata_host or
not cfg.CONF.METADATA.nova_metadata_port):
LOG.warning("Nova metadata API related options are not set. "
"Host metadata haproxy will not start. "
"Please check the config option of "
"'nova_metadata_*' in [METADATA] section.")
return
self.process_monitor = external_process.ProcessMonitor(
config=cfg.CONF,
resource_type='MetadataPath')
self.meta_daemon = host_metadata_proxy.HostMedataHAProxyDaemonMonitor(
self.process_monitor,
user=str(os.geteuid()),
group=str(os.getegid()))
self.provider_vlan_id = cfg.CONF.METADATA.provider_vlan_id
self.provider_cidr = cfg.CONF.METADATA.provider_cidr
# TODO(liuyulong): init related flows
self.provider_gateway_ip = str(netaddr.IPAddress(
netaddr.IPNetwork(cfg.CONF.METADATA.provider_cidr).first + 1))
self._create_internal_port()
def _set_port_vlan(self):
ovsdb = self.meta_br.ovsdb
with self.meta_br.ovsdb.transaction() as txn:
# When adding the port's tag,
# also clear port's vlan_mode and trunks,
# which were set to make sure all packets are dropped.
txn.add(ovsdb.db_set('Port', self.META_DEV_NAME,
('tag', self.provider_vlan_id)))
txn.add(ovsdb.db_clear('Port', self.META_DEV_NAME, 'vlan_mode'))
txn.add(ovsdb.db_clear('Port', self.META_DEV_NAME, 'trunks'))
def _create_internal_port(self):
attrs = [('type', 'internal'),
('external_ids', {'iface-status': 'active',
'attached-mac': DEFAULT_META_GATEWAY_MAC})]
self.meta_br.replace_port(self.META_DEV_NAME, *attrs)
ns_dev = ip_lib.IPDevice(self.META_DEV_NAME)
for _i in range(9):
try:
ns_dev.link.set_address(DEFAULT_META_GATEWAY_MAC)
break
except RuntimeError as e:
LOG.warning("Got error trying to set mac, retrying: %s", e)
time.sleep(1)
try:
ns_dev.link.set_address(DEFAULT_META_GATEWAY_MAC)
except RuntimeError as e:
msg = _("Failed to set mac address "
"for dev %s, error: %s") % (self.META_DEV_NAME, e)
raise RuntimeError(msg)
cidr = "%s/%s" % (
self.provider_gateway_ip,
netaddr.IPNetwork(self.provider_cidr).prefixlen)
ns_dev.addr.add(cidr)
ns_dev.link.set_up()
self.meta_br.set_value_to_other_config(
self.META_DEV_NAME,
"tag",
self.provider_vlan_id)
self._set_port_vlan()
def _reload_host_metadata_proxy(self, force_reload=False):
if (not cfg.CONF.METADATA.nova_metadata_host or
not cfg.CONF.METADATA.nova_metadata_port):
LOG.warning("Nova metadata API related options are not set. "
"Host metadata haproxy will not start.")
return
if not force_reload and not self.instance_infos:
return
# Haproxy does not suport 'kill -HUP' to reload config file,
# so just kill it and then re-spawn.
self.meta_daemon.disable()
self.meta_daemon.config(list(self.instance_infos.values()))
if self.instance_infos:
self.meta_daemon.enable()
def _get_port_info(self, port_detail):
device_owner = port_detail['device_owner']
if not device_owner.startswith(constants.DEVICE_OWNER_COMPUTE_PREFIX):
return
port = port_detail['vif_port']
provider_ip = self.int_br.get_value_from_other_config(
port.port_name, 'provider_ip')
provider_mac = self.int_br.get_value_from_other_config(
port.port_name, 'provider_mac')
ins_info = self.ext_api.get_provider_ip_info(port_detail['port_id'],
provider_ip,
provider_mac)
if not ins_info:
LOG.info("Failed to get port %s instance provider IP info.",
port_detail['port_id'])
return
self.instance_infos[port_detail['port_id']] = ins_info
if not provider_ip or provider_ip != ins_info['provider_ip']:
self.int_br.set_value_to_other_config(
port.port_name,
'provider_ip',
ins_info['provider_ip'])
if not provider_mac:
self.int_br.set_value_to_other_config(
port.port_name,
'provider_mac',
ins_info['provider_port_mac'])
vlan = self.int_br.get_value_from_other_config(
port.port_name, 'tag', int)
port_info = {"port_id": port_detail['port_id'],
"device_owner": device_owner,
"port_name": port.port_name,
"vlan": vlan,
"mac_address": port_detail["mac_address"],
"fixed_ips": port_detail["fixed_ips"],
"ofport": port.ofport,
"network_id": port_detail['network_id']}
LOG.debug("Metadata path got the port information: %s ",
port_info)
return port_info
def handle_port(self, context, port_detail):
try:
port_info = self._get_port_info(port_detail)
if not port_info:
return
self.set_port_info_cache(port_detail['port_id'], port_info)
except Exception as err:
LOG.info("Failed to get or set port %s info, error: %s",
port_detail['port_id'], err)
else:
# TODO(liuyulong): Add flows for metadata
self._reload_host_metadata_proxy()
def _get_fixed_ip(self, port_info):
for ip in port_info['fixed_ips']:
ip_addr = netaddr.IPAddress(ip['ip_address'])
if ip_addr.version == constants.IP_VERSION_4:
return ip['ip_address']
def delete_port(self, context, port_detail):
ins_info = self.instance_infos.pop(port_detail['port_id'], None)
self._reload_host_metadata_proxy(force_reload=True)
if not ins_info:
return
# TODO(liuyulong): Remove flows for metadata
self.ext_api.remove_allocated_ip(ins_info['provider_ip'])
self.ext_api.remove_allocated_mac(ins_info['provider_port_mac'])

View File

@ -257,6 +257,29 @@ local_ip_opts = [
]
metadata_opts = [
cfg.StrOpt('provider_cidr', default='240.0.0.0/16',
help=_("Local metadata CIDR for VMs metadata traffic, "
"will be used as the IP range to generate the "
"VM's metadata IP.")),
cfg.IntOpt('provider_vlan_id', default=1,
help=_("The metadata tap device local vlan ID. This is only "
"available on the metadata bridge device.")),
cfg.StrOpt('provider_base_mac', default="fa:16:ee:00:00:00",
help=_("The base MAC address Neutron Openvswitch agent "
"will use for metadata traffic.")),
cfg.IntOpt('host_proxy_listen_port', default=80,
help=_("Host haproxy listen port for metadata path. This "
"is transparent for metadata traffic, VMs still try to "
"access 169.254.169.254:80 for metadata. But in "
"the metadata datapath flow pipeline, the destination "
"TCP port 80 will be changed to the value of "
"`host_proxy_listen_port` which the host haproxy "
"will listen on. For return traffic, the TCP source "
"port will be changed back to 80.")),
]
def register_ovs_agent_opts(cfg=cfg.CONF):
cfg.register_opts(ovs_opts, "OVS")
cfg.register_opts(agent_opts, "AGENT")
@ -264,6 +287,7 @@ def register_ovs_agent_opts(cfg=cfg.CONF):
cfg.register_opts(common.DHCP_PROTOCOL_OPTS, "DHCP")
cfg.register_opts(local_ip_opts, "LOCAL_IP")
cfg.register_opts(meta_conf.METADATA_PROXY_HANDLER_OPTS, "METADATA")
cfg.register_opts(metadata_opts, "METADATA")
def register_ovs_opts(cfg=cfg.CONF):

View File

@ -341,6 +341,7 @@ def list_ovs_opts():
neutron.conf.agent.common.DHCP_PROTOCOL_OPTS)),
('metadata',
itertools.chain(
neutron.conf.plugins.ml2.drivers.ovs_conf.metadata_opts,
meta_conf.METADATA_PROXY_HANDLER_OPTS))
]

View File

@ -165,6 +165,8 @@ class OVSNeutronAgent(l2population_rpc.L2populationRpcCallBackTunnelMixin,
self.enable_openflow_dhcp = 'dhcp' in self.ext_manager.names()
self.enable_local_ips = 'local_ip' in self.ext_manager.names()
self.enable_openflow_metadata = (
'metadata_path' in self.ext_manager.names())
self.fullsync = False
# init bridge classes with configured datapath type.
@ -263,6 +265,11 @@ class OVSNeutronAgent(l2population_rpc.L2populationRpcCallBackTunnelMixin,
self.phys_brs = {}
self.int_ofports = {}
self.phys_ofports = {}
if (self.enable_openflow_metadata and
'meta' not in self.bridge_mappings):
self.bridge_mappings['meta'] = 'br-meta'
self.setup_physical_bridges(self.bridge_mappings)
self.vlan_manager = vlanmanager.LocalVlanManager()

View File

@ -0,0 +1,203 @@
# Copyright (c) 2023 China Unicom Cloud Data Co.,Ltd.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from unittest import mock
from neutron_lib import context
from oslo_config import cfg
from neutron.agent.common import ovs_lib
from neutron.agent.l2.extensions.metadata import metadata_path
from neutron.api.rpc.callbacks import resources
from neutron.conf.plugins.ml2.drivers import ovs_conf
from neutron.plugins.ml2.drivers.openvswitch.agent \
import ovs_agent_extension_api as ovs_ext_api
from neutron.tests import base
class MetadataPathAgentExtensionTestCase(base.BaseTestCase):
def setUp(self):
super(MetadataPathAgentExtensionTestCase, self).setUp()
ovs_conf.register_ovs_agent_opts(cfg=cfg.CONF)
cfg.CONF.set_override('provider_cidr', '240.0.0.0/31', 'METADATA')
self.context = context.get_admin_context()
self.int_br = mock.Mock()
self.meta_br = mock.Mock()
self.plugin_rpc = mock.Mock()
self.remote_resource_cache = mock.Mock()
self.plugin_rpc.remote_resource_cache = self.remote_resource_cache
self.meta_ext = metadata_path.MetadataPathAgentExtension()
self.bridge_mappings = {"meta": "br-meta"}
self.int_ofport = 200
self.phys_ofport = 100
self.agent_api = ovs_ext_api.OVSAgentExtensionAPI(
self.int_br,
tun_br=mock.Mock(),
phys_brs={"meta": self.meta_br},
plugin_rpc=self.plugin_rpc,
phys_ofports={"meta": self.phys_ofport},
bridge_mappings=self.bridge_mappings)
self.meta_ext.consume_api(self.agent_api)
mock.patch(
"neutron.agent.linux.ip_lib.IpLinkCommand.set_address").start()
mock.patch(
"neutron.agent.linux.ip_lib.IpAddrCommand.add").start()
mock.patch(
"neutron.agent.linux.ip_lib.IpLinkCommand.set_up").start()
self.meta_ext._set_port_vlan = mock.Mock()
self.meta_ext.initialize(None, None)
# set int_br back to mock
self.meta_ext.int_br = self.int_br
# set meta_br back to mock
self.meta_ext.meta_br = self.meta_br
self.get_port_ofport = mock.patch.object(
self.int_br, 'get_port_ofport',
return_value=self.int_ofport).start()
self.meta_daemon = mock.Mock()
self.meta_ext.meta_daemon = mock.Mock()
self.port_provider_ip = "100.100.100.100"
self.port_provider_mac = "fa:16:ee:11:22:33"
def m_get_value_from_ovsdb_other_config(p, key, value_type=None):
if key == "provider_ip":
return self.port_provider_ip
if key == "provider_mac":
return self.port_provider_mac
mock.patch.object(
self.int_br, 'get_value_from_other_config',
side_effect=m_get_value_from_ovsdb_other_config).start()
mock.patch.object(
self.int_br, 'set_value_to_other_config').start()
mock.patch.object(
self.meta_br, 'set_value_to_other_config').start()
def test_handle_port(self):
port_mac_address = "aa:aa:aa:aa:aa:aa"
port_name = "tap-p1"
port_id = "p1"
port_ofport = 1
port_device_owner = "compute:test"
with mock.patch.object(self.meta_ext.meta_daemon,
"config") as h_config, mock.patch.object(
self.meta_ext.ext_api,
"get_provider_ip_info") as get_p_info:
get_p_info.return_value = {
'instance_id': 'instance_uuid_1',
'project_id': 'project_id_1',
'provider_ip': self.port_provider_ip,
'provider_port_mac': self.port_provider_mac
}
port = {"port_id": port_id,
"fixed_ips": [{"ip_address": "1.1.1.1",
"subnet_id": "1"}],
"vif_port": ovs_lib.VifPort(port_name, port_ofport,
port_id,
port_mac_address, "br-int"),
"device_owner": port_device_owner,
"network_id": "net_id_1",
"mac_address": port_mac_address}
self.meta_ext.handle_port(self.context, port)
get_p_info.assert_called_once_with(
port['port_id'],
self.port_provider_ip,
self.port_provider_mac)
h_config.assert_called_once_with(
list(self.meta_ext.instance_infos.values()))
def test_get_port_no_more_provider_ip(self):
def m_get_value_from_ovsdb_other_config(p, key, value_type=None):
if key == "provider_ip":
return
if key == "provider_mac":
return
mock.patch.object(
self.int_br, 'get_value_from_other_config',
side_effect=m_get_value_from_ovsdb_other_config).start()
mock.patch.object(
self.int_br, 'set_value_to_other_config').start()
port_device_owner = "compute:test"
class Port(object):
def __init__(self):
self.device_id = "d1"
self.project_id = "p1"
with mock.patch.object(self.meta_ext.meta_daemon,
"config"), mock.patch.object(
self.meta_ext.ext_api.cache_api,
"get_resource_by_id",
return_value=Port()) as get_res:
port1_mac_address = "aa:aa:aa:aa:aa:aa"
port1_name = "tap-p1"
port1_id = "p1"
port1_ofport = 1
port1 = {"port_id": port1_id,
"fixed_ips": [{"ip_address": "1.1.1.1",
"subnet_id": "1"}],
"vif_port": ovs_lib.VifPort(port1_name, port1_ofport,
port1_id,
port1_mac_address, "br-int"),
"device_owner": port_device_owner,
"network_id": "net_id_1",
"mac_address": port1_mac_address}
self.meta_ext.handle_port(self.context, port1)
get_res.assert_called_once_with(
resources.PORT,
port1['port_id'])
port2_id = "p2"
self.assertRaises(
metadata_path.NoMoreProviderRes,
self.meta_ext.ext_api.get_provider_ip_info,
port2_id, None, None)
def test_delete_port(self):
port_mac_address = "aa:aa:aa:aa:aa:aa"
port_name = "tap-p1"
port_id = "p1"
port_ofport = 1
port_device_owner = "compute:test"
with mock.patch.object(self.meta_ext.meta_daemon,
"config") as h_config:
port = {"port_id": port_id,
"fixed_ips": [{"ip_address": "1.1.1.1",
"subnet_id": "1"}],
"vif_port": ovs_lib.VifPort(port_name, port_ofport,
port_id,
port_mac_address, "br-int"),
"device_owner": port_device_owner,
"network_id": "net_id_1",
"mac_address": port_mac_address}
self.meta_ext.handle_port(self.context, port)
instance_info_values = list(self.meta_ext.instance_infos.values())
self.meta_ext.delete_port(self.context, {"port_id": port_id})
h_config.assert_has_calls([mock.call(instance_info_values),
mock.call([])])
self.assertNotIn(self.port_provider_ip,
self.meta_ext.ext_api.allocated_ips)
self.assertNotIn(self.port_provider_mac,
self.meta_ext.ext_api.allocated_macs)

View File

@ -0,0 +1,13 @@
---
features:
- |
A new openvswitch agent extension ``metadata_path`` was added to implement
a distributed approach for virtual machines to retrieve metadata in
each running host without a traditional metadata-agent and its dependent
router or DHCP namespace.
For a new host, users need to create the OVS bridge
named ``br-meta``. The OVS-agent will implicitly add an entry
``meta:br-meta`` to the list of ``bridge_mappings``.
New config options ``provider_cidr``, ``provider_vlan_id``,
``provider_base_mac`` and ``host_proxy_listen_port`` are added to the
openvswitch agent ``[METADATA]`` section.

View File

@ -135,6 +135,7 @@ neutron.agent.l2.extensions =
log = neutron.services.logapi.agent.log_extension:LoggingExtension
dhcp = neutron.agent.l2.extensions.dhcp.extension:DHCPAgentExtension
local_ip = neutron.agent.l2.extensions.local_ip:LocalIPAgentExtension
metadata_path = neutron.agent.l2.extensions.metadata.metadata_path:MetadataPathAgentExtension
neutron.agent.l3.extensions =
fip_qos = neutron.agent.l3.extensions.qos.fip:FipQosAgentExtension
gateway_ip_qos = neutron.agent.l3.extensions.qos.gateway_ip:RouterGatewayIPQosAgentExtension