From d3daa09bc49bcc5c66791565e1720a9a484ea149 Mon Sep 17 00:00:00 2001 From: Hemanth Nakkina Date: Tue, 31 Aug 2021 11:14:43 +0530 Subject: [PATCH] Add extension fwaas_v2 based on neutron-api relation data fwaas_v2 extension is added in l3_agent.ini by default from stein. However with the newly introduced neutron-api configuration parameter enable-fwaas, the extension have to added only when it is set to true on neutron-api charm. neutron-api charm adds enabled-fwaas as relation data in the commit https://review.opendev.org/c/openstack/charm-neutron-api/+/806676. This patch removes special handling of fwaas_v2 as l3_extension_plugins is already updated in relation data based on neutron-api enable-fwaas configuration. Add python3-neutron-fwaas in py3 package list only for rocky release. The package is already part of dependency of neutron-l3-agent until Ussuri release and the dependency will be removed from Victoria release in package control files. Synced charmhelpers to get related changes https://github.com/juju/charm-helpers/pull/635 Closes-Bug: #1934129 Change-Id: I1546f45635bfd7af76001c1df4f99d219a9d8039 --- .../charmhelpers/contrib/openstack/context.py | 42 ++++++++++++++----- .../contrib/openstack/templates/haproxy.cfg | 16 ++++++- hooks/charmhelpers/contrib/openstack/utils.py | 4 ++ hooks/charmhelpers/fetch/ubuntu.py | 13 +++++- hooks/charmhelpers/fetch/ubuntu_apt_pkg.py | 2 +- hooks/neutron_ovs_context.py | 11 ----- hooks/neutron_ovs_utils.py | 6 ++- unit_tests/test_neutron_ovs_context.py | 10 ++--- 8 files changed, 71 insertions(+), 33 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 0ec935eb..1611a8f4 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -25,7 +25,10 @@ import socket import time from base64 import b64decode -from subprocess import check_call, CalledProcessError +from subprocess import ( + check_call, + check_output, + CalledProcessError) import six @@ -453,18 +456,24 @@ class IdentityServiceContext(OSContextGenerator): serv_host = format_ipv6_addr(serv_host) or serv_host auth_host = rdata.get('auth_host') auth_host = format_ipv6_addr(auth_host) or auth_host + int_host = rdata.get('internal_host') + int_host = format_ipv6_addr(int_host) or int_host svc_protocol = rdata.get('service_protocol') or 'http' auth_protocol = rdata.get('auth_protocol') or 'http' + int_protocol = rdata.get('internal_protocol') or 'http' api_version = rdata.get('api_version') or '2.0' ctxt.update({'service_port': rdata.get('service_port'), 'service_host': serv_host, 'auth_host': auth_host, 'auth_port': rdata.get('auth_port'), + 'internal_host': int_host, + 'internal_port': rdata.get('internal_port'), 'admin_tenant_name': rdata.get('service_tenant'), 'admin_user': rdata.get('service_username'), 'admin_password': rdata.get('service_password'), 'service_protocol': svc_protocol, 'auth_protocol': auth_protocol, + 'internal_protocol': int_protocol, 'api_version': api_version}) if float(api_version) > 2: @@ -1781,6 +1790,10 @@ class NeutronAPIContext(OSContextGenerator): 'rel_key': 'enable-port-forwarding', 'default': False, }, + 'enable_fwaas': { + 'rel_key': 'enable-fwaas', + 'default': False, + }, 'global_physnet_mtu': { 'rel_key': 'global-physnet-mtu', 'default': 1500, @@ -1815,6 +1828,11 @@ class NeutronAPIContext(OSContextGenerator): if ctxt['enable_port_forwarding']: l3_extension_plugins.append('port_forwarding') + if ctxt['enable_fwaas']: + l3_extension_plugins.append('fwaas_v2') + if ctxt['enable_nfg_logging']: + l3_extension_plugins.append('fwaas_v2_log') + ctxt['l3_extension_plugins'] = l3_extension_plugins return ctxt @@ -2379,12 +2397,6 @@ class DHCPAgentContext(OSContextGenerator): ctxt['enable_metadata_network'] = True ctxt['enable_isolated_metadata'] = True - ctxt['append_ovs_config'] = False - cmp_release = CompareOpenStackReleases( - os_release('neutron-common', base='icehouse')) - if cmp_release >= 'queens' and config('enable-dpdk'): - ctxt['append_ovs_config'] = True - return ctxt @staticmethod @@ -2584,14 +2596,22 @@ class OVSDPDKDeviceContext(OSContextGenerator): return format(mask, '#04x') def socket_memory(self): - """Formatted list of socket memory configuration per NUMA node + """Formatted list of socket memory configuration per socket. - :returns: socket memory configuration per NUMA node + :returns: socket memory configuration per socket. :rtype: str """ + lscpu_out = check_output( + ['lscpu', '-p=socket']).decode('UTF-8').strip() + sockets = set() + for line in lscpu_out.split('\n'): + try: + sockets.add(int(line)) + except ValueError: + # lscpu output is headed by comments so ignore them. + pass sm_size = config('dpdk-socket-memory') - node_regex = '/sys/devices/system/node/node*' - mem_list = [str(sm_size) for _ in glob.glob(node_regex)] + mem_list = [str(sm_size) for _ in sockets] if mem_list: return ','.join(mem_list) else: diff --git a/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg b/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg index d36af2aa..875e1393 100644 --- a/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg +++ b/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg @@ -1,10 +1,22 @@ global - log /var/lib/haproxy/dev/log local0 - log /var/lib/haproxy/dev/log local1 notice + # NOTE: on startup haproxy chroot's to /var/lib/haproxy. + # + # Unfortunately the program will open some files prior to the call to + # chroot never to reopen them, and some after. So looking at the on-disk + # layout of haproxy resources you will find some resources relative to / + # such as the admin socket, and some relative to /var/lib/haproxy such as + # the log socket. + # + # The logging socket is (re-)opened after the chroot and must be relative + # to /var/lib/haproxy. + log /dev/log local0 + log /dev/log local1 notice maxconn 20000 user haproxy group haproxy spread-checks 0 + # The admin socket is opened prior to the chroot never to be reopened, so + # it lives outside the chroot directory in the filesystem. stats socket /var/run/haproxy/admin.sock mode 600 level admin stats timeout 2m diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index a41f9959..008a8ec0 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -135,6 +135,8 @@ DISTRO_PROPOSED = ('deb http://archive.ubuntu.com/ubuntu/ %s-proposed ' 'restricted main multiverse universe') OPENSTACK_CODENAMES = OrderedDict([ + # NOTE(lourot): 'yyyy.i' isn't actually mapping with any real version + # number. This just means the i-th version of the year yyyy. ('2011.2', 'diablo'), ('2012.1', 'essex'), ('2012.2', 'folsom'), @@ -155,6 +157,8 @@ OPENSTACK_CODENAMES = OrderedDict([ ('2020.1', 'ussuri'), ('2020.2', 'victoria'), ('2021.1', 'wallaby'), + ('2021.2', 'xena'), + ('2022.1', 'yoga'), ]) # The ugly duckling - must list releases oldest to newest diff --git a/hooks/charmhelpers/fetch/ubuntu.py b/hooks/charmhelpers/fetch/ubuntu.py index 07298096..c9433c12 100644 --- a/hooks/charmhelpers/fetch/ubuntu.py +++ b/hooks/charmhelpers/fetch/ubuntu.py @@ -208,6 +208,14 @@ CLOUD_ARCHIVE_POCKETS = { 'wallaby/proposed': 'focal-proposed/wallaby', 'focal-wallaby/proposed': 'focal-proposed/wallaby', 'focal-proposed/wallaby': 'focal-proposed/wallaby', + # Xena + 'xena': 'focal-updates/xena', + 'focal-xena': 'focal-updates/xena', + 'focal-xena/updates': 'focal-updates/xena', + 'focal-updates/xena': 'focal-updates/xena', + 'xena/proposed': 'focal-proposed/xena', + 'focal-xena/proposed': 'focal-proposed/xena', + 'focal-proposed/xena': 'focal-proposed/xena', # Yoga 'yoga': 'focal-updates/yoga', 'focal-yoga': 'focal-updates/yoga', @@ -272,7 +280,7 @@ UBUNTU_OPENSTACK_RELEASE = OrderedDict([ APT_NO_LOCK = 100 # The return code for "couldn't acquire lock" in APT. CMD_RETRY_DELAY = 10 # Wait 10 seconds between command retries. -CMD_RETRY_COUNT = 3 # Retry a failing fatal command X times. +CMD_RETRY_COUNT = 10 # Retry a failing fatal command X times. def filter_installed_packages(packages): @@ -734,7 +742,8 @@ def _add_apt_repository(spec): series = get_distrib_codename() spec = spec.replace('{series}', series) _run_with_retries(['add-apt-repository', '--yes', spec], - cmd_env=env_proxy_settings(['https', 'http'])) + cmd_env=env_proxy_settings(['https', 'http', 'no_proxy']) + ) def _add_cloud_pocket(pocket): diff --git a/hooks/charmhelpers/fetch/ubuntu_apt_pkg.py b/hooks/charmhelpers/fetch/ubuntu_apt_pkg.py index a2fbe0e5..1f9bc73a 100644 --- a/hooks/charmhelpers/fetch/ubuntu_apt_pkg.py +++ b/hooks/charmhelpers/fetch/ubuntu_apt_pkg.py @@ -264,7 +264,7 @@ def version_compare(a, b): else: raise RuntimeError('Unable to compare "{}" and "{}", according to ' 'our logic they are neither greater, equal nor ' - 'less than each other.') + 'less than each other.'.format(a, b)) class PkgVersion(): diff --git a/hooks/neutron_ovs_context.py b/hooks/neutron_ovs_context.py index dd8d387b..1c97eba8 100644 --- a/hooks/neutron_ovs_context.py +++ b/hooks/neutron_ovs_context.py @@ -344,20 +344,9 @@ class L3AgentContext(OSContextGenerator): NFG_LOG_BURST_LIMIT_MIN ) - cmp_os_release = CompareOpenStackReleases(os_release('neutron-common')) - l3_extension_plugins = neutron_api_settings.get( 'l3_extension_plugins', []) - # per Change-Id If1b332eb0f581e9acba111f79ba578a0b7081dd2 - # only enable it for stein although fwaasv2 was added in Queens - is_stein = cmp_os_release >= 'stein' - if is_stein: - l3_extension_plugins.append('fwaas_v2') - - if (is_stein and neutron_api_settings.get('enable_nfg_logging')): - l3_extension_plugins.append('fwaas_v2_log') - ctxt['l3_extension_plugins'] = ','.join(l3_extension_plugins) return ctxt diff --git a/hooks/neutron_ovs_utils.py b/hooks/neutron_ovs_utils.py index bf5828d4..4ec7de52 100644 --- a/hooks/neutron_ovs_utils.py +++ b/hooks/neutron_ovs_utils.py @@ -329,12 +329,16 @@ def determine_packages(): pkgs.extend(plugin_pkg) if use_dvr(): pkgs.extend(DVR_PACKAGES) - py3_pkgs.append('python3-neutron-fwaas') _os_release = os_release('neutron-common', base='icehouse') # per 17.08 release notes L3HA + DVR is a Newton+ feature if (use_l3ha() and CompareOpenStackReleases(_os_release) >= 'newton'): pkgs.extend(L3HA_PACKAGES) + # python3-neutron-fwaas is already dependency package on + # neutron-l3-agent. However this need to be added to py3_pkgs + # to support switch from py2 to py3 in Rocky release. + if CompareOpenStackReleases(_os_release) == 'rocky': + py3_pkgs.append('python3-neutron-fwaas') if enable_local_dhcp(): pkgs.extend(DHCP_PACKAGES) pkgs.extend(METADATA_PACKAGES) diff --git a/unit_tests/test_neutron_ovs_context.py b/unit_tests/test_neutron_ovs_context.py index 23cbbe0d..eb306147 100644 --- a/unit_tests/test_neutron_ovs_context.py +++ b/unit_tests/test_neutron_ovs_context.py @@ -372,10 +372,10 @@ class L3AgentContextTest(CharmTestCase): rdata = { 'neutron-security-groups': 'True', 'enable-dvr': 'True', + 'enable-fwaas': 'True', 'l2-population': 'True', 'overlay-network-type': 'vxlan', 'network-device-mtu': 1500, - 'l3_extension_plugins': 'fwaas_v2', } _rget.side_effect = lambda *args, **kwargs: rdata self.assertEqual( @@ -440,12 +440,12 @@ class L3AgentContextTest(CharmTestCase): rdata = { 'neutron-security-groups': 'True', 'enable-dvr': 'True', + 'enable-fwaas': 'True', 'l2-population': 'True', 'overlay-network-type': 'vxlan', 'network-device-mtu': 1500, 'enable-nfg-logging': 'True', 'use_l3ha': False, - 'l3_extension_plugins': 'fwaas_v2,fwaas_v2_log', } _rget.side_effect = lambda *args, **kwargs: rdata _validate_nfg_log_path.side_effect = lambda x: x @@ -482,11 +482,11 @@ class L3AgentContextTest(CharmTestCase): rdata = { 'neutron-security-groups': 'True', 'enable-dvr': 'True', + 'enable-fwaas': 'True', 'l2-population': 'True', 'overlay-network-type': 'vxlan', 'network-device-mtu': 1500, 'enable-nfg-logging': 'True', - 'l3_extension_plugins': 'fwaas_v2,fwaas_v2_log', } _rget.side_effect = lambda *args, **kwargs: rdata _validate_nfg_log_path.side_effect = lambda x: x @@ -522,10 +522,10 @@ class L3AgentContextTest(CharmTestCase): rdata = { 'neutron-security-groups': 'True', 'enable-dvr': 'True', + 'enable-fwaas': 'True', 'l2-population': 'True', 'overlay-network-type': 'vxlan', 'network-device-mtu': 1500, - 'l3_extension_plugins': 'fwaas_v2,fwaas_v2_log', } _rget.side_effect = lambda *args, **kwargs: rdata self.assertEqual( @@ -555,10 +555,10 @@ class L3AgentContextTest(CharmTestCase): rdata = { 'neutron-security-groups': 'True', 'enable-dvr': 'False', + 'enable-fwaas': 'True', 'l2-population': 'True', 'overlay-network-type': 'vxlan', 'network-device-mtu': 1500, - 'l3_extension_plugins': 'fwaas_v2', } _rget.side_effect = lambda *args, **kwargs: rdata self.assertEqual(context.L3AgentContext()(), {