Add extension fwaas_v2 based on neutron-api relation data

fwaas_v2 extension is added in l3_agent.ini by default from stein.
However with the newly introduced neutron-api configuration parameter
enable-fwaas, the extension have to added only when it is set to true
on neutron-api charm.

neutron-api charm adds enabled-fwaas as relation data in the commit
https://review.opendev.org/c/openstack/charm-neutron-api/+/806676.
This patch removes special handling of fwaas_v2 as l3_extension_plugins
is already updated in relation data based on neutron-api enable-fwaas
configuration.

Add python3-neutron-fwaas in py3 package list only for rocky release. The
package is already part of dependency of neutron-l3-agent until Ussuri
release and the dependency will be removed from Victoria release in
package control files.

Synced charmhelpers to get related changes
https://github.com/juju/charm-helpers/pull/635

Closes-Bug: #1934129
Change-Id: I1546f45635bfd7af76001c1df4f99d219a9d8039

hooks/charmhelpers/contrib/openstack/context.py

@@ -25,7 +25,10 @@ import socket
 import time
 
 from base64 import b64decode
-from subprocess import check_call, CalledProcessError
+from subprocess import (
+    check_call,
+    check_output,
+    CalledProcessError)
 
 import six
 
@@ -453,18 +456,24 @@ class IdentityServiceContext(OSContextGenerator):
                 serv_host = format_ipv6_addr(serv_host) or serv_host
                 auth_host = rdata.get('auth_host')
                 auth_host = format_ipv6_addr(auth_host) or auth_host
+                int_host = rdata.get('internal_host')
+                int_host = format_ipv6_addr(int_host) or int_host
                 svc_protocol = rdata.get('service_protocol') or 'http'
                 auth_protocol = rdata.get('auth_protocol') or 'http'
+                int_protocol = rdata.get('internal_protocol') or 'http'
                 api_version = rdata.get('api_version') or '2.0'
                 ctxt.update({'service_port': rdata.get('service_port'),
                              'service_host': serv_host,
                              'auth_host': auth_host,
                              'auth_port': rdata.get('auth_port'),
+                             'internal_host': int_host,
+                             'internal_port': rdata.get('internal_port'),
                              'admin_tenant_name': rdata.get('service_tenant'),
                              'admin_user': rdata.get('service_username'),
                              'admin_password': rdata.get('service_password'),
                              'service_protocol': svc_protocol,
                              'auth_protocol': auth_protocol,
+                             'internal_protocol': int_protocol,
                              'api_version': api_version})
 
                 if float(api_version) > 2:
@@ -1781,6 +1790,10 @@ class NeutronAPIContext(OSContextGenerator):
                 'rel_key': 'enable-port-forwarding',
                 'default': False,
             },
+            'enable_fwaas': {
+                'rel_key': 'enable-fwaas',
+                'default': False,
+            },
             'global_physnet_mtu': {
                 'rel_key': 'global-physnet-mtu',
                 'default': 1500,
@@ -1815,6 +1828,11 @@ class NeutronAPIContext(OSContextGenerator):
         if ctxt['enable_port_forwarding']:
             l3_extension_plugins.append('port_forwarding')
 
+        if ctxt['enable_fwaas']:
+            l3_extension_plugins.append('fwaas_v2')
+            if ctxt['enable_nfg_logging']:
+                l3_extension_plugins.append('fwaas_v2_log')
+
         ctxt['l3_extension_plugins'] = l3_extension_plugins
 
         return ctxt
@@ -2379,12 +2397,6 @@ class DHCPAgentContext(OSContextGenerator):
             ctxt['enable_metadata_network'] = True
             ctxt['enable_isolated_metadata'] = True
 
-        ctxt['append_ovs_config'] = False
-        cmp_release = CompareOpenStackReleases(
-            os_release('neutron-common', base='icehouse'))
-        if cmp_release >= 'queens' and config('enable-dpdk'):
-            ctxt['append_ovs_config'] = True
-
         return ctxt
 
     @staticmethod
@@ -2584,14 +2596,22 @@ class OVSDPDKDeviceContext(OSContextGenerator):
         return format(mask, '#04x')
 
     def socket_memory(self):
-        """Formatted list of socket memory configuration per NUMA node
+        """Formatted list of socket memory configuration per socket.
 
-        :returns: socket memory configuration per NUMA node
+        :returns: socket memory configuration per socket.
         :rtype: str
         """
+        lscpu_out = check_output(
+            ['lscpu', '-p=socket']).decode('UTF-8').strip()
+        sockets = set()
+        for line in lscpu_out.split('\n'):
+            try:
+                sockets.add(int(line))
+            except ValueError:
+                # lscpu output is headed by comments so ignore them.
+                pass
         sm_size = config('dpdk-socket-memory')
-        node_regex = '/sys/devices/system/node/node*'
+        mem_list = [str(sm_size) for _ in sockets]
-        mem_list = [str(sm_size) for _ in glob.glob(node_regex)]
         if mem_list:
             return ','.join(mem_list)
         else:

hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg

@@ -1,10 +1,22 @@
 global
-    log /var/lib/haproxy/dev/log local0
+    # NOTE: on startup haproxy chroot's to /var/lib/haproxy.
-    log /var/lib/haproxy/dev/log local1 notice
+    #
+    # Unfortunately the program will open some files prior to the call to
+    # chroot never to reopen them, and some after. So looking at the on-disk
+    # layout of haproxy resources you will find some resources relative to /
+    # such as the admin socket, and some relative to /var/lib/haproxy such as
+    # the log socket.
+    #
+    # The logging socket is (re-)opened after the chroot and must be relative
+    # to /var/lib/haproxy.
+    log /dev/log local0
+    log /dev/log local1 notice
     maxconn 20000
     user haproxy
     group haproxy
     spread-checks 0
+    # The admin socket is opened prior to the chroot never to be reopened, so
+    # it lives outside the chroot directory in the filesystem.
     stats socket /var/run/haproxy/admin.sock mode 600 level admin
     stats timeout 2m
 

hooks/charmhelpers/contrib/openstack/utils.py

@@ -135,6 +135,8 @@ DISTRO_PROPOSED = ('deb http://archive.ubuntu.com/ubuntu/ %s-proposed '
                    'restricted main multiverse universe')
 
 OPENSTACK_CODENAMES = OrderedDict([
+    # NOTE(lourot): 'yyyy.i' isn't actually mapping with any real version
+    # number. This just means the i-th version of the year yyyy.
     ('2011.2', 'diablo'),
     ('2012.1', 'essex'),
     ('2012.2', 'folsom'),
@@ -155,6 +157,8 @@ OPENSTACK_CODENAMES = OrderedDict([
     ('2020.1', 'ussuri'),
     ('2020.2', 'victoria'),
     ('2021.1', 'wallaby'),
+    ('2021.2', 'xena'),
+    ('2022.1', 'yoga'),
 ])
 
 # The ugly duckling - must list releases oldest to newest

hooks/charmhelpers/fetch/ubuntu.py

@@ -208,6 +208,14 @@ CLOUD_ARCHIVE_POCKETS = {
     'wallaby/proposed': 'focal-proposed/wallaby',
     'focal-wallaby/proposed': 'focal-proposed/wallaby',
     'focal-proposed/wallaby': 'focal-proposed/wallaby',
+    # Xena
+    'xena': 'focal-updates/xena',
+    'focal-xena': 'focal-updates/xena',
+    'focal-xena/updates': 'focal-updates/xena',
+    'focal-updates/xena': 'focal-updates/xena',
+    'xena/proposed': 'focal-proposed/xena',
+    'focal-xena/proposed': 'focal-proposed/xena',
+    'focal-proposed/xena': 'focal-proposed/xena',
     # Yoga
     'yoga': 'focal-updates/yoga',
     'focal-yoga': 'focal-updates/yoga',
@@ -272,7 +280,7 @@ UBUNTU_OPENSTACK_RELEASE = OrderedDict([
 
 APT_NO_LOCK = 100  # The return code for "couldn't acquire lock" in APT.
 CMD_RETRY_DELAY = 10  # Wait 10 seconds between command retries.
-CMD_RETRY_COUNT = 3  # Retry a failing fatal command X times.
+CMD_RETRY_COUNT = 10  # Retry a failing fatal command X times.
 
 
 def filter_installed_packages(packages):
@@ -734,7 +742,8 @@ def _add_apt_repository(spec):
         series = get_distrib_codename()
         spec = spec.replace('{series}', series)
     _run_with_retries(['add-apt-repository', '--yes', spec],
-                      cmd_env=env_proxy_settings(['https', 'http']))
+                      cmd_env=env_proxy_settings(['https', 'http', 'no_proxy'])
+                      )
 
 
 def _add_cloud_pocket(pocket):

hooks/charmhelpers/fetch/ubuntu_apt_pkg.py

@@ -264,7 +264,7 @@ def version_compare(a, b):
     else:
         raise RuntimeError('Unable to compare "{}" and "{}", according to '
                            'our logic they are neither greater, equal nor '
-                           'less than each other.')
+                           'less than each other.'.format(a, b))
 
 
 class PkgVersion():

hooks/neutron_ovs_context.py

@@ -344,20 +344,9 @@ class L3AgentContext(OSContextGenerator):
                 NFG_LOG_BURST_LIMIT_MIN
             )
 
-        cmp_os_release = CompareOpenStackReleases(os_release('neutron-common'))
-
         l3_extension_plugins = neutron_api_settings.get(
             'l3_extension_plugins', [])
 
-        # per Change-Id If1b332eb0f581e9acba111f79ba578a0b7081dd2
-        # only enable it for stein although fwaasv2 was added in Queens
-        is_stein = cmp_os_release >= 'stein'
-        if is_stein:
-            l3_extension_plugins.append('fwaas_v2')
-
-        if (is_stein and neutron_api_settings.get('enable_nfg_logging')):
-            l3_extension_plugins.append('fwaas_v2_log')
-
         ctxt['l3_extension_plugins'] = ','.join(l3_extension_plugins)
 
         return ctxt

hooks/neutron_ovs_utils.py

@@ -329,12 +329,16 @@ def determine_packages():
         pkgs.extend(plugin_pkg)
     if use_dvr():
         pkgs.extend(DVR_PACKAGES)
-        py3_pkgs.append('python3-neutron-fwaas')
         _os_release = os_release('neutron-common', base='icehouse')
         # per 17.08 release notes L3HA + DVR is a Newton+ feature
         if (use_l3ha() and
                 CompareOpenStackReleases(_os_release) >= 'newton'):
             pkgs.extend(L3HA_PACKAGES)
+        # python3-neutron-fwaas is already dependency package on
+        # neutron-l3-agent. However this need to be added to py3_pkgs
+        # to support switch from py2 to py3 in Rocky release.
+        if CompareOpenStackReleases(_os_release) == 'rocky':
+            py3_pkgs.append('python3-neutron-fwaas')
     if enable_local_dhcp():
         pkgs.extend(DHCP_PACKAGES)
         pkgs.extend(METADATA_PACKAGES)

unit_tests/test_neutron_ovs_context.py

@@ -372,10 +372,10 @@ class L3AgentContextTest(CharmTestCase):
         rdata = {
             'neutron-security-groups': 'True',
             'enable-dvr': 'True',
+            'enable-fwaas': 'True',
             'l2-population': 'True',
             'overlay-network-type': 'vxlan',
             'network-device-mtu': 1500,
-            'l3_extension_plugins': 'fwaas_v2',
         }
         _rget.side_effect = lambda *args, **kwargs: rdata
         self.assertEqual(
@@ -440,12 +440,12 @@ class L3AgentContextTest(CharmTestCase):
         rdata = {
             'neutron-security-groups': 'True',
             'enable-dvr': 'True',
+            'enable-fwaas': 'True',
             'l2-population': 'True',
             'overlay-network-type': 'vxlan',
             'network-device-mtu': 1500,
             'enable-nfg-logging': 'True',
             'use_l3ha': False,
-            'l3_extension_plugins': 'fwaas_v2,fwaas_v2_log',
         }
         _rget.side_effect = lambda *args, **kwargs: rdata
         _validate_nfg_log_path.side_effect = lambda x: x
@@ -482,11 +482,11 @@ class L3AgentContextTest(CharmTestCase):
         rdata = {
             'neutron-security-groups': 'True',
             'enable-dvr': 'True',
+            'enable-fwaas': 'True',
             'l2-population': 'True',
             'overlay-network-type': 'vxlan',
             'network-device-mtu': 1500,
             'enable-nfg-logging': 'True',
-            'l3_extension_plugins': 'fwaas_v2,fwaas_v2_log',
         }
         _rget.side_effect = lambda *args, **kwargs: rdata
         _validate_nfg_log_path.side_effect = lambda x: x
@@ -522,10 +522,10 @@ class L3AgentContextTest(CharmTestCase):
         rdata = {
             'neutron-security-groups': 'True',
             'enable-dvr': 'True',
+            'enable-fwaas': 'True',
             'l2-population': 'True',
             'overlay-network-type': 'vxlan',
             'network-device-mtu': 1500,
-            'l3_extension_plugins': 'fwaas_v2,fwaas_v2_log',
         }
         _rget.side_effect = lambda *args, **kwargs: rdata
         self.assertEqual(
@@ -555,10 +555,10 @@ class L3AgentContextTest(CharmTestCase):
         rdata = {
             'neutron-security-groups': 'True',
             'enable-dvr': 'False',
+            'enable-fwaas': 'True',
             'l2-population': 'True',
             'overlay-network-type': 'vxlan',
             'network-device-mtu': 1500,
-            'l3_extension_plugins': 'fwaas_v2',
         }
         _rget.side_effect = lambda *args, **kwargs: rdata
         self.assertEqual(context.L3AgentContext()(), {