charmhelpers sync for yoga release

Change-Id: Ie2299afaf122090c466a228b131cdcf0cb7ca49c

charm-helpers-hooks.yaml

@@ -6,6 +6,7 @@ include:
     - cli
     - fetch
     - contrib.openstack|inc=*
+    - contrib.hardware.pci
     - contrib.hahelpers
     - contrib.network.ovs
     - contrib.storage.linux

hooks/charmhelpers/__init__.py

@@ -14,30 +14,15 @@
 
 # Bootstrap charm-helpers, installing its dependencies if necessary using
 # only standard libraries.
-from __future__ import print_function
-from __future__ import absolute_import
-
 import functools
 import inspect
 import subprocess
-import sys
 
-try:
-    import six  # NOQA:F401
-except ImportError:
-    if sys.version_info.major == 2:
-        subprocess.check_call(['apt-get', 'install', '-y', 'python-six'])
-    else:
-        subprocess.check_call(['apt-get', 'install', '-y', 'python3-six'])
-    import six  # NOQA:F401
 
 try:
     import yaml  # NOQA:F401
 except ImportError:
-    if sys.version_info.major == 2:
-        subprocess.check_call(['apt-get', 'install', '-y', 'python-yaml'])
-    else:
-        subprocess.check_call(['apt-get', 'install', '-y', 'python3-yaml'])
+    subprocess.check_call(['apt-get', 'install', '-y', 'python3-yaml'])
     import yaml  # NOQA:F401
 
 

hooks/charmhelpers/cli/__init__.py

@@ -16,9 +16,6 @@ import inspect
 import argparse
 import sys
 
-import six
-from six.moves import zip
-
 import charmhelpers.core.unitdata
 
 
@@ -149,10 +146,7 @@ class CommandLine(object):
     def run(self):
         "Run cli, processing arguments and executing subcommands."
         arguments = self.argument_parser.parse_args()
-        if six.PY2:
-            argspec = inspect.getargspec(arguments.func)
-        else:
-            argspec = inspect.getfullargspec(arguments.func)
+        argspec = inspect.getfullargspec(arguments.func)
         vargs = []
         for arg in argspec.args:
             vargs.append(getattr(arguments, arg))
@@ -177,10 +171,7 @@ def describe_arguments(func):
     Analyze a function's signature and return a data structure suitable for
     passing in as arguments to an argparse parser's add_argument() method."""
 
-    if six.PY2:
-        argspec = inspect.getargspec(func)
-    else:
-        argspec = inspect.getfullargspec(func)
+    argspec = inspect.getfullargspec(func)
     # we should probably raise an exception somewhere if func includes **kwargs
     if argspec.defaults:
         positional_args = argspec.args[:-len(argspec.defaults)]

hooks/charmhelpers/contrib/hahelpers/cluster.py

@@ -32,8 +32,6 @@ import time
 
 from socket import gethostname as get_unit_hostname
 
-import six
-
 from charmhelpers.core.hookenv import (
     log,
     relation_ids,
@@ -125,16 +123,16 @@ def is_crm_dc():
     """
     cmd = ['crm', 'status']
     try:
-        status = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
-        if not isinstance(status, six.text_type):
-            status = six.text_type(status, "utf-8")
+        status = subprocess.check_output(
+            cmd, stderr=subprocess.STDOUT).decode('utf-8')
     except subprocess.CalledProcessError as ex:
         raise CRMDCNotFound(str(ex))
 
     current_dc = ''
     for line in status.split('\n'):
         if line.startswith('Current DC'):
-            # Current DC: juju-lytrusty-machine-2 (168108163) - partition with quorum
+            # Current DC: juju-lytrusty-machine-2 (168108163)
+            #  - partition with quorum
             current_dc = line.split(':')[1].split()[0]
     if current_dc == get_unit_hostname():
         return True
@@ -158,9 +156,8 @@ def is_crm_leader(resource, retry=False):
         return is_crm_dc()
     cmd = ['crm', 'resource', 'show', resource]
     try:
-        status = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
-        if not isinstance(status, six.text_type):
-            status = six.text_type(status, "utf-8")
+        status = subprocess.check_output(
+            cmd, stderr=subprocess.STDOUT).decode('utf-8')
     except subprocess.CalledProcessError:
         status = None
 

hooks/charmhelpers/contrib/hardware/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2022 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

hooks/charmhelpers/contrib/hardware/pci.py

@@ -0,0 +1,288 @@
+#!/usr/bin/env python3
+#
+# Copyright 2016-2022 Canonical Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import glob
+import itertools
+import logging
+import os
+import re
+import shlex
+import subprocess
+import typing
+
+
+def format_pci_addr(pci_addr: str) -> str:
+    """Format a PCI address with 0 fill for parts
+
+    :param: pci_addr: unformatted PCI address
+    :type: str
+    :returns: formatted PCI address
+    :rtype: str
+    """
+    domain, bus, slot_func = pci_addr.split(":")
+    slot, func = slot_func.split(".")
+    return "{}:{}:{}.{}".format(
+        domain.zfill(4), bus.zfill(2), slot.zfill(2), func
+    )
+
+
+def get_sysnet_interfaces_and_macs() -> list:
+    """Catalog interface information from local system
+
+    each device dict contains:
+
+        interface: logical name
+        mac_address: MAC address
+        pci_address: PCI address
+        state: Current interface state (up/down)
+        sriov: Boolean indicating whether interface is an SR-IOV
+               capable device.
+        sriov_totalvfs: Total VF capacity of device
+        sriov_numvfs: Configured VF capacity of device
+
+    :returns: array of dict objects containing details of each interface
+    :rtype: list
+    """
+    net_devs = []
+    for sdir in itertools.chain(
+            glob.glob("/sys/bus/pci/devices/*/net/../"),
+            glob.glob("/sys/bus/pci/devices/*/virtio*/net/../")):
+        fq_path = os.path.realpath(sdir)
+        path = fq_path.split("/")
+        if "virtio" in path[-1]:
+            pci_address = path[-2]
+        else:
+            pci_address = path[-1]
+        ifname = get_sysnet_interface(sdir)
+        if not ifname:
+            logging.warn("Unable to determine interface name for PCI "
+                         "device {}".format(pci_address))
+            continue
+        device = {
+            "interface": ifname,
+            "mac_address": get_sysnet_mac(sdir, ifname),
+            "pci_address": pci_address,
+            "state": get_sysnet_device_state(sdir, ifname),
+            "sriov": is_sriov(sdir),
+        }
+        if device["sriov"]:
+            device["sriov_totalvfs"] = get_sriov_totalvfs(sdir)
+            device["sriov_numvfs"] = get_sriov_numvfs(sdir)
+        net_devs.append(device)
+
+    return net_devs
+
+
+def get_sysnet_mac(sysdir: str, ifname: str) -> str:
+    """Determine MAC address for a device
+
+    :param: sysdir: path to device /sys directory
+    :type: str
+    :returns: MAC address of device
+    :rtype: str
+    """
+    mac_addr_file = os.path.join(sysdir, "net", ifname, "address")
+    with open(mac_addr_file, "r") as f:
+        read_data = f.read()
+    return read_data.strip()
+
+
+def get_sysnet_device_state(sysdir: str, ifname: str) -> str:
+    """Read operational state of a device
+
+    :param: sysdir: path to device /sys directory
+    :type: str
+    :returns: current device state
+    :rtype: str
+    """
+    state_file = os.path.join(sysdir, "net", ifname, "operstate")
+    with open(state_file, "r") as f:
+        read_data = f.read()
+    return read_data.strip()
+
+
+def is_sriov(sysdir: str) -> bool:
+    """Determine whether a device is SR-IOV capable
+
+    :param: sysdir: path to device /sys directory
+    :type: str
+    :returns: whether device is SR-IOV capable or not
+    :rtype: bool
+    """
+    return os.path.exists(os.path.join(sysdir, "sriov_totalvfs"))
+
+
+def get_sriov_totalvfs(sysdir: str) -> int:
+    """Read total VF capacity for a device
+
+    :param: sysdir: path to device /sys directory
+    :type: str
+    :returns: number of VF's the device supports
+    :rtype: int
+    """
+    sriov_totalvfs_file = os.path.join(sysdir, "sriov_totalvfs")
+    with open(sriov_totalvfs_file, "r") as f:
+        read_data = f.read()
+    return int(read_data.strip())
+
+
+def get_sriov_numvfs(sysdir: str) -> int:
+    """Read configured VF capacity for a device
+
+    :param: sysdir: path to device /sys directory
+    :type: str
+    :returns: number of VF's the device is configured with
+    :rtype: int
+    """
+    sriov_numvfs_file = os.path.join(sysdir, "sriov_numvfs")
+    with open(sriov_numvfs_file, "r") as f:
+        read_data = f.read()
+    return int(read_data.strip())
+
+
+# https://github.com/libvirt/libvirt/commit/5b1c525b1f3608156884aed0dc5e925306c1e260
+PF_PHYS_PORT_NAME_REGEX = re.compile(r"(p[0-9]+$)|(p[0-9]+s[0-9]+$)",
+                                     re.IGNORECASE)
+
+
+def _phys_port_name_is_pf(sysnetdir: str) -> typing.Optional[bool]:
+    try:
+        with open(os.path.join(sysnetdir, "phys_port_name"), "r") as fin:
+            return (PF_PHYS_PORT_NAME_REGEX.match(fin.read().strip())
+                    is not None)
+    except OSError:
+        return
+
+
+def get_sysnet_interface(sysdir: str) -> typing.Optional[str]:
+    sysnetdir = os.path.join(sysdir, "net")
+    netdevs = os.listdir(sysnetdir)
+    # Return early in case the PCI device only has one netdev
+    if len(netdevs) == 1:
+        return netdevs[0]
+
+    # When a PCI device has multiple netdevs we need to figure out which one
+    # represents the PF
+    for netdev in netdevs:
+        if _phys_port_name_is_pf(os.path.join(sysnetdir, netdev)):
+            return netdev
+
+
+def get_pci_ethernet_addresses() -> list:
+    """Generate list of PCI addresses for all network adapters
+
+    :returns: list of PCI addresses
+    :rtype: list
+    """
+    cmd = ["lspci", "-m", "-D"]
+    lspci_output = subprocess.check_output(cmd).decode("UTF-8")
+    pci_addresses = []
+    for line in lspci_output.split("\n"):
+        columns = shlex.split(line)
+        if len(columns) > 1 and columns[1] == "Ethernet controller":
+            pci_address = columns[0]
+            pci_addresses.append(format_pci_addr(pci_address))
+    return pci_addresses
+
+
+class PCINetDevice(object):
+    def __init__(self, pci_address):
+        self.pci_address = pci_address
+        self.interface_name = None
+        self.mac_address = None
+        self.state = None
+        self.sriov = False
+        self.sriov_totalvfs = None
+        self.sriov_numvfs = None
+        self.update_attributes()
+
+    def update_attributes(self):
+        self.update_interface_info()
+
+    def update_interface_info(self):
+        net_devices = get_sysnet_interfaces_and_macs()
+        for interface in net_devices:
+            if self.pci_address == interface["pci_address"]:
+                self.interface_name = interface["interface"]
+                self.mac_address = interface["mac_address"]
+                self.state = interface["state"]
+                self.sriov = interface["sriov"]
+                if self.sriov:
+                    self.sriov_totalvfs = interface["sriov_totalvfs"]
+                    self.sriov_numvfs = interface["sriov_numvfs"]
+
+    def _set_sriov_numvfs(self, numvfs: int):
+        sdevice = os.path.join(
+            "/sys/bus/pci/devices", self.pci_address, "sriov_numvfs"
+        )
+        with open(sdevice, "w") as sh:
+            sh.write(str(numvfs))
+        self.update_attributes()
+
+    def set_sriov_numvfs(self, numvfs: int) -> bool:
+        """Set the number of VF devices for a SR-IOV PF
+
+        Assuming the device is an SR-IOV device, this function will attempt
+        to change the number of VF's created by the PF.
+
+        @param numvfs: integer to set the current number of VF's to
+        @returns boolean indicating whether any changes where made
+        """
+        if self.sriov and numvfs != self.sriov_numvfs:
+            # NOTE(fnordahl): run-time change of numvfs is disallowed
+            # without resetting to 0 first.
+            self._set_sriov_numvfs(0)
+            self._set_sriov_numvfs(numvfs)
+            return True
+        return False
+
+
+class PCINetDevices(object):
+    def __init__(self):
+        self.pci_devices = [
+            PCINetDevice(dev) for dev in get_pci_ethernet_addresses()
+        ]
+
+    def update_devices(self):
+        for pcidev in self.pci_devices:
+            pcidev.update_attributes()
+
+    def get_macs(self) -> list:
+        macs = []
+        for pcidev in self.pci_devices:
+            if pcidev.mac_address:
+                macs.append(pcidev.mac_address)
+        return macs
+
+    def get_device_from_mac(self, mac: str) -> PCINetDevice:
+        for pcidev in self.pci_devices:
+            if pcidev.mac_address == mac:
+                return pcidev
+        return None
+
+    def get_device_from_pci_address(self, pci_addr: str) -> PCINetDevice:
+        for pcidev in self.pci_devices:
+            if pcidev.pci_address == pci_addr:
+                return pcidev
+        return None
+
+    def get_device_from_interface_name(
+        self, interface_name: str
+    ) -> PCINetDevice:
+        for pcidev in self.pci_devices:
+            if pcidev.interface_name == interface_name:
+                return pcidev
+        return None

hooks/charmhelpers/contrib/network/ip.py

@@ -15,7 +15,6 @@
 import glob
 import re
 import subprocess
-import six
 import socket
 
 from functools import partial
@@ -39,20 +38,14 @@ try:
     import netifaces
 except ImportError:
     apt_update(fatal=True)
-    if six.PY2:
-        apt_install('python-netifaces', fatal=True)
-    else:
-        apt_install('python3-netifaces', fatal=True)
+    apt_install('python3-netifaces', fatal=True)
     import netifaces
 
 try:
     import netaddr
 except ImportError:
     apt_update(fatal=True)
-    if six.PY2:
-        apt_install('python-netaddr', fatal=True)
-    else:
-        apt_install('python3-netaddr', fatal=True)
+    apt_install('python3-netaddr', fatal=True)
     import netaddr
 
 
@@ -462,15 +455,12 @@ def ns_query(address):
     try:
         import dns.resolver
     except ImportError:
-        if six.PY2:
-            apt_install('python-dnspython', fatal=True)
-        else:
-            apt_install('python3-dnspython', fatal=True)
+        apt_install('python3-dnspython', fatal=True)
         import dns.resolver
 
     if isinstance(address, dns.name.Name):
         rtype = 'PTR'
-    elif isinstance(address, six.string_types):
+    elif isinstance(address, str):
         rtype = 'A'
     else:
         return None
@@ -513,10 +503,7 @@ def get_hostname(address, fqdn=True):
         try:
             import dns.reversename
         except ImportError:
-            if six.PY2:
-                apt_install("python-dnspython", fatal=True)
-            else:
-                apt_install("python3-dnspython", fatal=True)
+            apt_install("python3-dnspython", fatal=True)
             import dns.reversename
 
         rev = dns.reversename.from_address(address)

hooks/charmhelpers/contrib/network/ovs/__init__.py

@@ -17,7 +17,6 @@ import collections
 import hashlib
 import os
 import re
-import six
 import subprocess
 
 from charmhelpers import deprecate
@@ -367,10 +366,7 @@ def add_ovsbridge_linuxbridge(name, bridge, ifdata=None, portdata=None):
     try:
         import netifaces
     except ImportError:
-        if six.PY2:
-            apt_install('python-netifaces', fatal=True)
-        else:
-            apt_install('python3-netifaces', fatal=True)
+        apt_install('python3-netifaces', fatal=True)
         import netifaces
 
     # NOTE(jamespage):

hooks/charmhelpers/contrib/network/ovs/ovsdb.py

@@ -126,24 +126,27 @@ class SimpleOVSDB(object):
         ),
     }
 
-    def __init__(self, tool):
+    def __init__(self, tool, args=None):
         """SimpleOVSDB constructor.
 
         :param tool: Which tool with database commands to operate on.
                      Usually one of `ovs-vsctl`, `ovn-nbctl`, `ovn-sbctl`
         :type tool: str
+        :param args: Extra arguments to pass to the tool
+        :type args: Optional[List[str]]
         """
         if tool not in self._tool_table_map:
             raise RuntimeError(
                 'tool must be one of "{}"'.format(self._tool_table_map.keys()))
         self._tool = tool
+        self._args = args
 
     def __getattr__(self, table):
         if table not in self._tool_table_map[self._tool]:
             raise AttributeError(
                 'table "{}" not known for use with "{}"'
                 .format(table, self._tool))
-        return self.Table(self._tool, table)
+        return self.Table(self._tool, table, args=self._args)
 
     class Table(object):
         """Methods to interact with contents of OVSDB tables.
@@ -153,14 +156,17 @@ class SimpleOVSDB(object):
         JSON output.
         """
 
-        def __init__(self, tool, table):
+        def __init__(self, tool, table, args=None):
             """SimpleOVSDBTable constructor.
 
             :param table: Which table to operate on
             :type table: str
+            :param args: Extra arguments to pass to the tool
+            :type args: Optional[List[str]]
             """
             self._tool = tool
             self._table = table
+            self._args = args
 
         def _deserialize_ovsdb(self, data):
             """Deserialize OVSDB RFC7047 section 5.1 data.
@@ -215,7 +221,10 @@ class SimpleOVSDB(object):
             :returns: Dictionary with data
             :rtype: Dict[str, any]
             """
-            cmd = [self._tool, '-f', 'json', 'find', self._table]
+            cmd = [self._tool]
+            if self._args:
+                cmd.extend(self._args)
+            cmd.extend(['-f', 'json', 'find', self._table])
             if condition:
                 cmd.append(condition)
             output = utils._run(*cmd)

hooks/charmhelpers/contrib/openstack/context.py

@@ -30,8 +30,6 @@ from subprocess import (
     check_output,
     CalledProcessError)
 
-import six
-
 import charmhelpers.contrib.storage.linux.ceph as ch_ceph
 
 from charmhelpers.contrib.openstack.audits.openstack_security_guide import (
@@ -120,20 +118,12 @@ from charmhelpers.contrib.openstack.utils import (
 )
 from charmhelpers.core.unitdata import kv
 
-try:
-    from sriov_netplan_shim import pci
-except ImportError:
-    # The use of the function and contexts that require the pci module is
-    # optional.
-    pass
+from charmhelpers.contrib.hardware import pci
 
 try:
     import psutil
 except ImportError:
-    if six.PY2:
-        apt_install('python-psutil', fatal=True)
-    else:
-        apt_install('python3-psutil', fatal=True)
+    apt_install('python3-psutil', fatal=True)
     import psutil
 
 CA_CERT_PATH = '/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt'
@@ -150,10 +140,7 @@ def ensure_packages(packages):
 
 
 def context_complete(ctxt):
-    _missing = []
-    for k, v in six.iteritems(ctxt):
-        if v is None or v == '':
-            _missing.append(k)
+    _missing = [k for k, v in ctxt.items() if v is None or v == '']
 
     if _missing:
         log('Missing required data: %s' % ' '.join(_missing), level=INFO)
@@ -180,7 +167,7 @@ class OSContextGenerator(object):
         # Fresh start
         self.complete = False
         self.missing_data = []
-        for k, v in six.iteritems(ctxt):
+        for k, v in ctxt.items():
             if v is None or v == '':
                 if k not in self.missing_data:
                     self.missing_data.append(k)
@@ -434,6 +421,9 @@ class IdentityServiceContext(OSContextGenerator):
             ('password', ctxt.get('admin_password', '')),
             ('signing_dir', ctxt.get('signing_dir', '')),))
 
+        if ctxt.get('service_type'):
+            c.update((('service_type', ctxt.get('service_type')),))
+
         return c
 
     def __call__(self):
@@ -476,6 +466,9 @@ class IdentityServiceContext(OSContextGenerator):
                              'internal_protocol': int_protocol,
                              'api_version': api_version})
 
+                if rdata.get('service_type'):
+                    ctxt['service_type'] = rdata.get('service_type')
+
                 if float(api_version) > 2:
                     ctxt.update({
                         'admin_domain_name': rdata.get('service_domain'),
@@ -547,6 +540,9 @@ class IdentityCredentialsContext(IdentityServiceContext):
                     'api_version': api_version
                 })
 
+                if rdata.get('service_type'):
+                    ctxt['service_type'] = rdata.get('service_type')
+
                 if float(api_version) > 2:
                     ctxt.update({'admin_domain_name':
                                  rdata.get('domain')})
@@ -1111,10 +1107,14 @@ class ApacheSSLContext(OSContextGenerator):
             endpoint = resolve_address(net_type)
             addresses.append((addr, endpoint))
 
-        return sorted(set(addresses))
+        # Log the set of addresses to have a trail log and capture if tuples
+        # change over time in the same unit (LP: #1952414).
+        sorted_addresses = sorted(set(addresses))
+        log('get_network_addresses: {}'.format(sorted_addresses))
+        return sorted_addresses
 
     def __call__(self):
-        if isinstance(self.external_ports, six.string_types):
+        if isinstance(self.external_ports, str):
             self.external_ports = [self.external_ports]
 
         if not self.external_ports or not https():
@@ -1531,9 +1531,9 @@ class SubordinateConfigContext(OSContextGenerator):
                             continue
 
                         sub_config = sub_config[self.config_file]
-                        for k, v in six.iteritems(sub_config):
+                        for k, v in sub_config.items():
                             if k == 'sections':
-                                for section, config_list in six.iteritems(v):
+                                for section, config_list in v.items():
                                     log("adding section '%s'" % (section),
                                         level=DEBUG)
                                     if ctxt[k].get(section):
@@ -1887,8 +1887,11 @@ class DataPortContext(NeutronPortContext):
             normalized.update({port: port for port in resolved
                                if port in ports})
             if resolved:
-                return {normalized[port]: bridge for port, bridge in
-                        six.iteritems(portmap) if port in normalized.keys()}
+                return {
+                    normalized[port]: bridge
+                    for port, bridge in portmap.items()
+                    if port in normalized.keys()
+                }
 
         return None
 
@@ -2291,15 +2294,10 @@ class HostInfoContext(OSContextGenerator):
         name = name or socket.gethostname()
         fqdn = ''
 
-        if six.PY2:
-            exc = socket.error
-        else:
-            exc = OSError
-
         try:
             addrs = socket.getaddrinfo(
                 name, None, 0, socket.SOCK_DGRAM, 0, socket.AI_CANONNAME)
-        except exc:
+        except OSError:
             pass
         else:
             for addr in addrs:
@@ -2416,12 +2414,12 @@ class DHCPAgentContext(OSContextGenerator):
         existing_ovs_use_veth = None
         # If there is a dhcp_agent.ini file read the current setting
         if os.path.isfile(DHCP_AGENT_INI):
-            # config_ini does the right thing and returns None if the setting is
-            # commented.
+            # config_ini does the right thing and returns None if the setting
+            # is commented.
             existing_ovs_use_veth = (
                 config_ini(DHCP_AGENT_INI)["DEFAULT"].get("ovs_use_veth"))
         # Convert to Bool if necessary
-        if isinstance(existing_ovs_use_veth, six.string_types):
+        if isinstance(existing_ovs_use_veth, str):
             return bool_from_string(existing_ovs_use_veth)
         return existing_ovs_use_veth
 
@@ -3126,7 +3124,7 @@ class SRIOVContext(OSContextGenerator):
         """Determine number of Virtual Functions (VFs) configured for device.
 
         :param device: Object describing a PCI Network interface card (NIC)/
-        :type device: sriov_netplan_shim.pci.PCINetDevice
+        :type device: contrib.hardware.pci.PCINetDevice
         :param sriov_numvfs: Number of VFs requested for blanket configuration.
         :type sriov_numvfs: int
         :returns: Number of VFs to configure for device

hooks/charmhelpers/contrib/openstack/files/check_deferred_restarts.py

@@ -0,0 +1,128 @@
+#!/usr/bin/python3
+
+# Copyright 2014-2022 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Checks for services with deferred restarts.
+
+This Nagios check will parse /var/lib/policy-rd.d/
+to find any restarts that are currently deferred.
+"""
+
+import argparse
+import glob
+import sys
+import yaml
+
+
+DEFERRED_EVENTS_DIR = '/var/lib/policy-rc.d'
+
+
+def get_deferred_events():
+    """Return a list of deferred events dicts from policy-rc.d files.
+
+    Events are read from DEFERRED_EVENTS_DIR and are of the form:
+    {
+        action: restart,
+        policy_requestor_name: rabbitmq-server,
+        policy_requestor_type: charm,
+        reason: 'Pkg update',
+        service: rabbitmq-server,
+        time: 1614328743
+    }
+
+    :raises OSError: Raised in case of a system error while reading a policy file
+    :raises yaml.YAMLError: Raised if parsing a policy file fails
+
+    :returns: List of deferred event dictionaries
+    :rtype: list
+    """
+    deferred_events_files = glob.glob(
+        '{}/*.deferred'.format(DEFERRED_EVENTS_DIR))
+
+    deferred_events = []
+    for event_file in deferred_events_files:
+        with open(event_file, 'r') as f:
+            event = yaml.safe_load(f)
+            deferred_events.append(event)
+
+    return deferred_events
+
+
+def get_deferred_restart_services(application=None):
+    """Returns a list of services with deferred restarts.
+
+    :param str application: Name of the application that blocked the service restart.
+                            If application is None, all services with deferred restarts
+                            are returned. Services which are blocked by a non-charm
+                            requestor are always returned.
+
+    :raises OSError: Raised in case of a system error while reading a policy file
+    :raises yaml.YAMLError: Raised if parsing a policy file fails
+
+    :returns: List of services with deferred restarts belonging to application.
+    :rtype: list
+    """
+
+    deferred_restart_events = filter(
+        lambda e: e['action'] == 'restart', get_deferred_events())
+
+    deferred_restart_services = set()
+    for restart_event in deferred_restart_events:
+        if application:
+            if (
+                restart_event['policy_requestor_type'] != 'charm' or
+                restart_event['policy_requestor_type'] == 'charm' and
+                restart_event['policy_requestor_name'] == application
+            ):
+                deferred_restart_services.add(restart_event['service'])
+        else:
+            deferred_restart_services.add(restart_event['service'])
+
+    return list(deferred_restart_services)
+
+
+def main():
+    """Check for services with deferred restarts."""
+    parser = argparse.ArgumentParser(
+        description='Check for services with deferred restarts')
+    parser.add_argument(
+        '--application', help='Check services belonging to this application only')
+
+    args = parser.parse_args()
+
+    services = set(get_deferred_restart_services(args.application))
+
+    if len(services) == 0:
+        print('OK: No deferred service restarts.')
+        sys.exit(0)
+    else:
+        print(
+            'CRITICAL: Restarts are deferred for services: {}.'.format(', '.join(services)))
+        sys.exit(1)
+
+
+if __name__ == '__main__':
+    try:
+        main()
+    except OSError as e:
+        print('CRITICAL: A system error occurred: {} ({})'.format(e.errno, e.strerror))
+        sys.exit(1)
+    except yaml.YAMLError as e:
+        print('CRITICAL: Failed to parse a policy file: {}'.format(str(e)))
+        sys.exit(1)
+    except Exception as e:
+        print('CRITICAL: An unknown error occurred: {}'.format(str(e)))
+        sys.exit(1)

hooks/charmhelpers/contrib/openstack/keystone.py

@@ -1,4 +1,3 @@
-#!/usr/bin/python
 #
 # Copyright 2017 Canonical Ltd
 #
@@ -14,7 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import six
 from charmhelpers.fetch import apt_install
 from charmhelpers.contrib.openstack.context import IdentityServiceContext
 from charmhelpers.core.hookenv import (
@@ -117,10 +115,7 @@ class KeystoneManager2(KeystoneManager):
             from keystoneclient.auth.identity import v2
             from keystoneclient import session
         except ImportError:
-            if six.PY2:
-                apt_install(["python-keystoneclient"], fatal=True)
-            else:
-                apt_install(["python3-keystoneclient"], fatal=True)
+            apt_install(["python3-keystoneclient"], fatal=True)
 
             from keystoneclient.v2_0 import client
             from keystoneclient.auth.identity import v2
@@ -151,10 +146,7 @@ class KeystoneManager3(KeystoneManager):
             from keystoneclient import session
             from keystoneclient.auth.identity import v3
         except ImportError:
-            if six.PY2:
-                apt_install(["python-keystoneclient"], fatal=True)
-            else:
-                apt_install(["python3-keystoneclient"], fatal=True)
+            apt_install(["python3-keystoneclient"], fatal=True)
 
             from keystoneclient.v3 import client
             from keystoneclient.auth import token_endpoint

hooks/charmhelpers/contrib/openstack/neutron.py

@@ -14,7 +14,6 @@
 
 # Various utilities for dealing with Neutron and the renaming from Quantum.
 
-import six
 from subprocess import check_output
 
 from charmhelpers.core.hookenv import (
@@ -349,11 +348,4 @@ def parse_vlan_range_mappings(mappings):
     Returns dict of the form {provider: (start, end)}.
     """
     _mappings = parse_mappings(mappings)
-    if not _mappings:
-        return {}
-
-    mappings = {}
-    for p, r in six.iteritems(_mappings):
-        mappings[p] = tuple(r.split(':'))
-
-    return mappings
+    return {p: tuple(r.split(':')) for p, r in _mappings.items()}

hooks/charmhelpers/contrib/openstack/policyd.py

@@ -15,7 +15,6 @@
 import collections
 import contextlib
 import os
-import six
 import shutil
 import yaml
 import zipfile
@@ -204,12 +203,6 @@ class BadPolicyYamlFile(Exception):
         return self.log_message
 
 
-if six.PY2:
-    BadZipFile = zipfile.BadZipfile
-else:
-    BadZipFile = zipfile.BadZipFile
-
-
 def is_policyd_override_valid_on_this_release(openstack_release):
     """Check that the charm is running on at least Ubuntu Xenial, and at
     least the queens release.
@@ -487,10 +480,10 @@ def read_and_validate_yaml(stream_or_doc, blacklist_keys=None):
     if blacklisted_keys_present:
         raise BadPolicyYamlFile("blacklisted keys {} present."
                                 .format(", ".join(blacklisted_keys_present)))
-    if not all(isinstance(k, six.string_types) for k in keys):
+    if not all(isinstance(k, str) for k in keys):
         raise BadPolicyYamlFile("keys in yaml aren't all strings?")
     # check that the dictionary looks like a mapping of str to str
-    if not all(isinstance(v, six.string_types) for v in doc.values()):
+    if not all(isinstance(v, str) for v in doc.values()):
         raise BadPolicyYamlFile("values in yaml aren't all strings?")
     return doc
 
@@ -530,8 +523,7 @@ def clean_policyd_dir_for(service, keep_paths=None, user=None, group=None):
     hookenv.log("Cleaning path: {}".format(path), level=hookenv.DEBUG)
     if not os.path.exists(path):
         ch_host.mkdir(path, owner=_user, group=_group, perms=0o775)
-    _scanner = os.scandir if hasattr(os, 'scandir') else _fallback_scandir
-    for direntry in _scanner(path):
+    for direntry in os.scandir(path):
         # see if the path should be kept.
         if direntry.path in keep_paths:
             continue
@@ -558,36 +550,6 @@ def maybe_create_directory_for(path, user, group):
         ch_host.mkdir(_dir, owner=user, group=group, perms=0o775)
 
 
-@contextlib.contextmanager
-def _fallback_scandir(path):
-    """Fallback os.scandir implementation.
-
-    provide a fallback implementation of os.scandir if this module ever gets
-    used in a py2 or py34 charm. Uses os.listdir() to get the names in the path,
-    and then mocks the is_dir() function using os.path.isdir() to check for
-    directory.
-
-    :param path: the path to list the directories for
-    :type path: str
-    :returns: Generator that provides _FBDirectory objects
-    :rtype: ContextManager[_FBDirectory]
-    """
-    for f in os.listdir(path):
-        yield _FBDirectory(f)
-
-
-class _FBDirectory(object):
-    """Mock a scandir Directory object with enough to use in
-    clean_policyd_dir_for
-    """
-
-    def __init__(self, path):
-        self.path = path
-
-    def is_dir(self):
-        return os.path.isdir(self.path)
-
-
 def path_for_policy_file(service, name):
     """Return the full path for a policy.d file that will be written to the
     service's policy.d directory.
@@ -768,7 +730,7 @@ def process_policy_resource_file(resource_file,
                                    _group)
         # Every thing worked, so we mark up a success.
         completed = True
-    except (BadZipFile, BadPolicyZipFile, BadPolicyYamlFile) as e:
+    except (zipfile.BadZipFile, BadPolicyZipFile, BadPolicyYamlFile) as e:
         hookenv.log("Processing {} failed: {}".format(resource_file, str(e)),
                     level=POLICYD_LOG_LEVEL_DEFAULT)
     except IOError as e:

hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg

@@ -82,7 +82,11 @@ backend {{ service }}_{{ frontend }}
     {% endif -%}
     {% endif -%}
     {% for unit, address in frontends[frontend]['backends'].items() -%}
+    {% if https -%}
+    server {{ unit }} {{ address }}:{{ ports[1] }} check check-ssl verify none
+    {% else -%}
     server {{ unit }} {{ address }}:{{ ports[1] }} check
+    {% endif -%}
     {% endfor %}
 {% endfor -%}
 {% endfor -%}

hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend

@@ -22,6 +22,8 @@ Listen {{ ext_port }}
     ProxyPassReverse / http://localhost:{{ int }}/
     ProxyPreserveHost on
     RequestHeader set X-Forwarded-Proto "https"
+    KeepAliveTimeout 75
+    MaxKeepAliveRequests 1000
 </VirtualHost>
 {% endfor -%}
 <Proxy *>

hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend.conf

@@ -22,6 +22,8 @@ Listen {{ ext_port }}
     ProxyPassReverse / http://localhost:{{ int }}/
     ProxyPreserveHost on
     RequestHeader set X-Forwarded-Proto "https"
+    KeepAliveTimeout 75
+    MaxKeepAliveRequests 1000
 </VirtualHost>
 {% endfor -%}
 <Proxy *>

hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken

@@ -9,4 +9,7 @@ project_name = {{ admin_tenant_name }}
 username = {{ admin_user }}
 password = {{ admin_password }}
 signing_dir = {{ signing_dir }}
+{% if service_type -%}
+service_type = {{ service_type }}
+{% endif -%}
 {% endif -%}

hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken-mitaka

@@ -6,6 +6,9 @@ auth_uri = {{ service_protocol }}://{{ service_host }}:{{ service_port }}/v3
 auth_url = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}/v3
 project_domain_name = {{ admin_domain_name }}
 user_domain_name = {{ admin_domain_name }}
+{% if service_type -%}
+service_type = {{ service_type }}
+{% endif -%}
 {% else -%}
 auth_uri = {{ service_protocol }}://{{ service_host }}:{{ service_port }}
 auth_url = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}

hooks/charmhelpers/contrib/openstack/templates/wsgi-openstack-api.conf

@@ -20,6 +20,8 @@ Listen {{ public_port }}
     WSGIScriptAlias / {{ script }}
     WSGIApplicationGroup %{GLOBAL}
     WSGIPassAuthorization On
+    KeepAliveTimeout 75
+    MaxKeepAliveRequests 1000
     <IfVersion >= 2.4>
       ErrorLogFormat "%{cu}t %M"
     </IfVersion>
@@ -46,6 +48,8 @@ Listen {{ public_port }}
     WSGIScriptAlias / {{ admin_script }}
     WSGIApplicationGroup %{GLOBAL}
     WSGIPassAuthorization On
+    KeepAliveTimeout 75
+    MaxKeepAliveRequests 1000
     <IfVersion >= 2.4>
       ErrorLogFormat "%{cu}t %M"
     </IfVersion>
@@ -72,6 +76,8 @@ Listen {{ public_port }}
     WSGIScriptAlias / {{ public_script }}
     WSGIApplicationGroup %{GLOBAL}
     WSGIPassAuthorization On
+    KeepAliveTimeout 75
+    MaxKeepAliveRequests 1000
     <IfVersion >= 2.4>
       ErrorLogFormat "%{cu}t %M"
     </IfVersion>

hooks/charmhelpers/contrib/openstack/templates/wsgi-openstack-metadata.conf

@@ -20,6 +20,8 @@ Listen {{ public_port }}
     WSGIScriptAlias / {{ script }}
     WSGIApplicationGroup %{GLOBAL}
     WSGIPassAuthorization On
+    KeepAliveTimeout 75
+    MaxKeepAliveRequests 1000
     <IfVersion >= 2.4>
       ErrorLogFormat "%{cu}t %M"
     </IfVersion>
@@ -46,6 +48,8 @@ Listen {{ public_port }}
     WSGIScriptAlias / {{ admin_script }}
     WSGIApplicationGroup %{GLOBAL}
     WSGIPassAuthorization On
+    KeepAliveTimeout 75
+    MaxKeepAliveRequests 1000
     <IfVersion >= 2.4>
       ErrorLogFormat "%{cu}t %M"
     </IfVersion>
@@ -72,6 +76,8 @@ Listen {{ public_port }}
     WSGIScriptAlias / {{ public_script }}
     WSGIApplicationGroup %{GLOBAL}
     WSGIPassAuthorization On
+    KeepAliveTimeout 75
+    MaxKeepAliveRequests 1000
     <IfVersion >= 2.4>
       ErrorLogFormat "%{cu}t %M"
     </IfVersion>

hooks/charmhelpers/contrib/openstack/templating.py

@@ -14,8 +14,6 @@
 
 import os
 
-import six
-
 from charmhelpers.fetch import apt_install, apt_update
 from charmhelpers.core.hookenv import (
     log,
@@ -29,10 +27,7 @@ try:
     from jinja2 import FileSystemLoader, ChoiceLoader, Environment, exceptions
 except ImportError:
     apt_update(fatal=True)
-    if six.PY2:
-        apt_install('python-jinja2', fatal=True)
-    else:
-        apt_install('python3-jinja2', fatal=True)
+    apt_install('python3-jinja2', fatal=True)
     from jinja2 import FileSystemLoader, ChoiceLoader, Environment, exceptions
 
 
@@ -62,7 +57,7 @@ def get_loader(templates_dir, os_release):
         order by OpenStack release.
     """
     tmpl_dirs = [(rel, os.path.join(templates_dir, rel))
-                 for rel in six.itervalues(OPENSTACK_CODENAMES)]
+                 for rel in OPENSTACK_CODENAMES.values()]
 
     if not os.path.isdir(templates_dir):
         log('Templates directory not found @ %s.' % templates_dir,
@@ -225,10 +220,7 @@ class OSConfigRenderer(object):
             # if this code is running, the object is created pre-install hook.
             # jinja2 shouldn't get touched until the module is reloaded on next
             # hook execution, with proper jinja2 bits successfully imported.
-            if six.PY2:
-                apt_install('python-jinja2')
-            else:
-                apt_install('python3-jinja2')
+            apt_install('python3-jinja2')
 
     def register(self, config_file, contexts, config_template=None):
         """
@@ -318,9 +310,7 @@ class OSConfigRenderer(object):
             log('Config not registered: %s' % config_file, level=ERROR)
             raise OSConfigException
 
-        _out = self.render(config_file)
-        if six.PY3:
-            _out = _out.encode('UTF-8')
+        _out = self.render(config_file).encode('UTF-8')
 
         with open(config_file, 'wb') as out:
             out.write(_out)
@@ -331,7 +321,8 @@ class OSConfigRenderer(object):
         """
         Write out all registered config files.
         """
-        [self.write(k) for k in six.iterkeys(self.templates)]
+        for k in self.templates.keys():
+            self.write(k)
 
     def set_release(self, openstack_release):
         """
@@ -347,8 +338,8 @@ class OSConfigRenderer(object):
         Returns a list of context interfaces that yield a complete context.
         '''
         interfaces = []
-        [interfaces.extend(i.complete_contexts())
-         for i in six.itervalues(self.templates)]
+        for i in self.templates.values():
+            interfaces.extend(i.complete_contexts())
         return interfaces
 
     def get_incomplete_context_data(self, interfaces):
@@ -360,7 +351,7 @@ class OSConfigRenderer(object):
         '''
         incomplete_context_data = {}
 
-        for i in six.itervalues(self.templates):
+        for i in self.templates.values():
             for context in i.contexts:
                 for interface in interfaces:
                     related = False

hooks/charmhelpers/contrib/openstack/utils.py

@@ -25,7 +25,6 @@ import re
 import itertools
 import functools
 
-import six
 import traceback
 import uuid
 import yaml
@@ -362,6 +361,8 @@ def get_os_codename_install_source(src):
     rel = ''
     if src is None:
         return rel
+    if src in OPENSTACK_RELEASES:
+        return src
     if src in ['distro', 'distro-proposed', 'proposed']:
         try:
             rel = UBUNTU_OPENSTACK_RELEASE[ubuntu_rel]
@@ -401,7 +402,7 @@ def get_os_codename_version(vers):
 
 def get_os_version_codename(codename, version_map=OPENSTACK_CODENAMES):
     '''Determine OpenStack version number from codename.'''