Charmhelper sync for 20.02

Change-Id: I29a4d9cb49801d3d19c2e6a00b245ade05e9c32a

charmhelpers/contrib/hahelpers/cluster.py

@@ -25,6 +25,7 @@ Helpers for clustering and determining "cluster leadership" and other
 clustering-related helpers.
 """
 
+import functools
 import subprocess
 import os
 import time
@@ -281,6 +282,10 @@ def determine_apache_port(public_port, singlenode_mode=False):
     return public_port - (i * 10)
 
 
+determine_apache_port_single = functools.partial(
+    determine_apache_port, singlenode_mode=True)
+
+
 def get_hacluster_config(exclude_keys=None):
     '''
     Obtains all relevant configuration from charm configuration required
@@ -404,3 +409,43 @@ def distributed_wait(modulo=None, wait=None, operation_name='operation'):
     log(msg, DEBUG)
     status_set('maintenance', msg)
     time.sleep(calculated_wait)
+
+
+def get_managed_services_and_ports(services, external_ports,
+                                   external_services=None,
+                                   port_conv_f=determine_apache_port_single):
+    """Get the services and ports managed by this charm.
+
+    Return only the services and corresponding ports that are managed by this
+    charm. This excludes haproxy when there is a relation with hacluster. This
+    is because this charm passes responsability for stopping and starting
+    haproxy to hacluster.
+
+    Similarly, if a relation with hacluster exists then the ports returned by
+    this method correspond to those managed by the apache server rather than
+    haproxy.
+
+    :param services: List of services.
+    :type services: List[str]
+    :param external_ports: List of ports managed by external services.
+    :type external_ports: List[int]
+    :param external_services: List of services to be removed if ha relation is
+                              present.
+    :type external_services: List[str]
+    :param port_conv_f: Function to apply to ports to calculate the ports
+                        managed by services controlled by this charm.
+    :type port_convert_func: f()
+    :returns: A tuple containing a list of services first followed by a list of
+              ports.
+    :rtype: Tuple[List[str], List[int]]
+    """
+    if external_services is None:
+        external_services = ['haproxy']
+    if relation_ids('ha'):
+        for svc in external_services:
+            try:
+                services.remove(svc)
+            except ValueError:
+                pass
+        external_ports = [port_conv_f(p) for p in external_ports]
+    return services, external_ports

charmhelpers/contrib/hardening/audits/apt.py

@@ -52,7 +52,7 @@ class RestrictedPackages(BaseAudit):
     def __init__(self, pkgs, **kwargs):
         super(RestrictedPackages, self).__init__(**kwargs)
         if isinstance(pkgs, string_types) or not hasattr(pkgs, '__iter__'):
-            self.pkgs = [pkgs]
+            self.pkgs = pkgs.split()
         else:
             self.pkgs = pkgs
 
@@ -100,4 +100,5 @@ class RestrictedPackages(BaseAudit):
             apt_purge(pkg.name)
 
     def is_virtual_package(self, pkg):
-        return pkg.has_provides and not pkg.has_versions
+        return (pkg.get('has_provides', False) and
+                not pkg.get('has_versions', False))

charmhelpers/contrib/openstack/context.py

@@ -25,6 +25,10 @@ from subprocess import check_call, CalledProcessError
 
 import six
 
+from charmhelpers.contrib.openstack.audits.openstack_security_guide import (
+    _config_ini as config_ini
+)
+
 from charmhelpers.fetch import (
     apt_install,
     filter_installed_packages,
@@ -2244,3 +2248,151 @@ class HostInfoContext(OSContextGenerator):
                 self.use_fqdn_hint_cb() if self.use_fqdn_hint_cb else False)
         }
         return ctxt
+
+
+def validate_ovs_use_veth(*args, **kwargs):
+    """Validate OVS use veth setting for dhcp agents
+
+    The ovs_use_veth setting is considered immutable as it will break existing
+    deployments. Historically, we set ovs_use_veth=True in dhcp_agent.ini. It
+    turns out this is no longer necessary. Ideally, all new deployments would
+    have this set to False.
+
+    This function validates that the config value does not conflict with
+    previously deployed settings in dhcp_agent.ini.
+
+    See LP Bug#1831935 for details.
+
+    :returns: Status state and message
+    :rtype: Union[(None, None), (string, string)]
+    """
+    existing_ovs_use_veth = (
+        DHCPAgentContext.get_existing_ovs_use_veth())
+    config_ovs_use_veth = DHCPAgentContext.parse_ovs_use_veth()
+
+    # Check settings are set and not None
+    if existing_ovs_use_veth is not None and config_ovs_use_veth is not None:
+        # Check for mismatch between existing config ini and juju config
+        if existing_ovs_use_veth != config_ovs_use_veth:
+            # Stop the line to avoid breakage
+            msg = (
+                "The existing setting for dhcp_agent.ini ovs_use_veth, {}, "
+                "does not match the juju config setting, {}. This may lead to "
+                "VMs being unable to receive a DHCP IP. Either change the "
+                "juju config setting or dhcp agents may need to be recreated."
+                .format(existing_ovs_use_veth, config_ovs_use_veth))
+            log(msg, ERROR)
+            return (
+                "blocked",
+                "Mismatched existing and configured ovs-use-veth. See log.")
+
+    # Everything is OK
+    return None, None
+
+
+class DHCPAgentContext(OSContextGenerator):
+
+    def __call__(self):
+        """Return the DHCPAGentContext.
+
+        Return all DHCP Agent INI related configuration.
+        ovs unit is attached to (as a subordinate) and the 'dns_domain' from
+        the neutron-plugin-api relations (if one is set).
+
+        :returns: Dictionary context
+        :rtype: Dict
+        """
+
+        ctxt = {}
+        dnsmasq_flags = config('dnsmasq-flags')
+        if dnsmasq_flags:
+            ctxt['dnsmasq_flags'] = config_flags_parser(dnsmasq_flags)
+        ctxt['dns_servers'] = config('dns-servers')
+
+        neutron_api_settings = NeutronAPIContext()()
+
+        ctxt['debug'] = config('debug')
+        ctxt['instance_mtu'] = config('instance-mtu')
+        ctxt['ovs_use_veth'] = self.get_ovs_use_veth()
+
+        ctxt['enable_metadata_network'] = config('enable-metadata-network')
+        ctxt['enable_isolated_metadata'] = config('enable-isolated-metadata')
+
+        if neutron_api_settings.get('dns_domain'):
+            ctxt['dns_domain'] = neutron_api_settings.get('dns_domain')
+
+        # Override user supplied config for these plugins as these settings are
+        # mandatory
+        if config('plugin') in ['nvp', 'nsx', 'n1kv']:
+            ctxt['enable_metadata_network'] = True
+            ctxt['enable_isolated_metadata'] = True
+
+        return ctxt
+
+    @staticmethod
+    def get_existing_ovs_use_veth():
+        """Return existing ovs_use_veth setting from dhcp_agent.ini.
+
+        :returns: Boolean value of existing ovs_use_veth setting or None
+        :rtype: Optional[Bool]
+        """
+        DHCP_AGENT_INI = "/etc/neutron/dhcp_agent.ini"
+        existing_ovs_use_veth = None
+        # If there is a dhcp_agent.ini file read the current setting
+        if os.path.isfile(DHCP_AGENT_INI):
+            # config_ini does the right thing and returns None if the setting is
+            # commented.
+            existing_ovs_use_veth = (
+                config_ini(DHCP_AGENT_INI)["DEFAULT"].get("ovs_use_veth"))
+        # Convert to Bool if necessary
+        if isinstance(existing_ovs_use_veth, six.string_types):
+            return bool_from_string(existing_ovs_use_veth)
+        return existing_ovs_use_veth
+
+    @staticmethod
+    def parse_ovs_use_veth():
+        """Parse the ovs-use-veth config setting.
+
+        Parse the string config setting for ovs-use-veth and return a boolean
+        or None.
+
+        bool_from_string will raise a ValueError if the string is not falsy or
+        truthy.
+
+        :raises: ValueError for invalid input
+        :returns: Boolean value of ovs-use-veth or None
+        :rtype: Optional[Bool]
+        """
+        _config = config("ovs-use-veth")
+        # An unset parameter returns None. Just in case we will also check for
+        # an empty string: "". Ironically, (the problem we are trying to avoid)
+        # "False" returns True and "" returns False.
+        if _config is None or not _config:
+            # Return None
+            return
+        # bool_from_string handles many variations of true and false strings
+        # as well as upper and lowercases including:
+        # ['y', 'yes', 'true', 't', 'on', 'n', 'no', 'false', 'f', 'off']
+        return bool_from_string(_config)
+
+    def get_ovs_use_veth(self):
+        """Return correct ovs_use_veth setting for use in dhcp_agent.ini.
+
+        Get the right value from config or existing dhcp_agent.ini file.
+        Existing has precedence. Attempt to default to "False" without
+        disrupting existing deployments. Handle existing deployments and
+        upgrades safely. See LP Bug#1831935
+
+        :returns: Value to use for ovs_use_veth setting
+        :rtype: Bool
+        """
+        _existing = self.get_existing_ovs_use_veth()
+        if _existing is not None:
+            return _existing
+
+        _config = self.parse_ovs_use_veth()
+        if _config is None:
+            # New better default
+            return False
+        else:
+            return _config

charmhelpers/contrib/openstack/utils.py

@@ -44,6 +44,7 @@ from charmhelpers.core.hookenv import (
     INFO,
     ERROR,
     related_units,
+    relation_get,
     relation_ids,
     relation_set,
     status_set,
@@ -331,6 +332,10 @@ PACKAGE_CODENAMES = {
 
 DEFAULT_LOOPBACK_SIZE = '5G'
 
+DB_SERIES_UPGRADING_KEY = 'cluster-series-upgrading'
+
+DB_MAINTENANCE_KEYS = [DB_SERIES_UPGRADING_KEY]
+
 
 class CompareOpenStackReleases(BasicStringComparator):
     """Provide comparisons of OpenStack releases.
@@ -1912,3 +1917,33 @@ def set_db_initialised():
     """
     juju_log('Setting db-initialised to True', 'DEBUG')
     leader_set({'db-initialised': True})
+
+
+def is_db_maintenance_mode(relid=None):
+    """Check relation data from notifications of db in maintenance mode.
+
+    :returns: Whether db has notified it is in maintenance mode.
+    :rtype: bool
+    """
+    juju_log('Checking for maintenance notifications', 'DEBUG')
+    if relid:
+        r_ids = [relid]
+    else:
+        r_ids = relation_ids('shared-db')
+    rids_units = [(r, u) for r in r_ids for u in related_units(r)]
+    notifications = []
+    for r_id, unit in rids_units:
+        settings = relation_get(unit=unit, rid=r_id)
+        for key, value in settings.items():
+            if value and key in DB_MAINTENANCE_KEYS:
+                juju_log(
+                    'Unit: {}, Key: {}, Value: {}'.format(unit, key, value),
+                    'DEBUG')
+                try:
+                    notifications.append(bool_from_string(value))
+                except ValueError:
+                    juju_log(
+                        'Could not discern bool from {}'.format(value),
+                        'WARN')
+                    pass
+    return True in notifications

charmhelpers/fetch/ubuntu_apt_pkg.py

@@ -38,6 +38,7 @@ so with this we get rid of the dependency.
 import locale
 import os
 import subprocess
+import sys
 
 
 class _container(dict):
@@ -59,6 +60,13 @@ class Cache(object):
     def __init__(self, progress=None):
         pass
 
+    def __contains__(self, package):
+        try:
+            pkg = self.__getitem__(package)
+            return pkg is not None
+        except KeyError:
+            return False
+
     def __getitem__(self, package):
         """Get information about a package from apt and dpkg databases.
 
@@ -178,6 +186,28 @@ class Cache(object):
         return pkgs
 
 
+class Config(_container):
+    def __init__(self):
+        super(Config, self).__init__(self._populate())
+
+    def _populate(self):
+        cfgs = {}
+        cmd = ['apt-config', 'dump']
+        output = subprocess.check_output(cmd,
+                                         stderr=subprocess.STDOUT,
+                                         universal_newlines=True)
+        for line in output.splitlines():
+            if not line.startswith("CommandLine"):
+                k, v = line.split(" ", 1)
+                cfgs[k] = v.strip(";").strip("\"")
+
+        return cfgs
+
+
+# Backwards compatibility with old apt_pkg module
+sys.modules[__name__].config = Config()
+
+
 def init():
     """Compability shim that does nothing."""
     pass

charmhelpers/osplatform.py

@@ -20,6 +20,9 @@ def get_platform():
         # Stock Python does not detect Ubuntu and instead returns debian.
         # Or at least it does in some build environments like Travis CI
         return "ubuntu"
+    elif "elementary" in current_platform:
+        # ElementaryOS fails to run tests locally without this.
+        return "ubuntu"
     else:
         raise RuntimeError("This module is not supported on {}."
                            .format(current_platform))