diff --git a/hooks/charmhelpers/contrib/hahelpers/cluster.py b/hooks/charmhelpers/contrib/hahelpers/cluster.py index 4a737e24..ba34fba0 100644 --- a/hooks/charmhelpers/contrib/hahelpers/cluster.py +++ b/hooks/charmhelpers/contrib/hahelpers/cluster.py @@ -25,6 +25,7 @@ Helpers for clustering and determining "cluster leadership" and other clustering-related helpers. """ +import functools import subprocess import os import time @@ -281,6 +282,10 @@ def determine_apache_port(public_port, singlenode_mode=False): return public_port - (i * 10) +determine_apache_port_single = functools.partial( + determine_apache_port, singlenode_mode=True) + + def get_hacluster_config(exclude_keys=None): ''' Obtains all relevant configuration from charm configuration required @@ -404,3 +409,43 @@ def distributed_wait(modulo=None, wait=None, operation_name='operation'): log(msg, DEBUG) status_set('maintenance', msg) time.sleep(calculated_wait) + + +def get_managed_services_and_ports(services, external_ports, + external_services=None, + port_conv_f=determine_apache_port_single): + """Get the services and ports managed by this charm. + + Return only the services and corresponding ports that are managed by this + charm. This excludes haproxy when there is a relation with hacluster. This + is because this charm passes responsability for stopping and starting + haproxy to hacluster. + + Similarly, if a relation with hacluster exists then the ports returned by + this method correspond to those managed by the apache server rather than + haproxy. + + :param services: List of services. + :type services: List[str] + :param external_ports: List of ports managed by external services. + :type external_ports: List[int] + :param external_services: List of services to be removed if ha relation is + present. + :type external_services: List[str] + :param port_conv_f: Function to apply to ports to calculate the ports + managed by services controlled by this charm. + :type port_convert_func: f() + :returns: A tuple containing a list of services first followed by a list of + ports. + :rtype: Tuple[List[str], List[int]] + """ + if external_services is None: + external_services = ['haproxy'] + if relation_ids('ha'): + for svc in external_services: + try: + services.remove(svc) + except ValueError: + pass + external_ports = [port_conv_f(p) for p in external_ports] + return services, external_ports diff --git a/hooks/charmhelpers/contrib/hardening/audits/apt.py b/hooks/charmhelpers/contrib/hardening/audits/apt.py index 67521e17..cad7bf73 100644 --- a/hooks/charmhelpers/contrib/hardening/audits/apt.py +++ b/hooks/charmhelpers/contrib/hardening/audits/apt.py @@ -52,7 +52,7 @@ class RestrictedPackages(BaseAudit): def __init__(self, pkgs, **kwargs): super(RestrictedPackages, self).__init__(**kwargs) if isinstance(pkgs, string_types) or not hasattr(pkgs, '__iter__'): - self.pkgs = [pkgs] + self.pkgs = pkgs.split() else: self.pkgs = pkgs @@ -100,4 +100,5 @@ class RestrictedPackages(BaseAudit): apt_purge(pkg.name) def is_virtual_package(self, pkg): - return pkg.has_provides and not pkg.has_versions + return (pkg.get('has_provides', False) and + not pkg.get('has_versions', False)) diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index e99aba47..bc90804b 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -25,6 +25,10 @@ from subprocess import check_call, CalledProcessError import six +from charmhelpers.contrib.openstack.audits.openstack_security_guide import ( + _config_ini as config_ini +) + from charmhelpers.fetch import ( apt_install, filter_installed_packages, @@ -2244,3 +2248,151 @@ class HostInfoContext(OSContextGenerator): self.use_fqdn_hint_cb() if self.use_fqdn_hint_cb else False) } return ctxt + + +def validate_ovs_use_veth(*args, **kwargs): + """Validate OVS use veth setting for dhcp agents + + The ovs_use_veth setting is considered immutable as it will break existing + deployments. Historically, we set ovs_use_veth=True in dhcp_agent.ini. It + turns out this is no longer necessary. Ideally, all new deployments would + have this set to False. + + This function validates that the config value does not conflict with + previously deployed settings in dhcp_agent.ini. + + See LP Bug#1831935 for details. + + :returns: Status state and message + :rtype: Union[(None, None), (string, string)] + """ + existing_ovs_use_veth = ( + DHCPAgentContext.get_existing_ovs_use_veth()) + config_ovs_use_veth = DHCPAgentContext.parse_ovs_use_veth() + + # Check settings are set and not None + if existing_ovs_use_veth is not None and config_ovs_use_veth is not None: + # Check for mismatch between existing config ini and juju config + if existing_ovs_use_veth != config_ovs_use_veth: + # Stop the line to avoid breakage + msg = ( + "The existing setting for dhcp_agent.ini ovs_use_veth, {}, " + "does not match the juju config setting, {}. This may lead to " + "VMs being unable to receive a DHCP IP. Either change the " + "juju config setting or dhcp agents may need to be recreated." + .format(existing_ovs_use_veth, config_ovs_use_veth)) + log(msg, ERROR) + return ( + "blocked", + "Mismatched existing and configured ovs-use-veth. See log.") + + # Everything is OK + return None, None + + +class DHCPAgentContext(OSContextGenerator): + + def __call__(self): + """Return the DHCPAGentContext. + + Return all DHCP Agent INI related configuration. + ovs unit is attached to (as a subordinate) and the 'dns_domain' from + the neutron-plugin-api relations (if one is set). + + :returns: Dictionary context + :rtype: Dict + """ + + ctxt = {} + dnsmasq_flags = config('dnsmasq-flags') + if dnsmasq_flags: + ctxt['dnsmasq_flags'] = config_flags_parser(dnsmasq_flags) + ctxt['dns_servers'] = config('dns-servers') + + neutron_api_settings = NeutronAPIContext()() + + ctxt['debug'] = config('debug') + ctxt['instance_mtu'] = config('instance-mtu') + ctxt['ovs_use_veth'] = self.get_ovs_use_veth() + + ctxt['enable_metadata_network'] = config('enable-metadata-network') + ctxt['enable_isolated_metadata'] = config('enable-isolated-metadata') + + if neutron_api_settings.get('dns_domain'): + ctxt['dns_domain'] = neutron_api_settings.get('dns_domain') + + # Override user supplied config for these plugins as these settings are + # mandatory + if config('plugin') in ['nvp', 'nsx', 'n1kv']: + ctxt['enable_metadata_network'] = True + ctxt['enable_isolated_metadata'] = True + + return ctxt + + @staticmethod + def get_existing_ovs_use_veth(): + """Return existing ovs_use_veth setting from dhcp_agent.ini. + + :returns: Boolean value of existing ovs_use_veth setting or None + :rtype: Optional[Bool] + """ + DHCP_AGENT_INI = "/etc/neutron/dhcp_agent.ini" + existing_ovs_use_veth = None + # If there is a dhcp_agent.ini file read the current setting + if os.path.isfile(DHCP_AGENT_INI): + # config_ini does the right thing and returns None if the setting is + # commented. + existing_ovs_use_veth = ( + config_ini(DHCP_AGENT_INI)["DEFAULT"].get("ovs_use_veth")) + # Convert to Bool if necessary + if isinstance(existing_ovs_use_veth, six.string_types): + return bool_from_string(existing_ovs_use_veth) + return existing_ovs_use_veth + + @staticmethod + def parse_ovs_use_veth(): + """Parse the ovs-use-veth config setting. + + Parse the string config setting for ovs-use-veth and return a boolean + or None. + + bool_from_string will raise a ValueError if the string is not falsy or + truthy. + + :raises: ValueError for invalid input + :returns: Boolean value of ovs-use-veth or None + :rtype: Optional[Bool] + """ + _config = config("ovs-use-veth") + # An unset parameter returns None. Just in case we will also check for + # an empty string: "". Ironically, (the problem we are trying to avoid) + # "False" returns True and "" returns False. + if _config is None or not _config: + # Return None + return + # bool_from_string handles many variations of true and false strings + # as well as upper and lowercases including: + # ['y', 'yes', 'true', 't', 'on', 'n', 'no', 'false', 'f', 'off'] + return bool_from_string(_config) + + def get_ovs_use_veth(self): + """Return correct ovs_use_veth setting for use in dhcp_agent.ini. + + Get the right value from config or existing dhcp_agent.ini file. + Existing has precedence. Attempt to default to "False" without + disrupting existing deployments. Handle existing deployments and + upgrades safely. See LP Bug#1831935 + + :returns: Value to use for ovs_use_veth setting + :rtype: Bool + """ + _existing = self.get_existing_ovs_use_veth() + if _existing is not None: + return _existing + + _config = self.parse_ovs_use_veth() + if _config is None: + # New better default + return False + else: + return _config diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 566404a0..161199c4 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -44,6 +44,7 @@ from charmhelpers.core.hookenv import ( INFO, ERROR, related_units, + relation_get, relation_ids, relation_set, status_set, @@ -331,6 +332,10 @@ PACKAGE_CODENAMES = { DEFAULT_LOOPBACK_SIZE = '5G' +DB_SERIES_UPGRADING_KEY = 'cluster-series-upgrading' + +DB_MAINTENANCE_KEYS = [DB_SERIES_UPGRADING_KEY] + class CompareOpenStackReleases(BasicStringComparator): """Provide comparisons of OpenStack releases. @@ -1912,3 +1917,33 @@ def set_db_initialised(): """ juju_log('Setting db-initialised to True', 'DEBUG') leader_set({'db-initialised': True}) + + +def is_db_maintenance_mode(relid=None): + """Check relation data from notifications of db in maintenance mode. + + :returns: Whether db has notified it is in maintenance mode. + :rtype: bool + """ + juju_log('Checking for maintenance notifications', 'DEBUG') + if relid: + r_ids = [relid] + else: + r_ids = relation_ids('shared-db') + rids_units = [(r, u) for r in r_ids for u in related_units(r)] + notifications = [] + for r_id, unit in rids_units: + settings = relation_get(unit=unit, rid=r_id) + for key, value in settings.items(): + if value and key in DB_MAINTENANCE_KEYS: + juju_log( + 'Unit: {}, Key: {}, Value: {}'.format(unit, key, value), + 'DEBUG') + try: + notifications.append(bool_from_string(value)) + except ValueError: + juju_log( + 'Could not discern bool from {}'.format(value), + 'WARN') + pass + return True in notifications diff --git a/hooks/charmhelpers/fetch/ubuntu_apt_pkg.py b/hooks/charmhelpers/fetch/ubuntu_apt_pkg.py index 104f91f1..929a75d7 100644 --- a/hooks/charmhelpers/fetch/ubuntu_apt_pkg.py +++ b/hooks/charmhelpers/fetch/ubuntu_apt_pkg.py @@ -38,6 +38,7 @@ so with this we get rid of the dependency. import locale import os import subprocess +import sys class _container(dict): @@ -59,6 +60,13 @@ class Cache(object): def __init__(self, progress=None): pass + def __contains__(self, package): + try: + pkg = self.__getitem__(package) + return pkg is not None + except KeyError: + return False + def __getitem__(self, package): """Get information about a package from apt and dpkg databases. @@ -178,6 +186,28 @@ class Cache(object): return pkgs +class Config(_container): + def __init__(self): + super(Config, self).__init__(self._populate()) + + def _populate(self): + cfgs = {} + cmd = ['apt-config', 'dump'] + output = subprocess.check_output(cmd, + stderr=subprocess.STDOUT, + universal_newlines=True) + for line in output.splitlines(): + if not line.startswith("CommandLine"): + k, v = line.split(" ", 1) + cfgs[k] = v.strip(";").strip("\"") + + return cfgs + + +# Backwards compatibility with old apt_pkg module +sys.modules[__name__].config = Config() + + def init(): """Compability shim that does nothing.""" pass diff --git a/hooks/charmhelpers/osplatform.py b/hooks/charmhelpers/osplatform.py index d9a4d5c0..c7fd1363 100644 --- a/hooks/charmhelpers/osplatform.py +++ b/hooks/charmhelpers/osplatform.py @@ -20,6 +20,9 @@ def get_platform(): # Stock Python does not detect Ubuntu and instead returns debian. # Or at least it does in some build environments like Travis CI return "ubuntu" + elif "elementary" in current_platform: + # ElementaryOS fails to run tests locally without this. + return "ubuntu" else: raise RuntimeError("This module is not supported on {}." .format(current_platform)) diff --git a/lib/ceph/utils.py b/lib/ceph/utils.py index 1ec62d54..7c970784 100644 --- a/lib/ceph/utils.py +++ b/lib/ceph/utils.py @@ -80,12 +80,7 @@ LEADER = 'leader' PEON = 'peon' QUORUM = [LEADER, PEON] -if CompareHostReleases(lsb_release()['DISTRIB_CODENAME']) >= 'eoan': - btrfs_package = 'btrfs-progs' -else: - btrfs_package = 'btrfs-tools' - -PACKAGES = ['ceph', 'gdisk', btrfs_package, +PACKAGES = ['ceph', 'gdisk', 'radosgw', 'xfsprogs', 'lvm2', 'parted', 'smartmontools'] @@ -2917,7 +2912,13 @@ def determine_packages(): :returns: list of ceph packages """ - return PACKAGES + packages = PACKAGES.copy() + if CompareHostReleases(lsb_release()['DISTRIB_CODENAME']) >= 'eoan': + btrfs_package = 'btrfs-progs' + else: + btrfs_package = 'btrfs-tools' + packages.append(btrfs_package) + return packages def bootstrap_manager():