diff --git a/charmhelpers/contrib/hahelpers/apache.py b/charmhelpers/contrib/hahelpers/apache.py index 2c1e371..a54702b 100644 --- a/charmhelpers/contrib/hahelpers/apache.py +++ b/charmhelpers/contrib/hahelpers/apache.py @@ -34,6 +34,10 @@ from charmhelpers.core.hookenv import ( INFO, ) +# This file contains the CA cert from the charms ssl_ca configuration +# option, in future the file name should be updated reflect that. +CONFIG_CA_CERT_FILE = 'keystone_juju_ca_cert' + def get_cert(cn=None): # TODO: deal with multiple https endpoints via charm config @@ -83,4 +87,4 @@ def retrieve_ca_cert(cert_file): def install_ca_cert(ca_cert): - host.install_ca_cert(ca_cert, 'keystone_juju_ca_cert') + host.install_ca_cert(ca_cert, CONFIG_CA_CERT_FILE) diff --git a/charmhelpers/contrib/openstack/cert_utils.py b/charmhelpers/contrib/openstack/cert_utils.py index 1eb2154..fc36d0f 100644 --- a/charmhelpers/contrib/openstack/cert_utils.py +++ b/charmhelpers/contrib/openstack/cert_utils.py @@ -16,6 +16,7 @@ import os import json +from base64 import b64decode from charmhelpers.contrib.network.ip import ( get_hostname, @@ -28,10 +29,12 @@ from charmhelpers.core.hookenv import ( related_units, relation_get, relation_ids, + remote_service_name, unit_get, NoNetworkBinding, log, WARNING, + INFO, ) from charmhelpers.contrib.openstack.ip import ( resolve_address, @@ -44,12 +47,14 @@ from charmhelpers.contrib.network.ip import ( ) from charmhelpers.core.host import ( + CA_CERT_DIR, + install_ca_cert, mkdir, write_file, ) from charmhelpers.contrib.hahelpers.apache import ( - install_ca_cert + CONFIG_CA_CERT_FILE, ) @@ -129,38 +134,46 @@ def get_certificate_request(json_encode=True, bindings=None): """ if bindings: # Add default API bindings to bindings list - bindings = set(bindings + get_default_api_bindings()) + bindings = list(bindings + get_default_api_bindings()) else: # Use default API bindings bindings = get_default_api_bindings() req = CertRequest(json_encode=json_encode) req.add_hostname_cn() # Add os-hostname entries - _sans = get_certificate_sans() + _sans = get_certificate_sans(bindings=bindings) # Handle specific hostnames per binding for binding in bindings: - hostname_override = config(ADDRESS_MAP[binding]['override']) try: - net_addr = resolve_address(endpoint_type=binding) - ip = network_get_primary_address( - ADDRESS_MAP[binding]['binding']) + hostname_override = config(ADDRESS_MAP[binding]['override']) + except KeyError: + hostname_override = None + try: + try: + net_addr = resolve_address(endpoint_type=binding) + except KeyError: + net_addr = None + ip = network_get_primary_address(binding) addresses = [net_addr, ip] vip = get_vip_in_network(resolve_network_cidr(ip)) if vip: addresses.append(vip) + + # Clear any Nones or duplicates + addresses = list(set([i for i in addresses if i])) # Add hostname certificate request if hostname_override: req.add_entry( binding, hostname_override, addresses) - # Remove hostname specific addresses from _sans - for addr in addresses: - try: - _sans.remove(addr) - except (ValueError, KeyError): - pass + # Remove hostname specific addresses from _sans + for addr in addresses: + try: + _sans.remove(addr) + except (ValueError, KeyError): + pass except NoNetworkBinding: log("Skipping request for certificate for ip in {} space, no " @@ -174,11 +187,17 @@ def get_certificate_request(json_encode=True, bindings=None): def get_certificate_sans(bindings=None): """Get all possible IP addresses for certificate SANs. + + :param bindings: List of bindings to check in addition to default api + bindings. + :type bindings: list of strings + :returns: List of binding string names + :rtype: List[str] """ _sans = [unit_get('private-address')] if bindings: # Add default API bindings to bindings list - bindings = set(bindings + get_default_api_bindings()) + bindings = list(bindings + get_default_api_bindings()) else: # Use default API bindings bindings = get_default_api_bindings() @@ -192,25 +211,39 @@ def get_certificate_sans(bindings=None): net_config = None # Using resolve_address is likely redundant. Keeping it here in # case there is an edge case it handles. - net_addr = resolve_address(endpoint_type=binding) + try: + net_addr = resolve_address(endpoint_type=binding) + except KeyError: + net_addr = None ip = get_relation_ip(binding, cidr_network=net_config) _sans = _sans + [net_addr, ip] vip = get_vip_in_network(resolve_network_cidr(ip)) if vip: _sans.append(vip) - return set(_sans) + # Clear any Nones and duplicates + return list(set([i for i in _sans if i])) -def create_ip_cert_links(ssl_dir, custom_hostname_link=None): +def create_ip_cert_links(ssl_dir, custom_hostname_link=None, bindings=None): """Create symlinks for SAN records :param ssl_dir: str Directory to create symlinks in :param custom_hostname_link: str Additional link to be created + :param bindings: List of bindings to check in addition to default api + bindings. + :type bindings: list of strings """ + if bindings: + # Add default API bindings to bindings list + bindings = list(bindings + get_default_api_bindings()) + else: + # Use default API bindings + bindings = get_default_api_bindings() + # This includes the hostname cert and any specific bindng certs: # admin, internal, public - req = get_certificate_request(json_encode=False)["cert_requests"] + req = get_certificate_request(json_encode=False, bindings=bindings)["cert_requests"] # Specific certs for cert_req in req.keys(): requested_cert = os.path.join( @@ -274,8 +307,35 @@ def install_certs(ssl_dir, certs, chain=None, user='root', group='root'): content=bundle['key'], perms=0o640) +def _manage_ca_certs(ca, cert_relation_id): + """Manage CA certs. + + :param ca: CA Certificate from certificate relation. + :type ca: str + :param cert_relation_id: Relation id providing the certs + :type cert_relation_id: str + """ + config_ssl_ca = config('ssl_ca') + config_cert_file = '{}/{}.crt'.format(CA_CERT_DIR, CONFIG_CA_CERT_FILE) + if config_ssl_ca: + log("Installing CA certificate from charm ssl_ca config to {}".format( + config_cert_file), INFO) + install_ca_cert( + b64decode(config_ssl_ca).rstrip(), + name=CONFIG_CA_CERT_FILE) + elif os.path.exists(config_cert_file): + log("Removing CA certificate {}".format(config_cert_file), INFO) + os.remove(config_cert_file) + log("Installing CA certificate from certificate relation", INFO) + install_ca_cert( + ca.encode(), + name='{}_juju_ca_cert'.format( + remote_service_name(relid=cert_relation_id))) + + def process_certificates(service_name, relation_id, unit, - custom_hostname_link=None, user='root', group='root'): + custom_hostname_link=None, user='root', group='root', + bindings=None): """Process the certificates supplied down the relation :param service_name: str Name of service the certifcates are for. @@ -286,9 +346,19 @@ def process_certificates(service_name, relation_id, unit, :type user: str :param group: (Optional) Group of certificate files. Defaults to 'root' :type group: str + :param bindings: List of bindings to check in addition to default api + bindings. + :type bindings: list of strings :returns: True if certificates processed for local unit or False :rtype: bool """ + if bindings: + # Add default API bindings to bindings list + bindings = list(bindings + get_default_api_bindings()) + else: + # Use default API bindings + bindings = get_default_api_bindings() + data = relation_get(rid=relation_id, unit=unit) ssl_dir = os.path.join('/etc/apache2/ssl/', service_name) mkdir(path=ssl_dir) @@ -298,11 +368,12 @@ def process_certificates(service_name, relation_id, unit, ca = data.get('ca') if certs: certs = json.loads(certs) - install_ca_cert(ca.encode()) + _manage_ca_certs(ca, relation_id) install_certs(ssl_dir, certs, chain, user=user, group=group) create_ip_cert_links( ssl_dir, - custom_hostname_link=custom_hostname_link) + custom_hostname_link=custom_hostname_link, + bindings=bindings) return True return False diff --git a/charmhelpers/contrib/openstack/context.py b/charmhelpers/contrib/openstack/context.py index 54aed7f..6255dac 100644 --- a/charmhelpers/contrib/openstack/context.py +++ b/charmhelpers/contrib/openstack/context.py @@ -1534,8 +1534,23 @@ class SubordinateConfigContext(OSContextGenerator): ctxt[k][section] = config_list else: ctxt[k] = v - log("%d section(s) found" % (len(ctxt['sections'])), level=DEBUG) - return ctxt + if self.context_complete(ctxt): + log("%d section(s) found" % (len(ctxt['sections'])), level=DEBUG) + return ctxt + else: + return {} + + def context_complete(self, ctxt): + """Overridden here to ensure the context is actually complete. + + :param ctxt: The current context members + :type ctxt: Dict[str, ANY] + :returns: True if the context is complete + :rtype: bool + """ + if not ctxt.get('sections'): + return False + return super(SubordinateConfigContext, self).context_complete(ctxt) class LogLevelContext(OSContextGenerator): @@ -3050,6 +3065,9 @@ class SRIOVContext(OSContextGenerator): blanket = 'blanket' explicit = 'explicit' + PCIDeviceNumVFs = collections.namedtuple( + 'PCIDeviceNumVFs', ['device', 'numvfs']) + def _determine_numvfs(self, device, sriov_numvfs): """Determine number of Virtual Functions (VFs) configured for device. @@ -3165,14 +3183,15 @@ class SRIOVContext(OSContextGenerator): 'configuration.') self._map = { - device.interface_name: self._determine_numvfs(device, sriov_numvfs) + device.pci_address: self.PCIDeviceNumVFs( + device, self._determine_numvfs(device, sriov_numvfs)) for device in devices.pci_devices if device.sriov and self._determine_numvfs(device, sriov_numvfs) is not None } def __call__(self): - """Provide SR-IOV context. + """Provide backward compatible SR-IOV context. :returns: Map interface name: min(configured, max) virtual functions. Example: @@ -3183,6 +3202,23 @@ class SRIOVContext(OSContextGenerator): } :rtype: Dict[str,int] """ + return { + pcidnvfs.device.interface_name: pcidnvfs.numvfs + for _, pcidnvfs in self._map.items() + } + + @property + def get_map(self): + """Provide map of configured SR-IOV capable PCI devices. + + :returns: Map PCI-address: (PCIDevice, min(configured, max) VFs. + Example: + { + '0000:81:00.0': self.PCIDeviceNumVFs(, 32), + '0000:81:00.1': self.PCIDeviceNumVFs(, 32), + } + :rtype: Dict[str, self.PCIDeviceNumVFs] + """ return self._map diff --git a/charmhelpers/contrib/openstack/utils.py b/charmhelpers/contrib/openstack/utils.py index f4c7621..f27aa6c 100644 --- a/charmhelpers/contrib/openstack/utils.py +++ b/charmhelpers/contrib/openstack/utils.py @@ -90,13 +90,16 @@ from charmhelpers.core.host import ( service_start, restart_on_change_helper, ) + from charmhelpers.fetch import ( apt_cache, + apt_install, import_key as fetch_import_key, add_source as fetch_add_source, SourceConfigError, GPGKeyError, get_upstream_version, + filter_installed_packages, filter_missing_packages, ubuntu_apt_pkg as apt, ) @@ -480,9 +483,14 @@ def get_swift_codename(version): return None +@deprecate("moved to charmhelpers.contrib.openstack.utils.get_installed_os_version()", "2021-01", log=juju_log) def get_os_codename_package(package, fatal=True): '''Derive OpenStack release codename from an installed package.''' + codename = get_installed_os_version() + if codename: + return codename + if snap_install_requested(): cmd = ['snap', 'list', package] try: @@ -570,6 +578,28 @@ def get_os_version_package(pkg, fatal=True): # error_out(e) +def get_installed_os_version(): + apt_install(filter_installed_packages(['openstack-release']), fatal=False) + print("OpenStack Release: {}".format(openstack_release())) + return openstack_release().get('OPENSTACK_CODENAME') + + +@cached +def openstack_release(): + """Return /etc/os-release in a dict.""" + d = {} + try: + with open('/etc/openstack-release', 'r') as lsb: + for l in lsb: + s = l.split('=') + if len(s) != 2: + continue + d[s[0].strip()] = s[1].strip() + except FileNotFoundError: + pass + return d + + # Module local cache variable for the os_release. _os_rel = None diff --git a/charmhelpers/core/host.py b/charmhelpers/core/host.py index 697a5f4..f826f6f 100644 --- a/charmhelpers/core/host.py +++ b/charmhelpers/core/host.py @@ -60,6 +60,7 @@ elif __platform__ == "centos": ) # flake8: noqa -- ignore F401 for this import UPDATEDB_PATH = '/etc/updatedb.conf' +CA_CERT_DIR = '/usr/local/share/ca-certificates' def service_start(service_name, **kwargs): @@ -1082,7 +1083,7 @@ def install_ca_cert(ca_cert, name=None): ca_cert = ca_cert.encode('utf8') if not name: name = 'juju-{}'.format(charm_name()) - cert_file = '/usr/local/share/ca-certificates/{}.crt'.format(name) + cert_file = '{}/{}.crt'.format(CA_CERT_DIR, name) new_hash = hashlib.md5(ca_cert).hexdigest() if file_hash(cert_file) == new_hash: return diff --git a/charmhelpers/fetch/ubuntu.py b/charmhelpers/fetch/ubuntu.py index 3315284..b595301 100644 --- a/charmhelpers/fetch/ubuntu.py +++ b/charmhelpers/fetch/ubuntu.py @@ -646,7 +646,7 @@ def _add_apt_repository(spec): # passed as environment variables (See lp:1433761). This is not the case # LTS and non-LTS releases below bionic. _run_with_retries(['add-apt-repository', '--yes', spec], - cmd_env=env_proxy_settings(['https'])) + cmd_env=env_proxy_settings(['https', 'http'])) def _add_cloud_pocket(pocket): diff --git a/charmhelpers/fetch/ubuntu_apt_pkg.py b/charmhelpers/fetch/ubuntu_apt_pkg.py index 929a75d..a2fbe0e 100644 --- a/charmhelpers/fetch/ubuntu_apt_pkg.py +++ b/charmhelpers/fetch/ubuntu_apt_pkg.py @@ -129,7 +129,7 @@ class Cache(object): else: data = line.split(None, 4) status = data.pop(0) - if status != 'ii': + if status not in ('ii', 'hi'): continue pkg = {} pkg.update({k.lower(): v for k, v in zip(headings, data)}) @@ -265,3 +265,48 @@ def version_compare(a, b): raise RuntimeError('Unable to compare "{}" and "{}", according to ' 'our logic they are neither greater, equal nor ' 'less than each other.') + + +class PkgVersion(): + """Allow package versions to be compared. + + For example:: + + >>> import charmhelpers.fetch as fetch + >>> (fetch.apt_pkg.PkgVersion('2:20.4.0') < + ... fetch.apt_pkg.PkgVersion('2:20.5.0')) + True + >>> pkgs = [fetch.apt_pkg.PkgVersion('2:20.4.0'), + ... fetch.apt_pkg.PkgVersion('2:21.4.0'), + ... fetch.apt_pkg.PkgVersion('2:17.4.0')] + >>> pkgs.sort() + >>> pkgs + [2:17.4.0, 2:20.4.0, 2:21.4.0] + """ + + def __init__(self, version): + self.version = version + + def __lt__(self, other): + return version_compare(self.version, other.version) == -1 + + def __le__(self, other): + return self.__lt__(other) or self.__eq__(other) + + def __gt__(self, other): + return version_compare(self.version, other.version) == 1 + + def __ge__(self, other): + return self.__gt__(other) or self.__eq__(other) + + def __eq__(self, other): + return version_compare(self.version, other.version) == 0 + + def __ne__(self, other): + return not self.__eq__(other) + + def __repr__(self): + return self.version + + def __hash__(self): + return hash(repr(self)) diff --git a/test-requirements.txt b/test-requirements.txt index 1aa9635..9aea716 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -37,6 +37,8 @@ importlib-resources<3.0.0; python_version < '3.6' # dropped support for python 3.5: osprofiler<2.7.0;python_version<'3.6' stevedore<1.31.0;python_version<'3.6' +debtcollector<1.22.0;python_version<'3.6' +oslo.utils<=3.41.0;python_version<'3.6' coverage>=4.5.2 pyudev # for ceph-* charm unit tests (need to fix the ceph-* charm unit tests/mocking)