openstack/charm-ceph-mon
Code Issues Proposed changes

Merge "Updates for testing period for 20.01 release"

Browse Source
This commit is contained in:
Zuul 2021-01-15 20:26:49 +00:00 committed by Gerrit Code Review
commit 189577a27f
10 changed files with 238 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
hooks/charmhelpers/contrib/charmsupport/nrpe.py
View File

@ -139,10 +139,11 @@ define service {{
"""{description}
check_command check_nrpe!{command}
servicegroups {nagios_servicegroup}
{service_config_overrides}
}}
""")
def __init__(self, shortname, description, check_cmd):
def __init__(self, shortname, description, check_cmd, max_check_attempts=None):
super(Check, self).__init__()
# XXX: could be better to calculate this from the service name
if not re.match(self.shortname_re, shortname):
@ -155,6 +156,7 @@ define service {{
# The default is: illegal_object_name_chars=`~!$%^&*"|'<>?,()=
self.description = description
self.check_cmd = self._locate_cmd(check_cmd)
self.max_check_attempts = max_check_attempts
def _get_check_filename(self):
return os.path.join(NRPE.nrpe_confdir, '{}.cfg'.format(self.command))
@ -216,12 +218,19 @@ define service {{
nagios_servicegroups):
self._remove_service_files()
if self.max_check_attempts:
service_config_overrides = ' max_check_attempts {}'.format(
self.max_check_attempts
) # Note indentation is here rather than in the template to avoid trailing spaces
else:
service_config_overrides = '' # empty string to avoid printing 'None'
templ_vars = {
'nagios_hostname': hostname,
'nagios_servicegroup': nagios_servicegroups,
'description': self.description,
'shortname': self.shortname,
'command': self.command,
'service_config_overrides': service_config_overrides,
}
nrpe_service_text = Check.service_template.format(**templ_vars)
nrpe_service_file = self._get_service_filename(hostname)
@ -327,6 +336,11 @@ class NRPE(object):
nrpe_monitors[nrpecheck.shortname] = {
"command": nrpecheck.command,
}
# If we were passed max_check_attempts, add that to the relation data
try:
nrpe_monitors[nrpecheck.shortname]['max_check_attempts'] = nrpecheck.max_check_attempts
except AttributeError:
pass
# update-status hooks are configured to firing every 5 minutes by
# default. When nagios-nrpe-server is restarted, the nagios server

6
hooks/charmhelpers/contrib/hahelpers/apache.py
View File

@ -34,6 +34,10 @@ from charmhelpers.core.hookenv import (
INFO,
)
# This file contains the CA cert from the charms ssl_ca configuration
# option, in future the file name should be updated reflect that.
CONFIG_CA_CERT_FILE = 'keystone_juju_ca_cert'
def get_cert(cn=None):
# TODO: deal with multiple https endpoints via charm config
@ -83,4 +87,4 @@ def retrieve_ca_cert(cert_file):
def install_ca_cert(ca_cert):
host.install_ca_cert(ca_cert, 'keystone_juju_ca_cert')
host.install_ca_cert(ca_cert, CONFIG_CA_CERT_FILE)

5
hooks/charmhelpers/contrib/hardening/audits/apache.py
View File

@ -98,3 +98,8 @@ class DisabledModuleAudit(BaseAudit):
def _restart_apache():
"""Restarts the apache process"""
subprocess.check_output(['service', 'apache2', 'restart'])
@staticmethod
def is_ssl_enabled():
"""Check if SSL module is enabled or not"""
return 'ssl' in DisabledModuleAudit._get_loaded_modules()

113
hooks/charmhelpers/contrib/openstack/cert_utils.py
View File

@ -16,6 +16,7 @@
import os
import json
from base64 import b64decode
from charmhelpers.contrib.network.ip import (
get_hostname,
@ -28,10 +29,12 @@ from charmhelpers.core.hookenv import (
related_units,
relation_get,
relation_ids,
remote_service_name,
unit_get,
NoNetworkBinding,
log,
WARNING,
INFO,
)
from charmhelpers.contrib.openstack.ip import (
resolve_address,
@ -44,12 +47,14 @@ from charmhelpers.contrib.network.ip import (
)
from charmhelpers.core.host import (
CA_CERT_DIR,
install_ca_cert,
mkdir,
write_file,
)
from charmhelpers.contrib.hahelpers.apache import (
install_ca_cert
CONFIG_CA_CERT_FILE,
)
@ -129,38 +134,46 @@ def get_certificate_request(json_encode=True, bindings=None):
"""
if bindings:
# Add default API bindings to bindings list
bindings = set(bindings + get_default_api_bindings())
bindings = list(bindings + get_default_api_bindings())
else:
# Use default API bindings
bindings = get_default_api_bindings()
req = CertRequest(json_encode=json_encode)
req.add_hostname_cn()
# Add os-hostname entries
_sans = get_certificate_sans()
_sans = get_certificate_sans(bindings=bindings)
# Handle specific hostnames per binding
for binding in bindings:
hostname_override = config(ADDRESS_MAP[binding]['override'])
try:
net_addr = resolve_address(endpoint_type=binding)
ip = network_get_primary_address(
ADDRESS_MAP[binding]['binding'])
hostname_override = config(ADDRESS_MAP[binding]['override'])
except KeyError:
hostname_override = None
try:
try:
net_addr = resolve_address(endpoint_type=binding)
except KeyError:
net_addr = None
ip = network_get_primary_address(binding)
addresses = [net_addr, ip]
vip = get_vip_in_network(resolve_network_cidr(ip))
if vip:
addresses.append(vip)
# Clear any Nones or duplicates
addresses = list(set([i for i in addresses if i]))
# Add hostname certificate request
if hostname_override:
req.add_entry(
binding,
hostname_override,
addresses)
# Remove hostname specific addresses from _sans
for addr in addresses:
try:
_sans.remove(addr)
except (ValueError, KeyError):
pass
# Remove hostname specific addresses from _sans
for addr in addresses:
try:
_sans.remove(addr)
except (ValueError, KeyError):
pass
except NoNetworkBinding:
log("Skipping request for certificate for ip in {} space, no "
@ -174,11 +187,17 @@ def get_certificate_request(json_encode=True, bindings=None):
def get_certificate_sans(bindings=None):
"""Get all possible IP addresses for certificate SANs.
:param bindings: List of bindings to check in addition to default api
bindings.
:type bindings: list of strings
:returns: List of binding string names
:rtype: List[str]
"""
_sans = [unit_get('private-address')]
if bindings:
# Add default API bindings to bindings list
bindings = set(bindings + get_default_api_bindings())
bindings = list(bindings + get_default_api_bindings())
else:
# Use default API bindings
bindings = get_default_api_bindings()
@ -192,25 +211,39 @@ def get_certificate_sans(bindings=None):
net_config = None
# Using resolve_address is likely redundant. Keeping it here in
# case there is an edge case it handles.
net_addr = resolve_address(endpoint_type=binding)
try:
net_addr = resolve_address(endpoint_type=binding)
except KeyError:
net_addr = None
ip = get_relation_ip(binding, cidr_network=net_config)
_sans = _sans + [net_addr, ip]
vip = get_vip_in_network(resolve_network_cidr(ip))
if vip:
_sans.append(vip)
return set(_sans)
# Clear any Nones and duplicates
return list(set([i for i in _sans if i]))
def create_ip_cert_links(ssl_dir, custom_hostname_link=None):
def create_ip_cert_links(ssl_dir, custom_hostname_link=None, bindings=None):
"""Create symlinks for SAN records
:param ssl_dir: str Directory to create symlinks in
:param custom_hostname_link: str Additional link to be created
:param bindings: List of bindings to check in addition to default api
bindings.
:type bindings: list of strings
"""
if bindings:
# Add default API bindings to bindings list
bindings = list(bindings + get_default_api_bindings())
else:
# Use default API bindings
bindings = get_default_api_bindings()
# This includes the hostname cert and any specific bindng certs:
# admin, internal, public
req = get_certificate_request(json_encode=False)["cert_requests"]
req = get_certificate_request(json_encode=False, bindings=bindings)["cert_requests"]
# Specific certs
for cert_req in req.keys():
requested_cert = os.path.join(
@ -274,8 +307,35 @@ def install_certs(ssl_dir, certs, chain=None, user='root', group='root'):
content=bundle['key'], perms=0o640)
def _manage_ca_certs(ca, cert_relation_id):
"""Manage CA certs.
:param ca: CA Certificate from certificate relation.
:type ca: str
:param cert_relation_id: Relation id providing the certs
:type cert_relation_id: str
"""
config_ssl_ca = config('ssl_ca')
config_cert_file = '{}/{}.crt'.format(CA_CERT_DIR, CONFIG_CA_CERT_FILE)
if config_ssl_ca:
log("Installing CA certificate from charm ssl_ca config to {}".format(
config_cert_file), INFO)
install_ca_cert(
b64decode(config_ssl_ca).rstrip(),
name=CONFIG_CA_CERT_FILE)
elif os.path.exists(config_cert_file):
log("Removing CA certificate {}".format(config_cert_file), INFO)
os.remove(config_cert_file)
log("Installing CA certificate from certificate relation", INFO)
install_ca_cert(
ca.encode(),
name='{}_juju_ca_cert'.format(
remote_service_name(relid=cert_relation_id)))
def process_certificates(service_name, relation_id, unit,
custom_hostname_link=None, user='root', group='root'):
custom_hostname_link=None, user='root', group='root',
bindings=None):
"""Process the certificates supplied down the relation
:param service_name: str Name of service the certifcates are for.
@ -286,9 +346,19 @@ def process_certificates(service_name, relation_id, unit,
:type user: str
:param group: (Optional) Group of certificate files. Defaults to 'root'
:type group: str
:param bindings: List of bindings to check in addition to default api
bindings.
:type bindings: list of strings
:returns: True if certificates processed for local unit or False
:rtype: bool
"""
if bindings:
# Add default API bindings to bindings list
bindings = list(bindings + get_default_api_bindings())
else:
# Use default API bindings
bindings = get_default_api_bindings()
data = relation_get(rid=relation_id, unit=unit)
ssl_dir = os.path.join('/etc/apache2/ssl/', service_name)
mkdir(path=ssl_dir)
@ -298,11 +368,12 @@ def process_certificates(service_name, relation_id, unit,
ca = data.get('ca')
if certs:
certs = json.loads(certs)
install_ca_cert(ca.encode())
_manage_ca_certs(ca, relation_id)
install_certs(ssl_dir, certs, chain, user=user, group=group)
create_ip_cert_links(
ssl_dir,
custom_hostname_link=custom_hostname_link)
custom_hostname_link=custom_hostname_link,
bindings=bindings)
return True
return False

44
hooks/charmhelpers/contrib/openstack/context.py
View File

@ -1534,8 +1534,23 @@ class SubordinateConfigContext(OSContextGenerator):
ctxt[k][section] = config_list
else:
ctxt[k] = v
log("%d section(s) found" % (len(ctxt['sections'])), level=DEBUG)
return ctxt
if self.context_complete(ctxt):
log("%d section(s) found" % (len(ctxt['sections'])), level=DEBUG)
return ctxt
else:
return {}
def context_complete(self, ctxt):
"""Overridden here to ensure the context is actually complete.
:param ctxt: The current context members
:type ctxt: Dict[str, ANY]
:returns: True if the context is complete
:rtype: bool
"""
if not ctxt.get('sections'):
return False
return super(SubordinateConfigContext, self).context_complete(ctxt)
class LogLevelContext(OSContextGenerator):
@ -3050,6 +3065,9 @@ class SRIOVContext(OSContextGenerator):
blanket = 'blanket'
explicit = 'explicit'
PCIDeviceNumVFs = collections.namedtuple(
'PCIDeviceNumVFs', ['device', 'numvfs'])
def _determine_numvfs(self, device, sriov_numvfs):
"""Determine number of Virtual Functions (VFs) configured for device.
@ -3165,14 +3183,15 @@ class SRIOVContext(OSContextGenerator):
'configuration.')
self._map = {
device.interface_name: self._determine_numvfs(device, sriov_numvfs)
device.pci_address: self.PCIDeviceNumVFs(
device, self._determine_numvfs(device, sriov_numvfs))
for device in devices.pci_devices
if device.sriov and
self._determine_numvfs(device, sriov_numvfs) is not None
}
def __call__(self):
"""Provide SR-IOV context.
"""Provide backward compatible SR-IOV context.
:returns: Map interface name: min(configured, max) virtual functions.
Example:
@ -3183,6 +3202,23 @@ class SRIOVContext(OSContextGenerator):
}
:rtype: Dict[str,int]
"""
return {
pcidnvfs.device.interface_name: pcidnvfs.numvfs
for _, pcidnvfs in self._map.items()
}
@property
def get_map(self):
"""Provide map of configured SR-IOV capable PCI devices.
:returns: Map PCI-address: (PCIDevice, min(configured, max) VFs.
Example:
{
'0000:81:00.0': self.PCIDeviceNumVFs(<PCIDevice object>, 32),
'0000:81:00.1': self.PCIDeviceNumVFs(<PCIDevice object>, 32),
}
:rtype: Dict[str, self.PCIDeviceNumVFs]
"""
return self._map

30
hooks/charmhelpers/contrib/openstack/utils.py
View File

@ -90,13 +90,16 @@ from charmhelpers.core.host import (
service_start,
restart_on_change_helper,
)
from charmhelpers.fetch import (
apt_cache,
apt_install,
import_key as fetch_import_key,
add_source as fetch_add_source,
SourceConfigError,
GPGKeyError,
get_upstream_version,
filter_installed_packages,
filter_missing_packages,
ubuntu_apt_pkg as apt,
)
@ -480,9 +483,14 @@ def get_swift_codename(version):
return None
@deprecate("moved to charmhelpers.contrib.openstack.utils.get_installed_os_version()", "2021-01", log=juju_log)
def get_os_codename_package(package, fatal=True):
'''Derive OpenStack release codename from an installed package.'''
codename = get_installed_os_version()
if codename:
return codename
if snap_install_requested():
cmd = ['snap', 'list', package]
try:
@ -570,6 +578,28 @@ def get_os_version_package(pkg, fatal=True):
# error_out(e)
def get_installed_os_version():
apt_install(filter_installed_packages(['openstack-release']), fatal=False)
print("OpenStack Release: {}".format(openstack_release()))
return openstack_release().get('OPENSTACK_CODENAME')
@cached
def openstack_release():
"""Return /etc/os-release in a dict."""
d = {}
try:
with open('/etc/openstack-release', 'r') as lsb:
for l in lsb:
s = l.split('=')
if len(s) != 2:
continue
d[s[0].strip()] = s[1].strip()
except FileNotFoundError:
pass
return d
# Module local cache variable for the os_release.
_os_rel = None

3
hooks/charmhelpers/core/host.py
View File

@ -60,6 +60,7 @@ elif __platform__ == "centos":
) # flake8: noqa -- ignore F401 for this import
UPDATEDB_PATH = '/etc/updatedb.conf'
CA_CERT_DIR = '/usr/local/share/ca-certificates'
def service_start(service_name, **kwargs):
@ -1082,7 +1083,7 @@ def install_ca_cert(ca_cert, name=None):
ca_cert = ca_cert.encode('utf8')
if not name:
name = 'juju-{}'.format(charm_name())
cert_file = '/usr/local/share/ca-certificates/{}.crt'.format(name)
cert_file = '{}/{}.crt'.format(CA_CERT_DIR, name)
new_hash = hashlib.md5(ca_cert).hexdigest()
if file_hash(cert_file) == new_hash:
return

2
hooks/charmhelpers/fetch/ubuntu.py
View File

@ -646,7 +646,7 @@ def _add_apt_repository(spec):
# passed as environment variables (See lp:1433761). This is not the case
# LTS and non-LTS releases below bionic.
_run_with_retries(['add-apt-repository', '--yes', spec],
cmd_env=env_proxy_settings(['https']))
cmd_env=env_proxy_settings(['https', 'http']))
def _add_cloud_pocket(pocket):

47
hooks/charmhelpers/fetch/ubuntu_apt_pkg.py
View File

@ -129,7 +129,7 @@ class Cache(object):
else:
data = line.split(None, 4)
status = data.pop(0)
if status != 'ii':
if status not in ('ii', 'hi'):
continue
pkg = {}
pkg.update({k.lower(): v for k, v in zip(headings, data)})
@ -265,3 +265,48 @@ def version_compare(a, b):
raise RuntimeError('Unable to compare "{}" and "{}", according to '
'our logic they are neither greater, equal nor '
'less than each other.')
class PkgVersion():
"""Allow package versions to be compared.
For example::
>>> import charmhelpers.fetch as fetch
>>> (fetch.apt_pkg.PkgVersion('2:20.4.0') <
... fetch.apt_pkg.PkgVersion('2:20.5.0'))
True
>>> pkgs = [fetch.apt_pkg.PkgVersion('2:20.4.0'),
... fetch.apt_pkg.PkgVersion('2:21.4.0'),
... fetch.apt_pkg.PkgVersion('2:17.4.0')]
>>> pkgs.sort()
>>> pkgs
[2:17.4.0, 2:20.4.0, 2:21.4.0]
"""
def __init__(self, version):
self.version = version
def __lt__(self, other):
return version_compare(self.version, other.version) == -1
def __le__(self, other):
return self.__lt__(other) or self.__eq__(other)
def __gt__(self, other):
return version_compare(self.version, other.version) == 1
def __ge__(self, other):
return self.__gt__(other) or self.__eq__(other)
def __eq__(self, other):
return version_compare(self.version, other.version) == 0
def __ne__(self, other):
return not self.__eq__(other)
def __repr__(self):
return self.version
def __hash__(self):
return hash(repr(self))

2
test-requirements.txt
View File

@ -37,6 +37,8 @@ importlib-resources<3.0.0; python_version < '3.6'
# dropped support for python 3.5:
osprofiler<2.7.0;python_version<'3.6'
stevedore<1.31.0;python_version<'3.6'
debtcollector<1.22.0;python_version<'3.6'
oslo.utils<=3.41.0;python_version<'3.6'
coverage>=4.5.2
pyudev # for ceph-* charm unit tests (need to fix the ceph-* charm unit tests/mocking)