openstack
/
charm-cinder-ceph
Code Issues Proposed changes

Updates for testing period for 20.01 release 

* charm-helpers sync for classic charms
* rebuild for reactive charms
* ensure tox.ini is from release-tools
* ensure requirements.txt files are from release-tools
* On reactive charms:
  - ensure master branch for charms.openstack
  - ensure master branch for charm-helpers

Change-Id: I20570a90789e3d36339aa0bd4305e32af1226dfc
This commit is contained in:
Alex Kavanagh 2021-01-12 15:24:12 +00:00 committed by Aurelien Lourot
parent 2af10cfa7a
commit 9261c6c6be
8 changed files with 218 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
charmhelpers/contrib/hahelpers/apache.py
View File

@ -34,6 +34,10 @@ from charmhelpers.core.hookenv import (
INFO,
)
# This file contains the CA cert from the charms ssl_ca configuration
# option, in future the file name should be updated reflect that.
CONFIG_CA_CERT_FILE = 'keystone_juju_ca_cert'
def get_cert(cn=None):
# TODO: deal with multiple https endpoints via charm config
@ -83,4 +87,4 @@ def retrieve_ca_cert(cert_file):
def install_ca_cert(ca_cert):
host.install_ca_cert(ca_cert, 'keystone_juju_ca_cert')
host.install_ca_cert(ca_cert, CONFIG_CA_CERT_FILE)

113
charmhelpers/contrib/openstack/cert_utils.py
View File

@ -16,6 +16,7 @@
import os
import json
from base64 import b64decode
from charmhelpers.contrib.network.ip import (
get_hostname,
@ -28,10 +29,12 @@ from charmhelpers.core.hookenv import (
related_units,
relation_get,
relation_ids,
remote_service_name,
unit_get,
NoNetworkBinding,
log,
WARNING,
INFO,
)
from charmhelpers.contrib.openstack.ip import (
resolve_address,
@ -44,12 +47,14 @@ from charmhelpers.contrib.network.ip import (
)
from charmhelpers.core.host import (
CA_CERT_DIR,
install_ca_cert,
mkdir,
write_file,
)
from charmhelpers.contrib.hahelpers.apache import (
install_ca_cert
CONFIG_CA_CERT_FILE,
)
@ -129,38 +134,46 @@ def get_certificate_request(json_encode=True, bindings=None):
"""
if bindings:
# Add default API bindings to bindings list
bindings = set(bindings + get_default_api_bindings())
bindings = list(bindings + get_default_api_bindings())
else:
# Use default API bindings
bindings = get_default_api_bindings()
req = CertRequest(json_encode=json_encode)
req.add_hostname_cn()
# Add os-hostname entries
_sans = get_certificate_sans()
_sans = get_certificate_sans(bindings=bindings)
# Handle specific hostnames per binding
for binding in bindings:
hostname_override = config(ADDRESS_MAP[binding]['override'])
try:
net_addr = resolve_address(endpoint_type=binding)
ip = network_get_primary_address(
ADDRESS_MAP[binding]['binding'])
hostname_override = config(ADDRESS_MAP[binding]['override'])
except KeyError:
hostname_override = None
try:
try:
net_addr = resolve_address(endpoint_type=binding)
except KeyError:
net_addr = None
ip = network_get_primary_address(binding)
addresses = [net_addr, ip]
vip = get_vip_in_network(resolve_network_cidr(ip))
if vip:
addresses.append(vip)
# Clear any Nones or duplicates
addresses = list(set([i for i in addresses if i]))
# Add hostname certificate request
if hostname_override:
req.add_entry(
binding,
hostname_override,
addresses)
# Remove hostname specific addresses from _sans
for addr in addresses:
try:
_sans.remove(addr)
except (ValueError, KeyError):
pass
# Remove hostname specific addresses from _sans
for addr in addresses:
try:
_sans.remove(addr)
except (ValueError, KeyError):
pass
except NoNetworkBinding:
log("Skipping request for certificate for ip in {} space, no "
@ -174,11 +187,17 @@ def get_certificate_request(json_encode=True, bindings=None):
def get_certificate_sans(bindings=None):
"""Get all possible IP addresses for certificate SANs.
:param bindings: List of bindings to check in addition to default api
bindings.
:type bindings: list of strings
:returns: List of binding string names
:rtype: List[str]
"""
_sans = [unit_get('private-address')]
if bindings:
# Add default API bindings to bindings list
bindings = set(bindings + get_default_api_bindings())
bindings = list(bindings + get_default_api_bindings())
else:
# Use default API bindings
bindings = get_default_api_bindings()
@ -192,25 +211,39 @@ def get_certificate_sans(bindings=None):
net_config = None
# Using resolve_address is likely redundant. Keeping it here in
# case there is an edge case it handles.
net_addr = resolve_address(endpoint_type=binding)
try:
net_addr = resolve_address(endpoint_type=binding)
except KeyError:
net_addr = None
ip = get_relation_ip(binding, cidr_network=net_config)
_sans = _sans + [net_addr, ip]
vip = get_vip_in_network(resolve_network_cidr(ip))
if vip:
_sans.append(vip)
return set(_sans)
# Clear any Nones and duplicates
return list(set([i for i in _sans if i]))
def create_ip_cert_links(ssl_dir, custom_hostname_link=None):
def create_ip_cert_links(ssl_dir, custom_hostname_link=None, bindings=None):
"""Create symlinks for SAN records
:param ssl_dir: str Directory to create symlinks in
:param custom_hostname_link: str Additional link to be created
:param bindings: List of bindings to check in addition to default api
bindings.
:type bindings: list of strings
"""
if bindings:
# Add default API bindings to bindings list
bindings = list(bindings + get_default_api_bindings())
else:
# Use default API bindings
bindings = get_default_api_bindings()
# This includes the hostname cert and any specific bindng certs:
# admin, internal, public
req = get_certificate_request(json_encode=False)["cert_requests"]
req = get_certificate_request(json_encode=False, bindings=bindings)["cert_requests"]
# Specific certs
for cert_req in req.keys():
requested_cert = os.path.join(
@ -274,8 +307,35 @@ def install_certs(ssl_dir, certs, chain=None, user='root', group='root'):
content=bundle['key'], perms=0o640)
def _manage_ca_certs(ca, cert_relation_id):
"""Manage CA certs.
:param ca: CA Certificate from certificate relation.
:type ca: str
:param cert_relation_id: Relation id providing the certs
:type cert_relation_id: str
"""
config_ssl_ca = config('ssl_ca')
config_cert_file = '{}/{}.crt'.format(CA_CERT_DIR, CONFIG_CA_CERT_FILE)
if config_ssl_ca:
log("Installing CA certificate from charm ssl_ca config to {}".format(
config_cert_file), INFO)
install_ca_cert(
b64decode(config_ssl_ca).rstrip(),
name=CONFIG_CA_CERT_FILE)
elif os.path.exists(config_cert_file):
log("Removing CA certificate {}".format(config_cert_file), INFO)
os.remove(config_cert_file)
log("Installing CA certificate from certificate relation", INFO)
install_ca_cert(
ca.encode(),
name='{}_juju_ca_cert'.format(
remote_service_name(relid=cert_relation_id)))
def process_certificates(service_name, relation_id, unit,
custom_hostname_link=None, user='root', group='root'):
custom_hostname_link=None, user='root', group='root',
bindings=None):
"""Process the certificates supplied down the relation
:param service_name: str Name of service the certifcates are for.
@ -286,9 +346,19 @@ def process_certificates(service_name, relation_id, unit,
:type user: str
:param group: (Optional) Group of certificate files. Defaults to 'root'
:type group: str
:param bindings: List of bindings to check in addition to default api
bindings.
:type bindings: list of strings
:returns: True if certificates processed for local unit or False
:rtype: bool
"""
if bindings:
# Add default API bindings to bindings list
bindings = list(bindings + get_default_api_bindings())
else:
# Use default API bindings
bindings = get_default_api_bindings()
data = relation_get(rid=relation_id, unit=unit)
ssl_dir = os.path.join('/etc/apache2/ssl/', service_name)
mkdir(path=ssl_dir)
@ -298,11 +368,12 @@ def process_certificates(service_name, relation_id, unit,
ca = data.get('ca')
if certs:
certs = json.loads(certs)
install_ca_cert(ca.encode())
_manage_ca_certs(ca, relation_id)
install_certs(ssl_dir, certs, chain, user=user, group=group)
create_ip_cert_links(
ssl_dir,
custom_hostname_link=custom_hostname_link)
custom_hostname_link=custom_hostname_link,
bindings=bindings)
return True
return False

44
charmhelpers/contrib/openstack/context.py
View File

@ -1534,8 +1534,23 @@ class SubordinateConfigContext(OSContextGenerator):
ctxt[k][section] = config_list
else:
ctxt[k] = v
log("%d section(s) found" % (len(ctxt['sections'])), level=DEBUG)
return ctxt
if self.context_complete(ctxt):
log("%d section(s) found" % (len(ctxt['sections'])), level=DEBUG)
return ctxt
else:
return {}
def context_complete(self, ctxt):
"""Overridden here to ensure the context is actually complete.
:param ctxt: The current context members
:type ctxt: Dict[str, ANY]
:returns: True if the context is complete
:rtype: bool
"""
if not ctxt.get('sections'):
return False
return super(SubordinateConfigContext, self).context_complete(ctxt)
class LogLevelContext(OSContextGenerator):
@ -3050,6 +3065,9 @@ class SRIOVContext(OSContextGenerator):
blanket = 'blanket'
explicit = 'explicit'
PCIDeviceNumVFs = collections.namedtuple(
'PCIDeviceNumVFs', ['device', 'numvfs'])
def _determine_numvfs(self, device, sriov_numvfs):
"""Determine number of Virtual Functions (VFs) configured for device.
@ -3165,14 +3183,15 @@ class SRIOVContext(OSContextGenerator):
'configuration.')
self._map = {
device.interface_name: self._determine_numvfs(device, sriov_numvfs)
device.pci_address: self.PCIDeviceNumVFs(
device, self._determine_numvfs(device, sriov_numvfs))
for device in devices.pci_devices
if device.sriov and
self._determine_numvfs(device, sriov_numvfs) is not None
}
def __call__(self):
"""Provide SR-IOV context.
"""Provide backward compatible SR-IOV context.
:returns: Map interface name: min(configured, max) virtual functions.
Example:
@ -3183,6 +3202,23 @@ class SRIOVContext(OSContextGenerator):
}
:rtype: Dict[str,int]
"""
return {
pcidnvfs.device.interface_name: pcidnvfs.numvfs
for _, pcidnvfs in self._map.items()
}
@property
def get_map(self):
"""Provide map of configured SR-IOV capable PCI devices.
:returns: Map PCI-address: (PCIDevice, min(configured, max) VFs.
Example:
{
'0000:81:00.0': self.PCIDeviceNumVFs(<PCIDevice object>, 32),
'0000:81:00.1': self.PCIDeviceNumVFs(<PCIDevice object>, 32),
}
:rtype: Dict[str, self.PCIDeviceNumVFs]
"""
return self._map

30
charmhelpers/contrib/openstack/utils.py
View File

@ -90,13 +90,16 @@ from charmhelpers.core.host import (
service_start,
restart_on_change_helper,
)
from charmhelpers.fetch import (
apt_cache,
apt_install,
import_key as fetch_import_key,
add_source as fetch_add_source,
SourceConfigError,
GPGKeyError,
get_upstream_version,
filter_installed_packages,
filter_missing_packages,
ubuntu_apt_pkg as apt,
)
@ -480,9 +483,14 @@ def get_swift_codename(version):
return None
@deprecate("moved to charmhelpers.contrib.openstack.utils.get_installed_os_version()", "2021-01", log=juju_log)
def get_os_codename_package(package, fatal=True):
'''Derive OpenStack release codename from an installed package.'''
codename = get_installed_os_version()
if codename:
return codename
if snap_install_requested():
cmd = ['snap', 'list', package]
try:
@ -570,6 +578,28 @@ def get_os_version_package(pkg, fatal=True):
# error_out(e)
def get_installed_os_version():
apt_install(filter_installed_packages(['openstack-release']), fatal=False)
print("OpenStack Release: {}".format(openstack_release()))
return openstack_release().get('OPENSTACK_CODENAME')
@cached
def openstack_release():
"""Return /etc/os-release in a dict."""
d = {}
try:
with open('/etc/openstack-release', 'r') as lsb:
for l in lsb:
s = l.split('=')
if len(s) != 2:
continue
d[s[0].strip()] = s[1].strip()
except FileNotFoundError:
pass
return d
# Module local cache variable for the os_release.
_os_rel = None

3
charmhelpers/core/host.py
View File

@ -60,6 +60,7 @@ elif __platform__ == "centos":
) # flake8: noqa -- ignore F401 for this import
UPDATEDB_PATH = '/etc/updatedb.conf'
CA_CERT_DIR = '/usr/local/share/ca-certificates'
def service_start(service_name, **kwargs):
@ -1082,7 +1083,7 @@ def install_ca_cert(ca_cert, name=None):
ca_cert = ca_cert.encode('utf8')
if not name:
name = 'juju-{}'.format(charm_name())
cert_file = '/usr/local/share/ca-certificates/{}.crt'.format(name)
cert_file = '{}/{}.crt'.format(CA_CERT_DIR, name)
new_hash = hashlib.md5(ca_cert).hexdigest()
if file_hash(cert_file) == new_hash:
return

2
charmhelpers/fetch/ubuntu.py
View File

@ -646,7 +646,7 @@ def _add_apt_repository(spec):
# passed as environment variables (See lp:1433761). This is not the case
# LTS and non-LTS releases below bionic.
_run_with_retries(['add-apt-repository', '--yes', spec],
cmd_env=env_proxy_settings(['https']))
cmd_env=env_proxy_settings(['https', 'http']))
def _add_cloud_pocket(pocket):

47
charmhelpers/fetch/ubuntu_apt_pkg.py
View File

@ -129,7 +129,7 @@ class Cache(object):
else:
data = line.split(None, 4)
status = data.pop(0)
if status != 'ii':
if status not in ('ii', 'hi'):
continue
pkg = {}
pkg.update({k.lower(): v for k, v in zip(headings, data)})
@ -265,3 +265,48 @@ def version_compare(a, b):
raise RuntimeError('Unable to compare "{}" and "{}", according to '
'our logic they are neither greater, equal nor '
'less than each other.')
class PkgVersion():
"""Allow package versions to be compared.
For example::
>>> import charmhelpers.fetch as fetch
>>> (fetch.apt_pkg.PkgVersion('2:20.4.0') <
... fetch.apt_pkg.PkgVersion('2:20.5.0'))
True
>>> pkgs = [fetch.apt_pkg.PkgVersion('2:20.4.0'),
... fetch.apt_pkg.PkgVersion('2:21.4.0'),
... fetch.apt_pkg.PkgVersion('2:17.4.0')]
>>> pkgs.sort()
>>> pkgs
[2:17.4.0, 2:20.4.0, 2:21.4.0]
"""
def __init__(self, version):
self.version = version
def __lt__(self, other):
return version_compare(self.version, other.version) == -1
def __le__(self, other):
return self.__lt__(other) or self.__eq__(other)
def __gt__(self, other):
return version_compare(self.version, other.version) == 1
def __ge__(self, other):
return self.__gt__(other) or self.__eq__(other)
def __eq__(self, other):
return version_compare(self.version, other.version) == 0
def __ne__(self, other):
return not self.__eq__(other)
def __repr__(self):
return self.version
def __hash__(self):
return hash(repr(self))

2
test-requirements.txt
View File

@ -37,6 +37,8 @@ importlib-resources<3.0.0; python_version < '3.6'
# dropped support for python 3.5:
osprofiler<2.7.0;python_version<'3.6'
stevedore<1.31.0;python_version<'3.6'
debtcollector<1.22.0;python_version<'3.6'
oslo.utils<=3.41.0;python_version<'3.6'
coverage>=4.5.2
pyudev # for ceph-* charm unit tests (need to fix the ceph-* charm unit tests/mocking)