Updates for testing period for 20.01 release
Includes updates to charmhelpers/charms.openstack for cert_utils and unit-get for the install hook error on Juju 2.9 * charm-helpers sync for classic charms * rebuild for reactive charms * ensure tox.ini is from release-tools * ensure requirements.txt files are from release-tools * On reactive charms: - ensure master branch for charms.openstack - ensure master branch for charm-helpers * Remove mock for unit_get() as removed from the context module Change-Id: I1c68e1ad512d947b62c8cef6d1cc585270f64972
This commit is contained in:
Alex Kavanagh
parent
58e55f4b15
commit
055446e5c5
@ -34,6 +34,10 @@ from charmhelpers.core.hookenv import (
|
||||
INFO,
|
||||
)
|
||||
|
||||
# This file contains the CA cert from the charms ssl_ca configuration
|
||||
# option, in future the file name should be updated reflect that.
|
||||
CONFIG_CA_CERT_FILE = 'keystone_juju_ca_cert'
|
||||
|
||||
|
||||
def get_cert(cn=None):
|
||||
# TODO: deal with multiple https endpoints via charm config
|
||||
@ -83,4 +87,4 @@ def retrieve_ca_cert(cert_file):
|
||||
|
||||
|
||||
def install_ca_cert(ca_cert):
|
||||
host.install_ca_cert(ca_cert, 'keystone_juju_ca_cert')
|
||||
host.install_ca_cert(ca_cert, CONFIG_CA_CERT_FILE)
|
||||
|
||||
@ -396,7 +396,8 @@ def get_ipv6_addr(iface=None, inc_aliases=False, fatal=True, exc_list=None,
|
||||
if global_addrs:
|
||||
# Make sure any found global addresses are not temporary
|
||||
cmd = ['ip', 'addr', 'show', iface]
|
||||
out = subprocess.check_output(cmd).decode('UTF-8')
|
||||
out = subprocess.check_output(
|
||||
cmd).decode('UTF-8', errors='replace')
|
||||
if dynamic_only:
|
||||
key = re.compile("inet6 (.+)/[0-9]+ scope global.* dynamic.*")
|
||||
else:
|
||||
|
||||
@ -16,6 +16,7 @@
|
||||
|
||||
import os
|
||||
import json
|
||||
from base64 import b64decode
|
||||
|
||||
from charmhelpers.contrib.network.ip import (
|
||||
get_hostname,
|
||||
@ -28,26 +29,32 @@ from charmhelpers.core.hookenv import (
|
||||
related_units,
|
||||
relation_get,
|
||||
relation_ids,
|
||||
unit_get,
|
||||
remote_service_name,
|
||||
NoNetworkBinding,
|
||||
log,
|
||||
WARNING,
|
||||
INFO,
|
||||
)
|
||||
from charmhelpers.contrib.openstack.ip import (
|
||||
ADMIN,
|
||||
resolve_address,
|
||||
get_vip_in_network,
|
||||
INTERNAL,
|
||||
PUBLIC,
|
||||
ADDRESS_MAP)
|
||||
ADDRESS_MAP,
|
||||
get_default_api_bindings,
|
||||
local_address,
|
||||
)
|
||||
from charmhelpers.contrib.network.ip import (
|
||||
get_relation_ip,
|
||||
)
|
||||
|
||||
from charmhelpers.core.host import (
|
||||
CA_CERT_DIR,
|
||||
install_ca_cert,
|
||||
mkdir,
|
||||
write_file,
|
||||
)
|
||||
|
||||
from charmhelpers.contrib.hahelpers.apache import (
|
||||
install_ca_cert
|
||||
CONFIG_CA_CERT_FILE,
|
||||
)
|
||||
|
||||
|
||||
@ -74,7 +81,7 @@ class CertRequest(object):
|
||||
|
||||
def add_hostname_cn(self):
|
||||
"""Add a request for the hostname of the machine"""
|
||||
ip = unit_get('private-address')
|
||||
ip = local_address(unit_get_fallback='private-address')
|
||||
addresses = [ip]
|
||||
# If a vip is being used without os-hostname config or
|
||||
# network spaces then we need to ensure the local units
|
||||
@ -113,63 +120,153 @@ class CertRequest(object):
|
||||
return req
|
||||
|
||||
|
||||
def get_certificate_request(json_encode=True):
|
||||
"""Generate a certificatee requests based on the network confioguration
|
||||
def get_certificate_request(json_encode=True, bindings=None):
|
||||
"""Generate a certificate requests based on the network configuration
|
||||
|
||||
:param json_encode: Encode request in JSON or not. Used for setting
|
||||
directly on a relation.
|
||||
:type json_encode: boolean
|
||||
:param bindings: List of bindings to check in addition to default api
|
||||
bindings.
|
||||
:type bindings: list of strings
|
||||
:returns: CertRequest request as dictionary or JSON string.
|
||||
:rtype: Union[dict, json]
|
||||
"""
|
||||
if bindings:
|
||||
# Add default API bindings to bindings list
|
||||
bindings = list(bindings + get_default_api_bindings())
|
||||
else:
|
||||
# Use default API bindings
|
||||
bindings = get_default_api_bindings()
|
||||
req = CertRequest(json_encode=json_encode)
|
||||
req.add_hostname_cn()
|
||||
# Add os-hostname entries
|
||||
for net_type in [INTERNAL, ADMIN, PUBLIC]:
|
||||
net_config = config(ADDRESS_MAP[net_type]['override'])
|
||||
_sans = get_certificate_sans(bindings=bindings)
|
||||
|
||||
# Handle specific hostnames per binding
|
||||
for binding in bindings:
|
||||
try:
|
||||
net_addr = resolve_address(endpoint_type=net_type)
|
||||
ip = network_get_primary_address(
|
||||
ADDRESS_MAP[net_type]['binding'])
|
||||
hostname_override = config(ADDRESS_MAP[binding]['override'])
|
||||
except KeyError:
|
||||
hostname_override = None
|
||||
try:
|
||||
try:
|
||||
net_addr = resolve_address(endpoint_type=binding)
|
||||
except KeyError:
|
||||
net_addr = None
|
||||
ip = network_get_primary_address(binding)
|
||||
addresses = [net_addr, ip]
|
||||
vip = get_vip_in_network(resolve_network_cidr(ip))
|
||||
if vip:
|
||||
addresses.append(vip)
|
||||
if net_config:
|
||||
|
||||
# Clear any Nones or duplicates
|
||||
addresses = list(set([i for i in addresses if i]))
|
||||
# Add hostname certificate request
|
||||
if hostname_override:
|
||||
req.add_entry(
|
||||
net_type,
|
||||
net_config,
|
||||
binding,
|
||||
hostname_override,
|
||||
addresses)
|
||||
else:
|
||||
# There is network address with no corresponding hostname.
|
||||
# Add the ip to the hostname cert to allow for this.
|
||||
req.add_hostname_cn_ip(addresses)
|
||||
# Remove hostname specific addresses from _sans
|
||||
for addr in addresses:
|
||||
try:
|
||||
_sans.remove(addr)
|
||||
except (ValueError, KeyError):
|
||||
pass
|
||||
|
||||
except NoNetworkBinding:
|
||||
log("Skipping request for certificate for ip in {} space, no "
|
||||
"local address found".format(net_type), WARNING)
|
||||
"local address found".format(binding), WARNING)
|
||||
# Gurantee all SANs are covered
|
||||
# These are network addresses with no corresponding hostname.
|
||||
# Add the ips to the hostname cert to allow for this.
|
||||
req.add_hostname_cn_ip(_sans)
|
||||
return req.get_request()
|
||||
|
||||
|
||||
def create_ip_cert_links(ssl_dir, custom_hostname_link=None):
|
||||
def get_certificate_sans(bindings=None):
|
||||
"""Get all possible IP addresses for certificate SANs.
|
||||
|
||||
:param bindings: List of bindings to check in addition to default api
|
||||
bindings.
|
||||
:type bindings: list of strings
|
||||
:returns: List of binding string names
|
||||
:rtype: List[str]
|
||||
"""
|
||||
_sans = [local_address(unit_get_fallback='private-address')]
|
||||
if bindings:
|
||||
# Add default API bindings to bindings list
|
||||
bindings = list(bindings + get_default_api_bindings())
|
||||
else:
|
||||
# Use default API bindings
|
||||
bindings = get_default_api_bindings()
|
||||
|
||||
for binding in bindings:
|
||||
# Check for config override
|
||||
try:
|
||||
net_config = config(ADDRESS_MAP[binding]['config'])
|
||||
except KeyError:
|
||||
# There is no configuration network for this binding name
|
||||
net_config = None
|
||||
# Using resolve_address is likely redundant. Keeping it here in
|
||||
# case there is an edge case it handles.
|
||||
try:
|
||||
net_addr = resolve_address(endpoint_type=binding)
|
||||
except KeyError:
|
||||
net_addr = None
|
||||
ip = get_relation_ip(binding, cidr_network=net_config)
|
||||
_sans = _sans + [net_addr, ip]
|
||||
vip = get_vip_in_network(resolve_network_cidr(ip))
|
||||
if vip:
|
||||
_sans.append(vip)
|
||||
# Clear any Nones and duplicates
|
||||
return list(set([i for i in _sans if i]))
|
||||
|
||||
|
||||
def create_ip_cert_links(ssl_dir, custom_hostname_link=None, bindings=None):
|
||||
"""Create symlinks for SAN records
|
||||
|
||||
:param ssl_dir: str Directory to create symlinks in
|
||||
:param custom_hostname_link: str Additional link to be created
|
||||
:param bindings: List of bindings to check in addition to default api
|
||||
bindings.
|
||||
:type bindings: list of strings
|
||||
"""
|
||||
hostname = get_hostname(unit_get('private-address'))
|
||||
|
||||
if bindings:
|
||||
# Add default API bindings to bindings list
|
||||
bindings = list(bindings + get_default_api_bindings())
|
||||
else:
|
||||
# Use default API bindings
|
||||
bindings = get_default_api_bindings()
|
||||
|
||||
# This includes the hostname cert and any specific bindng certs:
|
||||
# admin, internal, public
|
||||
req = get_certificate_request(json_encode=False, bindings=bindings)["cert_requests"]
|
||||
# Specific certs
|
||||
for cert_req in req.keys():
|
||||
requested_cert = os.path.join(
|
||||
ssl_dir,
|
||||
'cert_{}'.format(cert_req))
|
||||
requested_key = os.path.join(
|
||||
ssl_dir,
|
||||
'key_{}'.format(cert_req))
|
||||
for addr in req[cert_req]['sans']:
|
||||
cert = os.path.join(ssl_dir, 'cert_{}'.format(addr))
|
||||
key = os.path.join(ssl_dir, 'key_{}'.format(addr))
|
||||
if os.path.isfile(requested_cert) and not os.path.isfile(cert):
|
||||
os.symlink(requested_cert, cert)
|
||||
os.symlink(requested_key, key)
|
||||
|
||||
# Handle custom hostnames
|
||||
hostname = get_hostname(local_address(unit_get_fallback='private-address'))
|
||||
hostname_cert = os.path.join(
|
||||
ssl_dir,
|
||||
'cert_{}'.format(hostname))
|
||||
hostname_key = os.path.join(
|
||||
ssl_dir,
|
||||
'key_{}'.format(hostname))
|
||||
# Add links to hostname cert, used if os-hostname vars not set
|
||||
for net_type in [INTERNAL, ADMIN, PUBLIC]:
|
||||
try:
|
||||
addr = resolve_address(endpoint_type=net_type)
|
||||
cert = os.path.join(ssl_dir, 'cert_{}'.format(addr))
|
||||
key = os.path.join(ssl_dir, 'key_{}'.format(addr))
|
||||
if os.path.isfile(hostname_cert) and not os.path.isfile(cert):
|
||||
os.symlink(hostname_cert, cert)
|
||||
os.symlink(hostname_key, key)
|
||||
except NoNetworkBinding:
|
||||
log("Skipping creating cert symlink for ip in {} space, no "
|
||||
"local address found".format(net_type), WARNING)
|
||||
if custom_hostname_link:
|
||||
custom_cert = os.path.join(
|
||||
ssl_dir,
|
||||
@ -210,8 +307,35 @@ def install_certs(ssl_dir, certs, chain=None, user='root', group='root'):
|
||||
content=bundle['key'], perms=0o640)
|
||||
|
||||
|
||||
def _manage_ca_certs(ca, cert_relation_id):
|
||||
"""Manage CA certs.
|
||||
|
||||
:param ca: CA Certificate from certificate relation.
|
||||
:type ca: str
|
||||
:param cert_relation_id: Relation id providing the certs
|
||||
:type cert_relation_id: str
|
||||
"""
|
||||
config_ssl_ca = config('ssl_ca')
|
||||
config_cert_file = '{}/{}.crt'.format(CA_CERT_DIR, CONFIG_CA_CERT_FILE)
|
||||
if config_ssl_ca:
|
||||
log("Installing CA certificate from charm ssl_ca config to {}".format(
|
||||
config_cert_file), INFO)
|
||||
install_ca_cert(
|
||||
b64decode(config_ssl_ca).rstrip(),
|
||||
name=CONFIG_CA_CERT_FILE)
|
||||
elif os.path.exists(config_cert_file):
|
||||
log("Removing CA certificate {}".format(config_cert_file), INFO)
|
||||
os.remove(config_cert_file)
|
||||
log("Installing CA certificate from certificate relation", INFO)
|
||||
install_ca_cert(
|
||||
ca.encode(),
|
||||
name='{}_juju_ca_cert'.format(
|
||||
remote_service_name(relid=cert_relation_id)))
|
||||
|
||||
|
||||
def process_certificates(service_name, relation_id, unit,
|
||||
custom_hostname_link=None, user='root', group='root'):
|
||||
custom_hostname_link=None, user='root', group='root',
|
||||
bindings=None):
|
||||
"""Process the certificates supplied down the relation
|
||||
|
||||
:param service_name: str Name of service the certifcates are for.
|
||||
@ -222,9 +346,19 @@ def process_certificates(service_name, relation_id, unit,
|
||||
:type user: str
|
||||
:param group: (Optional) Group of certificate files. Defaults to 'root'
|
||||
:type group: str
|
||||
:param bindings: List of bindings to check in addition to default api
|
||||
bindings.
|
||||
:type bindings: list of strings
|
||||
:returns: True if certificates processed for local unit or False
|
||||
:rtype: bool
|
||||
"""
|
||||
if bindings:
|
||||
# Add default API bindings to bindings list
|
||||
bindings = list(bindings + get_default_api_bindings())
|
||||
else:
|
||||
# Use default API bindings
|
||||
bindings = get_default_api_bindings()
|
||||
|
||||
data = relation_get(rid=relation_id, unit=unit)
|
||||
ssl_dir = os.path.join('/etc/apache2/ssl/', service_name)
|
||||
mkdir(path=ssl_dir)
|
||||
@ -234,11 +368,12 @@ def process_certificates(service_name, relation_id, unit,
|
||||
ca = data.get('ca')
|
||||
if certs:
|
||||
certs = json.loads(certs)
|
||||
install_ca_cert(ca.encode())
|
||||
_manage_ca_certs(ca, relation_id)
|
||||
install_certs(ssl_dir, certs, chain, user=user, group=group)
|
||||
create_ip_cert_links(
|
||||
ssl_dir,
|
||||
custom_hostname_link=custom_hostname_link)
|
||||
custom_hostname_link=custom_hostname_link,
|
||||
bindings=bindings)
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
@ -49,7 +49,6 @@ from charmhelpers.core.hookenv import (
|
||||
relation_ids,
|
||||
related_units,
|
||||
relation_set,
|
||||
unit_get,
|
||||
unit_private_ip,
|
||||
charm_name,
|
||||
DEBUG,
|
||||
@ -98,6 +97,7 @@ from charmhelpers.contrib.openstack.ip import (
|
||||
ADMIN,
|
||||
PUBLIC,
|
||||
ADDRESS_MAP,
|
||||
local_address,
|
||||
)
|
||||
from charmhelpers.contrib.network.ip import (
|
||||
get_address_in_network,
|
||||
@ -247,7 +247,7 @@ class SharedDBContext(OSContextGenerator):
|
||||
hostname_key = "hostname"
|
||||
access_hostname = get_address_in_network(
|
||||
access_network,
|
||||
unit_get('private-address'))
|
||||
local_address(unit_get_fallback='private-address'))
|
||||
set_hostname = relation_get(attribute=hostname_key,
|
||||
unit=local_unit())
|
||||
if set_hostname != access_hostname:
|
||||
@ -1088,7 +1088,7 @@ class ApacheSSLContext(OSContextGenerator):
|
||||
# NOTE(jamespage): Fallback must always be private address
|
||||
# as this is used to bind services on the
|
||||
# local unit.
|
||||
fallback = unit_get("private-address")
|
||||
fallback = local_address(unit_get_fallback="private-address")
|
||||
if net_config:
|
||||
addr = get_address_in_network(net_config,
|
||||
fallback)
|
||||
@ -1260,7 +1260,7 @@ class NeutronContext(OSContextGenerator):
|
||||
if is_clustered():
|
||||
host = config('vip')
|
||||
else:
|
||||
host = unit_get('private-address')
|
||||
host = local_address(unit_get_fallback='private-address')
|
||||
|
||||
ctxt = {'network_manager': self.network_manager,
|
||||
'neutron_url': '%s://%s:%s' % (proto, host, '9696')}
|
||||
@ -1534,8 +1534,23 @@ class SubordinateConfigContext(OSContextGenerator):
|
||||
ctxt[k][section] = config_list
|
||||
else:
|
||||
ctxt[k] = v
|
||||
log("%d section(s) found" % (len(ctxt['sections'])), level=DEBUG)
|
||||
return ctxt
|
||||
if self.context_complete(ctxt):
|
||||
log("%d section(s) found" % (len(ctxt['sections'])), level=DEBUG)
|
||||
return ctxt
|
||||
else:
|
||||
return {}
|
||||
|
||||
def context_complete(self, ctxt):
|
||||
"""Overridden here to ensure the context is actually complete.
|
||||
|
||||
:param ctxt: The current context members
|
||||
:type ctxt: Dict[str, ANY]
|
||||
:returns: True if the context is complete
|
||||
:rtype: bool
|
||||
"""
|
||||
if not ctxt.get('sections'):
|
||||
return False
|
||||
return super(SubordinateConfigContext, self).context_complete(ctxt)
|
||||
|
||||
|
||||
class LogLevelContext(OSContextGenerator):
|
||||
@ -3050,6 +3065,9 @@ class SRIOVContext(OSContextGenerator):
|
||||
blanket = 'blanket'
|
||||
explicit = 'explicit'
|
||||
|
||||
PCIDeviceNumVFs = collections.namedtuple(
|
||||
'PCIDeviceNumVFs', ['device', 'numvfs'])
|
||||
|
||||
def _determine_numvfs(self, device, sriov_numvfs):
|
||||
"""Determine number of Virtual Functions (VFs) configured for device.
|
||||
|
||||
@ -3165,14 +3183,15 @@ class SRIOVContext(OSContextGenerator):
|
||||
'configuration.')
|
||||
|
||||
self._map = {
|
||||
device.interface_name: self._determine_numvfs(device, sriov_numvfs)
|
||||
device.pci_address: self.PCIDeviceNumVFs(
|
||||
device, self._determine_numvfs(device, sriov_numvfs))
|
||||
for device in devices.pci_devices
|
||||
if device.sriov and
|
||||
self._determine_numvfs(device, sriov_numvfs) is not None
|
||||
}
|
||||
|
||||
def __call__(self):
|
||||
"""Provide SR-IOV context.
|
||||
"""Provide backward compatible SR-IOV context.
|
||||
|
||||
:returns: Map interface name: min(configured, max) virtual functions.
|
||||
Example:
|
||||
@ -3183,6 +3202,23 @@ class SRIOVContext(OSContextGenerator):
|
||||
}
|
||||
:rtype: Dict[str,int]
|
||||
"""
|
||||
return {
|
||||
pcidnvfs.device.interface_name: pcidnvfs.numvfs
|
||||
for _, pcidnvfs in self._map.items()
|
||||
}
|
||||
|
||||
@property
|
||||
def get_map(self):
|
||||
"""Provide map of configured SR-IOV capable PCI devices.
|
||||
|
||||
:returns: Map PCI-address: (PCIDevice, min(configured, max) VFs.
|
||||
Example:
|
||||
{
|
||||
'0000:81:00.0': self.PCIDeviceNumVFs(<PCIDevice object>, 32),
|
||||
'0000:81:00.1': self.PCIDeviceNumVFs(<PCIDevice object>, 32),
|
||||
}
|
||||
:rtype: Dict[str, self.PCIDeviceNumVFs]
|
||||
"""
|
||||
return self._map
|
||||
|
||||
|
||||
|
||||
@ -33,6 +33,7 @@ INTERNAL = 'int'
|
||||
ADMIN = 'admin'
|
||||
ACCESS = 'access'
|
||||
|
||||
# TODO: reconcile 'int' vs 'internal' binding names
|
||||
ADDRESS_MAP = {
|
||||
PUBLIC: {
|
||||
'binding': 'public',
|
||||
@ -58,6 +59,14 @@ ADDRESS_MAP = {
|
||||
'fallback': 'private-address',
|
||||
'override': 'os-access-hostname',
|
||||
},
|
||||
# Note (thedac) bridge to begin the reconciliation between 'int' vs
|
||||
# 'internal' binding names
|
||||
'internal': {
|
||||
'binding': 'internal',
|
||||
'config': 'os-internal-network',
|
||||
'fallback': 'private-address',
|
||||
'override': 'os-internal-hostname',
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@ -114,6 +123,25 @@ def _get_address_override(endpoint_type=PUBLIC):
|
||||
return addr_override.format(service_name=service_name())
|
||||
|
||||
|
||||
def local_address(unit_get_fallback='public-address'):
|
||||
"""Return a network address for this unit.
|
||||
|
||||
Attempt to retrieve a 'default' IP address for this unit
|
||||
from network-get. If this is running with an old version of Juju then
|
||||
fallback to unit_get.
|
||||
|
||||
:param unit_get_fallback: Either 'public-address' or 'private-address'.
|
||||
Only used with old versions of Juju.
|
||||
:type unit_get_fallback: str
|
||||
:returns: IP Address
|
||||
:rtype: str
|
||||
"""
|
||||
try:
|
||||
return network_get_primary_address('juju-info')
|
||||
except NotImplementedError:
|
||||
return unit_get(unit_get_fallback)
|
||||
|
||||
|
||||
def resolve_address(endpoint_type=PUBLIC, override=True):
|
||||
"""Return unit address depending on net config.
|
||||
|
||||
@ -167,7 +195,7 @@ def resolve_address(endpoint_type=PUBLIC, override=True):
|
||||
if config('prefer-ipv6'):
|
||||
fallback_addr = get_ipv6_addr(exc_list=vips)[0]
|
||||
else:
|
||||
fallback_addr = unit_get(net_fallback)
|
||||
fallback_addr = local_address(unit_get_fallback=net_fallback)
|
||||
|
||||
if net_addr:
|
||||
resolved_address = get_address_in_network(net_addr, fallback_addr)
|
||||
@ -195,3 +223,10 @@ def get_vip_in_network(network):
|
||||
if is_address_in_network(network, vip):
|
||||
matching_vip = vip
|
||||
return matching_vip
|
||||
|
||||
|
||||
def get_default_api_bindings():
|
||||
_default_bindings = []
|
||||
for binding in [INTERNAL, ADMIN, PUBLIC]:
|
||||
_default_bindings.append(ADDRESS_MAP[binding]['binding'])
|
||||
return _default_bindings
|
||||
|
||||
@ -15,5 +15,6 @@ password = {{ admin_password }}
|
||||
{% endif -%}
|
||||
{% if region -%}
|
||||
os_region_name = {{ region }}
|
||||
region_name = {{ region }}
|
||||
{% endif -%}
|
||||
randomize_allocation_candidates = true
|
||||
|
||||
@ -18,6 +18,7 @@ from functools import wraps
|
||||
|
||||
import subprocess
|
||||
import json
|
||||
import operator
|
||||
import os
|
||||
import sys
|
||||
import re
|
||||
@ -33,7 +34,7 @@ from charmhelpers import deprecate
|
||||
|
||||
from charmhelpers.contrib.network import ip
|
||||
|
||||
from charmhelpers.core import unitdata
|
||||
from charmhelpers.core import decorators, unitdata
|
||||
|
||||
from charmhelpers.core.hookenv import (
|
||||
WORKLOAD_STATES,
|
||||
@ -89,13 +90,16 @@ from charmhelpers.core.host import (
|
||||
service_start,
|
||||
restart_on_change_helper,
|
||||
)
|
||||
|
||||
from charmhelpers.fetch import (
|
||||
apt_cache,
|
||||
apt_install,
|
||||
import_key as fetch_import_key,
|
||||
add_source as fetch_add_source,
|
||||
SourceConfigError,
|
||||
GPGKeyError,
|
||||
get_upstream_version,
|
||||
filter_installed_packages,
|
||||
filter_missing_packages,
|
||||
ubuntu_apt_pkg as apt,
|
||||
)
|
||||
@ -230,7 +234,7 @@ SWIFT_CODENAMES = OrderedDict([
|
||||
('ussuri',
|
||||
['2.24.0', '2.25.0']),
|
||||
('victoria',
|
||||
['2.25.0']),
|
||||
['2.25.0', '2.26.0']),
|
||||
])
|
||||
|
||||
# >= Liberty version->codename mapping
|
||||
@ -479,9 +483,14 @@ def get_swift_codename(version):
|
||||
return None
|
||||
|
||||
|
||||
@deprecate("moved to charmhelpers.contrib.openstack.utils.get_installed_os_version()", "2021-01", log=juju_log)
|
||||
def get_os_codename_package(package, fatal=True):
|
||||
'''Derive OpenStack release codename from an installed package.'''
|
||||
|
||||
codename = get_installed_os_version()
|
||||
if codename:
|
||||
return codename
|
||||
|
||||
if snap_install_requested():
|
||||
cmd = ['snap', 'list', package]
|
||||
try:
|
||||
@ -569,6 +578,28 @@ def get_os_version_package(pkg, fatal=True):
|
||||
# error_out(e)
|
||||
|
||||
|
||||
def get_installed_os_version():
|
||||
apt_install(filter_installed_packages(['openstack-release']), fatal=False)
|
||||
print("OpenStack Release: {}".format(openstack_release()))
|
||||
return openstack_release().get('OPENSTACK_CODENAME')
|
||||
|
||||
|
||||
@cached
|
||||
def openstack_release():
|
||||
"""Return /etc/os-release in a dict."""
|
||||
d = {}
|
||||
try:
|
||||
with open('/etc/openstack-release', 'r') as lsb:
|
||||
for l in lsb:
|
||||
s = l.split('=')
|
||||
if len(s) != 2:
|
||||
continue
|
||||
d[s[0].strip()] = s[1].strip()
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
return d
|
||||
|
||||
|
||||
# Module local cache variable for the os_release.
|
||||
_os_rel = None
|
||||
|
||||
@ -1295,7 +1326,7 @@ def _check_listening_on_ports_list(ports):
|
||||
Returns a list of ports being listened to and a list of the
|
||||
booleans.
|
||||
|
||||
@param ports: LIST or port numbers.
|
||||
@param ports: LIST of port numbers.
|
||||
@returns [(port_num, boolean), ...], [boolean]
|
||||
"""
|
||||
ports_open = [port_has_listener('0.0.0.0', p) for p in ports]
|
||||
@ -1564,6 +1595,21 @@ def manage_payload_services(action, services=None, charm_func=None):
|
||||
return success, messages
|
||||
|
||||
|
||||
def make_wait_for_ports_barrier(ports, retry_count=5):
|
||||
"""Make a function to wait for port shutdowns.
|
||||
|
||||
Create a function which closes over the provided ports. The function will
|
||||
retry probing ports until they are closed or the retry count has been reached.
|
||||
|
||||
"""
|
||||
@decorators.retry_on_predicate(retry_count, operator.not_, base_delay=0.1)
|
||||
def retry_port_check():
|
||||
_, ports_states = _check_listening_on_ports_list(ports)
|
||||
juju_log("Probe ports {}, result: {}".format(ports, ports_states), level="DEBUG")
|
||||
return any(ports_states)
|
||||
return retry_port_check
|
||||
|
||||
|
||||
def pause_unit(assess_status_func, services=None, ports=None,
|
||||
charm_func=None):
|
||||
"""Pause a unit by stopping the services and setting 'unit-paused'
|
||||
@ -1599,6 +1645,7 @@ def pause_unit(assess_status_func, services=None, ports=None,
|
||||
services=services,
|
||||
charm_func=charm_func)
|
||||
set_unit_paused()
|
||||
|
||||
if assess_status_func:
|
||||
message = assess_status_func()
|
||||
if message:
|
||||
|
||||
@ -41,6 +41,7 @@ from subprocess import (
|
||||
)
|
||||
from charmhelpers import deprecate
|
||||
from charmhelpers.core.hookenv import (
|
||||
application_name,
|
||||
config,
|
||||
service_name,
|
||||
local_unit,
|
||||
@ -162,6 +163,17 @@ def get_osd_settings(relation_name):
|
||||
return _order_dict_by_key(osd_settings)
|
||||
|
||||
|
||||
def send_application_name(relid=None):
|
||||
"""Send the application name down the relation.
|
||||
|
||||
:param relid: Relation id to set application name in.
|
||||
:type relid: str
|
||||
"""
|
||||
relation_set(
|
||||
relation_id=relid,
|
||||
relation_settings={'application-name': application_name()})
|
||||
|
||||
|
||||
def send_osd_settings():
|
||||
"""Pass on requested OSD settings to osd units."""
|
||||
try:
|
||||
@ -256,6 +268,7 @@ class BasePool(object):
|
||||
'compression-max-blob-size': (int, None),
|
||||
'compression-max-blob-size-hdd': (int, None),
|
||||
'compression-max-blob-size-ssd': (int, None),
|
||||
'rbd-mirroring-mode': (str, ('image', 'pool'))
|
||||
}
|
||||
|
||||
def __init__(self, service, name=None, percent_data=None, app_name=None,
|
||||
@ -1074,7 +1087,10 @@ def create_erasure_profile(service, profile_name,
|
||||
erasure_plugin_technique=None):
|
||||
"""Create a new erasure code profile if one does not already exist for it.
|
||||
|
||||
Updates the profile if it exists. Please refer to [0] for more details.
|
||||
Profiles are considered immutable so will not be updated if the named
|
||||
profile already exists.
|
||||
|
||||
Please refer to [0] for more details.
|
||||
|
||||
0: http://docs.ceph.com/docs/master/rados/operations/erasure-code-profile/
|
||||
|
||||
@ -1110,6 +1126,11 @@ def create_erasure_profile(service, profile_name,
|
||||
:type erasure_plugin_technique: str
|
||||
:return: None. Can raise CalledProcessError, ValueError or AssertionError
|
||||
"""
|
||||
if erasure_profile_exists(service, profile_name):
|
||||
log('EC profile {} exists, skipping update'.format(profile_name),
|
||||
level=WARNING)
|
||||
return
|
||||
|
||||
plugin_techniques = {
|
||||
'jerasure': [
|
||||
'reed_sol_van',
|
||||
@ -1209,9 +1230,6 @@ def create_erasure_profile(service, profile_name,
|
||||
if scalar_mds:
|
||||
cmd.append('scalar-mds={}'.format(scalar_mds))
|
||||
|
||||
if erasure_profile_exists(service, profile_name):
|
||||
cmd.append('--force')
|
||||
|
||||
check_call(cmd)
|
||||
|
||||
|
||||
@ -1750,6 +1768,7 @@ class CephBrokerRq(object):
|
||||
max_bytes=None,
|
||||
max_objects=None,
|
||||
namespace=None,
|
||||
rbd_mirroring_mode='pool',
|
||||
weight=None):
|
||||
"""Build common part of a create pool operation.
|
||||
|
||||
@ -1808,6 +1827,9 @@ class CephBrokerRq(object):
|
||||
:type max_objects: Optional[int]
|
||||
:param namespace: Group namespace
|
||||
:type namespace: Optional[str]
|
||||
:param rbd_mirroring_mode: Pool mirroring mode used when Ceph RBD
|
||||
mirroring is enabled.
|
||||
:type rbd_mirroring_mode: Optional[str]
|
||||
:param weight: The percentage of data that is expected to be contained
|
||||
in the pool from the total available space on the OSDs.
|
||||
Used to calculate number of Placement Groups to create
|
||||
@ -1832,6 +1854,7 @@ class CephBrokerRq(object):
|
||||
'max-bytes': max_bytes,
|
||||
'max-objects': max_objects,
|
||||
'group-namespace': namespace,
|
||||
'rbd-mirroring-mode': rbd_mirroring_mode,
|
||||
'weight': weight,
|
||||
}
|
||||
|
||||
@ -2198,6 +2221,7 @@ def send_request_if_needed(request, relation='ceph'):
|
||||
for rid in relation_ids(relation):
|
||||
log('Sending request {}'.format(request.request_id), level=DEBUG)
|
||||
relation_set(relation_id=rid, broker_req=request.request)
|
||||
relation_set(relation_id=rid, relation_settings={'unit-name': local_unit()})
|
||||
|
||||
|
||||
def has_broker_rsp(rid=None, unit=None):
|
||||
|
||||
@ -53,3 +53,41 @@ def retry_on_exception(num_retries, base_delay=0, exc_type=Exception):
|
||||
return _retry_on_exception_inner_2
|
||||
|
||||
return _retry_on_exception_inner_1
|
||||
|
||||
|
||||
def retry_on_predicate(num_retries, predicate_fun, base_delay=0):
|
||||
"""Retry based on return value
|
||||
|
||||
The return value of the decorated function is passed to the given predicate_fun. If the
|
||||
result of the predicate is False, retry the decorated function up to num_retries times
|
||||
|
||||
An exponential backoff up to base_delay^num_retries seconds can be introduced by setting
|
||||
base_delay to a nonzero value. The default is to run with a zero (i.e. no) delay
|
||||
|
||||
:param num_retries: Max. number of retries to perform
|
||||
:type num_retries: int
|
||||
:param predicate_fun: Predicate function to determine if a retry is necessary
|
||||
:type predicate_fun: callable
|
||||
:param base_delay: Starting value in seconds for exponential delay, defaults to 0 (no delay)
|
||||
:type base_delay: float
|
||||
"""
|
||||
def _retry_on_pred_inner_1(f):
|
||||
def _retry_on_pred_inner_2(*args, **kwargs):
|
||||
retries = num_retries
|
||||
multiplier = 1
|
||||
delay = base_delay
|
||||
while True:
|
||||
result = f(*args, **kwargs)
|
||||
if predicate_fun(result) or retries <= 0:
|
||||
return result
|
||||
delay *= multiplier
|
||||
multiplier += 1
|
||||
log("Result {}, retrying '{}' {} more times (delay={})".format(
|
||||
result, f.__name__, retries, delay), level=INFO)
|
||||
retries -= 1
|
||||
if delay:
|
||||
time.sleep(delay)
|
||||
|
||||
return _retry_on_pred_inner_2
|
||||
|
||||
return _retry_on_pred_inner_1
|
||||
|
||||
@ -19,6 +19,7 @@
|
||||
# Nick Moffitt <nick.moffitt@canonical.com>
|
||||
# Matthew Wedgwood <matthew.wedgwood@canonical.com>
|
||||
|
||||
import errno
|
||||
import os
|
||||
import re
|
||||
import pwd
|
||||
@ -59,6 +60,7 @@ elif __platform__ == "centos":
|
||||
) # flake8: noqa -- ignore F401 for this import
|
||||
|
||||
UPDATEDB_PATH = '/etc/updatedb.conf'
|
||||
CA_CERT_DIR = '/usr/local/share/ca-certificates'
|
||||
|
||||
|
||||
def service_start(service_name, **kwargs):
|
||||
@ -677,7 +679,7 @@ def check_hash(path, checksum, hash_type='md5'):
|
||||
|
||||
:param str checksum: Value of the checksum used to validate the file.
|
||||
:param str hash_type: Hash algorithm used to generate `checksum`.
|
||||
Can be any hash alrgorithm supported by :mod:`hashlib`,
|
||||
Can be any hash algorithm supported by :mod:`hashlib`,
|
||||
such as md5, sha1, sha256, sha512, etc.
|
||||
:raises ChecksumError: If the file fails the checksum
|
||||
|
||||
@ -825,7 +827,8 @@ def list_nics(nic_type=None):
|
||||
if nic_type:
|
||||
for int_type in int_types:
|
||||
cmd = ['ip', 'addr', 'show', 'label', int_type + '*']
|
||||
ip_output = subprocess.check_output(cmd).decode('UTF-8')
|
||||
ip_output = subprocess.check_output(
|
||||
cmd).decode('UTF-8', errors='replace')
|
||||
ip_output = ip_output.split('\n')
|
||||
ip_output = (line for line in ip_output if line)
|
||||
for line in ip_output:
|
||||
@ -841,7 +844,8 @@ def list_nics(nic_type=None):
|
||||
interfaces.append(iface)
|
||||
else:
|
||||
cmd = ['ip', 'a']
|
||||
ip_output = subprocess.check_output(cmd).decode('UTF-8').split('\n')
|
||||
ip_output = subprocess.check_output(
|
||||
cmd).decode('UTF-8', errors='replace').split('\n')
|
||||
ip_output = (line.strip() for line in ip_output if line)
|
||||
|
||||
key = re.compile(r'^[0-9]+:\s+(.+):')
|
||||
@ -865,7 +869,8 @@ def set_nic_mtu(nic, mtu):
|
||||
def get_nic_mtu(nic):
|
||||
"""Return the Maximum Transmission Unit (MTU) for a network interface."""
|
||||
cmd = ['ip', 'addr', 'show', nic]
|
||||
ip_output = subprocess.check_output(cmd).decode('UTF-8').split('\n')
|
||||
ip_output = subprocess.check_output(
|
||||
cmd).decode('UTF-8', errors='replace').split('\n')
|
||||
mtu = ""
|
||||
for line in ip_output:
|
||||
words = line.split()
|
||||
@ -877,7 +882,7 @@ def get_nic_mtu(nic):
|
||||
def get_nic_hwaddr(nic):
|
||||
"""Return the Media Access Control (MAC) for a network interface."""
|
||||
cmd = ['ip', '-o', '-0', 'addr', 'show', nic]
|
||||
ip_output = subprocess.check_output(cmd).decode('UTF-8')
|
||||
ip_output = subprocess.check_output(cmd).decode('UTF-8', errors='replace')
|
||||
hwaddr = ""
|
||||
words = ip_output.split()
|
||||
if 'link/ether' in words:
|
||||
@ -889,7 +894,7 @@ def get_nic_hwaddr(nic):
|
||||
def chdir(directory):
|
||||
"""Change the current working directory to a different directory for a code
|
||||
block and return the previous directory after the block exits. Useful to
|
||||
run commands from a specificed directory.
|
||||
run commands from a specified directory.
|
||||
|
||||
:param str directory: The directory path to change to for this context.
|
||||
"""
|
||||
@ -924,9 +929,13 @@ def chownr(path, owner, group, follow_links=True, chowntopdir=False):
|
||||
for root, dirs, files in os.walk(path, followlinks=follow_links):
|
||||
for name in dirs + files:
|
||||
full = os.path.join(root, name)
|
||||
broken_symlink = os.path.lexists(full) and not os.path.exists(full)
|
||||
if not broken_symlink:
|
||||
try:
|
||||
chown(full, uid, gid)
|
||||
except (IOError, OSError) as e:
|
||||
# Intended to ignore "file not found". Catching both to be
|
||||
# compatible with both Python 2.7 and 3.x.
|
||||
if e.errno == errno.ENOENT:
|
||||
pass
|
||||
|
||||
|
||||
def lchownr(path, owner, group):
|
||||
@ -1074,7 +1083,7 @@ def install_ca_cert(ca_cert, name=None):
|
||||
ca_cert = ca_cert.encode('utf8')
|
||||
if not name:
|
||||
name = 'juju-{}'.format(charm_name())
|
||||
cert_file = '/usr/local/share/ca-certificates/{}.crt'.format(name)
|
||||
cert_file = '{}/{}.crt'.format(CA_CERT_DIR, name)
|
||||
new_hash = hashlib.md5(ca_cert).hexdigest()
|
||||
if file_hash(cert_file) == new_hash:
|
||||
return
|
||||
|
||||
@ -646,7 +646,7 @@ def _add_apt_repository(spec):
|
||||
# passed as environment variables (See lp:1433761). This is not the case
|
||||
# LTS and non-LTS releases below bionic.
|
||||
_run_with_retries(['add-apt-repository', '--yes', spec],
|
||||
cmd_env=env_proxy_settings(['https']))
|
||||
cmd_env=env_proxy_settings(['https', 'http']))
|
||||
|
||||
|
||||
def _add_cloud_pocket(pocket):
|
||||
|
||||
@ -129,7 +129,7 @@ class Cache(object):
|
||||
else:
|
||||
data = line.split(None, 4)
|
||||
status = data.pop(0)
|
||||
if status != 'ii':
|
||||
if status not in ('ii', 'hi'):
|
||||
continue
|
||||
pkg = {}
|
||||
pkg.update({k.lower(): v for k, v in zip(headings, data)})
|
||||
@ -265,3 +265,48 @@ def version_compare(a, b):
|
||||
raise RuntimeError('Unable to compare "{}" and "{}", according to '
|
||||
'our logic they are neither greater, equal nor '
|
||||
'less than each other.')
|
||||
|
||||
|
||||
class PkgVersion():
|
||||
"""Allow package versions to be compared.
|
||||
|
||||
For example::
|
||||
|
||||
>>> import charmhelpers.fetch as fetch
|
||||
>>> (fetch.apt_pkg.PkgVersion('2:20.4.0') <
|
||||
... fetch.apt_pkg.PkgVersion('2:20.5.0'))
|
||||
True
|
||||
>>> pkgs = [fetch.apt_pkg.PkgVersion('2:20.4.0'),
|
||||
... fetch.apt_pkg.PkgVersion('2:21.4.0'),
|
||||
... fetch.apt_pkg.PkgVersion('2:17.4.0')]
|
||||
>>> pkgs.sort()
|
||||
>>> pkgs
|
||||
[2:17.4.0, 2:20.4.0, 2:21.4.0]
|
||||
"""
|
||||
|
||||
def __init__(self, version):
|
||||
self.version = version
|
||||
|
||||
def __lt__(self, other):
|
||||
return version_compare(self.version, other.version) == -1
|
||||
|
||||
def __le__(self, other):
|
||||
return self.__lt__(other) or self.__eq__(other)
|
||||
|
||||
def __gt__(self, other):
|
||||
return version_compare(self.version, other.version) == 1
|
||||
|
||||
def __ge__(self, other):
|
||||
return self.__gt__(other) or self.__eq__(other)
|
||||
|
||||
def __eq__(self, other):
|
||||
return version_compare(self.version, other.version) == 0
|
||||
|
||||
def __ne__(self, other):
|
||||
return not self.__eq__(other)
|
||||
|
||||
def __repr__(self):
|
||||
return self.version
|
||||
|
||||
def __hash__(self):
|
||||
return hash(repr(self))
|
||||
|
||||
@ -37,6 +37,8 @@ importlib-resources<3.0.0; python_version < '3.6'
|
||||
# dropped support for python 3.5:
|
||||
osprofiler<2.7.0;python_version<'3.6'
|
||||
stevedore<1.31.0;python_version<'3.6'
|
||||
debtcollector<1.22.0;python_version<'3.6'
|
||||
oslo.utils<=3.41.0;python_version<'3.6'
|
||||
|
||||
coverage>=4.5.2
|
||||
pyudev # for ceph-* charm unit tests (need to fix the ceph-* charm unit tests/mocking)
|
||||
|
||||
@ -106,7 +106,7 @@ class OVSPluginContextTest(CharmTestCase):
|
||||
@patch.object(charmhelpers.contrib.openstack.context, 'relation_ids')
|
||||
@patch.object(charmhelpers.contrib.openstack.context, 'related_units')
|
||||
@patch.object(charmhelpers.contrib.openstack.context, 'config')
|
||||
@patch.object(charmhelpers.contrib.openstack.context, 'unit_get')
|
||||
@patch.object(charmhelpers.contrib.openstack.context, 'local_address')
|
||||
@patch.object(charmhelpers.contrib.openstack.context, 'is_clustered')
|
||||
@patch.object(charmhelpers.contrib.openstack.context, 'https')
|
||||
@patch.object(context.OVSPluginContext, '_ensure_packages')
|
||||
@ -114,7 +114,8 @@ class OVSPluginContextTest(CharmTestCase):
|
||||
'neutron_plugin_attribute')
|
||||
@patch.object(charmhelpers.contrib.openstack.context, 'unit_private_ip')
|
||||
def test_neutroncc_context_api_rel(self, _unit_priv_ip, _npa, _ens_pkgs,
|
||||
_https, _is_clus, _unit_get,
|
||||
_https, _is_clus,
|
||||
_local_address,
|
||||
_config, _runits, _rids, _rget,
|
||||
_get_os_cdnm_pkg):
|
||||
def mock_npa(plugin, section, manager):
|
||||
@ -148,7 +149,7 @@ class OVSPluginContextTest(CharmTestCase):
|
||||
self.maxDiff = None
|
||||
self.config.side_effect = mock_config
|
||||
_npa.side_effect = mock_npa
|
||||
_unit_get.return_value = '127.0.0.13'
|
||||
_local_address.return_value = '127.0.0.13'
|
||||
_unit_priv_ip.return_value = '127.0.0.14'
|
||||
_is_clus.return_value = False
|
||||
_runits.return_value = ['unit1']
|
||||
@ -206,7 +207,7 @@ class OVSPluginContextTest(CharmTestCase):
|
||||
@patch.object(charmhelpers.contrib.openstack.context, 'relation_ids')
|
||||
@patch.object(charmhelpers.contrib.openstack.context, 'related_units')
|
||||
@patch.object(charmhelpers.contrib.openstack.context, 'config')
|
||||
@patch.object(charmhelpers.contrib.openstack.context, 'unit_get')
|
||||
@patch.object(charmhelpers.contrib.openstack.context, 'local_address')
|
||||
@patch.object(charmhelpers.contrib.openstack.context, 'is_clustered')
|
||||
@patch.object(charmhelpers.contrib.openstack.context, 'https')
|
||||
@patch.object(context.OVSPluginContext, '_ensure_packages')
|
||||
@ -217,7 +218,7 @@ class OVSPluginContextTest(CharmTestCase):
|
||||
_unit_priv_ip, _npa,
|
||||
_ens_pkgs,
|
||||
_https, _is_clus,
|
||||
_unit_get,
|
||||
_local_address,
|
||||
_config, _runits,
|
||||
_rids, _rget,
|
||||
_get_os_cdnm_pkg):
|
||||
@ -230,7 +231,7 @@ class OVSPluginContextTest(CharmTestCase):
|
||||
_get_os_cdnm_pkg.return_value = 'ocata'
|
||||
_npa.side_effect = mock_npa
|
||||
_config.return_value = 'ovs'
|
||||
_unit_get.return_value = '127.0.0.13'
|
||||
_local_address.return_value = '127.0.0.13'
|
||||
_unit_priv_ip.return_value = '127.0.0.14'
|
||||
_is_clus.return_value = False
|
||||
self.test_config.set('disable-security-groups', True)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user