openstack/charm-ceph-proxy
Code Issues Proposed changes

Add Groovy to the test gate

Browse Source 
Also sync libraries

Change-Id: I60d6b713c152c14b5af37b5c87308c72408801e3
This commit is contained in:
Aurelien Lourot 2020-11-05 12:44:16 +01:00
parent 3249ce6585
commit bee30372d6
11 changed files with 143 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
charmhelpers/contrib/hahelpers/apache.py
View File

@ -34,6 +34,10 @@ from charmhelpers.core.hookenv import (
INFO,
)
# This file contains the CA cert from the charms ssl_ca configuration
# option, in future the file name should be updated reflect that.
CONFIG_CA_CERT_FILE = 'keystone_juju_ca_cert'
def get_cert(cn=None):
# TODO: deal with multiple https endpoints via charm config
@ -83,4 +87,4 @@ def retrieve_ca_cert(cert_file):
def install_ca_cert(ca_cert):
host.install_ca_cert(ca_cert, 'keystone_juju_ca_cert')
host.install_ca_cert(ca_cert, CONFIG_CA_CERT_FILE)

3
charmhelpers/contrib/network/ip.py
View File

@ -396,7 +396,8 @@ def get_ipv6_addr(iface=None, inc_aliases=False, fatal=True, exc_list=None,
if global_addrs:
# Make sure any found global addresses are not temporary
cmd = ['ip', 'addr', 'show', iface]
out = subprocess.check_output(cmd).decode('UTF-8')
out = subprocess.check_output(
cmd).decode('UTF-8', errors='replace')
if dynamic_only:
key = re.compile("inet6 (.+)/[0-9]+ scope global.* dynamic.*")
else:

16
charmhelpers/contrib/openstack/ip.py
View File

@ -33,6 +33,7 @@ INTERNAL = 'int'
ADMIN = 'admin'
ACCESS = 'access'
# TODO: reconcile 'int' vs 'internal' binding names
ADDRESS_MAP = {
PUBLIC: {
'binding': 'public',
@ -58,6 +59,14 @@ ADDRESS_MAP = {
'fallback': 'private-address',
'override': 'os-access-hostname',
},
# Note (thedac) bridge to begin the reconciliation between 'int' vs
# 'internal' binding names
'internal': {
'binding': 'internal',
'config': 'os-internal-network',
'fallback': 'private-address',
'override': 'os-internal-hostname',
},
}
@ -195,3 +204,10 @@ def get_vip_in_network(network):
if is_address_in_network(network, vip):
matching_vip = vip
return matching_vip
def get_default_api_bindings():
_default_bindings = []
for binding in [INTERNAL, ADMIN, PUBLIC]:
_default_bindings.append(ADDRESS_MAP[binding]['binding'])
return _default_bindings

23
charmhelpers/contrib/openstack/utils.py
View File

@ -18,6 +18,7 @@ from functools import wraps
import subprocess
import json
import operator
import os
import sys
import re
@ -33,7 +34,7 @@ from charmhelpers import deprecate
from charmhelpers.contrib.network import ip
from charmhelpers.core import unitdata
from charmhelpers.core import decorators, unitdata
from charmhelpers.core.hookenv import (
WORKLOAD_STATES,
@ -230,7 +231,7 @@ SWIFT_CODENAMES = OrderedDict([
('ussuri',
['2.24.0', '2.25.0']),
('victoria',
['2.25.0']),
['2.25.0', '2.26.0']),
])
# >= Liberty version->codename mapping
@ -1295,7 +1296,7 @@ def _check_listening_on_ports_list(ports):
Returns a list of ports being listened to and a list of the
booleans.
@param ports: LIST or port numbers.
@param ports: LIST of port numbers.
@returns [(port_num, boolean), ...], [boolean]
"""
ports_open = [port_has_listener('0.0.0.0', p) for p in ports]
@ -1564,6 +1565,21 @@ def manage_payload_services(action, services=None, charm_func=None):
return success, messages
def make_wait_for_ports_barrier(ports, retry_count=5):
"""Make a function to wait for port shutdowns.
Create a function which closes over the provided ports. The function will
retry probing ports until they are closed or the retry count has been reached.
"""
@decorators.retry_on_predicate(retry_count, operator.not_, base_delay=0.1)
def retry_port_check():
_, ports_states = _check_listening_on_ports_list(ports)
juju_log("Probe ports {}, result: {}".format(ports, ports_states), level="DEBUG")
return any(ports_states)
return retry_port_check
def pause_unit(assess_status_func, services=None, ports=None,
charm_func=None):
"""Pause a unit by stopping the services and setting 'unit-paused'
@ -1599,6 +1615,7 @@ def pause_unit(assess_status_func, services=None, ports=None,
services=services,
charm_func=charm_func)
set_unit_paused()
if assess_status_func:
message = assess_status_func()
if message:

19
charmhelpers/contrib/storage/linux/ceph.py
View File

@ -41,6 +41,7 @@ from subprocess import (
)
from charmhelpers import deprecate
from charmhelpers.core.hookenv import (
application_name,
config,
service_name,
local_unit,
@ -162,6 +163,17 @@ def get_osd_settings(relation_name):
return _order_dict_by_key(osd_settings)
def send_application_name(relid=None):
"""Send the application name down the relation.
:param relid: Relation id to set application name in.
:type relid: str
"""
relation_set(
relation_id=relid,
relation_settings={'application-name': application_name()})
def send_osd_settings():
"""Pass on requested OSD settings to osd units."""
try:
@ -256,6 +268,7 @@ class BasePool(object):
'compression-max-blob-size': (int, None),
'compression-max-blob-size-hdd': (int, None),
'compression-max-blob-size-ssd': (int, None),
'rbd-mirroring-mode': (str, ('image', 'pool'))
}
def __init__(self, service, name=None, percent_data=None, app_name=None,
@ -1755,6 +1768,7 @@ class CephBrokerRq(object):
max_bytes=None,
max_objects=None,
namespace=None,
rbd_mirroring_mode='pool',
weight=None):
"""Build common part of a create pool operation.
@ -1813,6 +1827,9 @@ class CephBrokerRq(object):
:type max_objects: Optional[int]
:param namespace: Group namespace
:type namespace: Optional[str]
:param rbd_mirroring_mode: Pool mirroring mode used when Ceph RBD
mirroring is enabled.
:type rbd_mirroring_mode: Optional[str]
:param weight: The percentage of data that is expected to be contained
in the pool from the total available space on the OSDs.
Used to calculate number of Placement Groups to create
@ -1837,6 +1854,7 @@ class CephBrokerRq(object):
'max-bytes': max_bytes,
'max-objects': max_objects,
'group-namespace': namespace,
'rbd-mirroring-mode': rbd_mirroring_mode,
'weight': weight,
}
@ -2203,6 +2221,7 @@ def send_request_if_needed(request, relation='ceph'):
for rid in relation_ids(relation):
log('Sending request {}'.format(request.request_id), level=DEBUG)
relation_set(relation_id=rid, broker_req=request.request)
relation_set(relation_id=rid, relation_settings={'unit-name': local_unit()})
def has_broker_rsp(rid=None, unit=None):

38
charmhelpers/core/decorators.py
View File

@ -53,3 +53,41 @@ def retry_on_exception(num_retries, base_delay=0, exc_type=Exception):
return _retry_on_exception_inner_2
return _retry_on_exception_inner_1
def retry_on_predicate(num_retries, predicate_fun, base_delay=0):
"""Retry based on return value
The return value of the decorated function is passed to the given predicate_fun. If the
result of the predicate is False, retry the decorated function up to num_retries times
An exponential backoff up to base_delay^num_retries seconds can be introduced by setting
base_delay to a nonzero value. The default is to run with a zero (i.e. no) delay
:param num_retries: Max. number of retries to perform
:type num_retries: int
:param predicate_fun: Predicate function to determine if a retry is necessary
:type predicate_fun: callable
:param base_delay: Starting value in seconds for exponential delay, defaults to 0 (no delay)
:type base_delay: float
"""
def _retry_on_pred_inner_1(f):
def _retry_on_pred_inner_2(*args, **kwargs):
retries = num_retries
multiplier = 1
delay = base_delay
while True:
result = f(*args, **kwargs)
if predicate_fun(result) or retries <= 0:
return result
delay *= multiplier
multiplier += 1
log("Result {}, retrying '{}' {} more times (delay={})".format(
result, f.__name__, retries, delay), level=INFO)
retries -= 1
if delay:
time.sleep(delay)
return _retry_on_pred_inner_2
return _retry_on_pred_inner_1

27
charmhelpers/core/host.py
View File

@ -19,6 +19,7 @@
# Nick Moffitt <nick.moffitt@canonical.com>
# Matthew Wedgwood <matthew.wedgwood@canonical.com>
import errno
import os
import re
import pwd
@ -59,6 +60,7 @@ elif __platform__ == "centos":
) # flake8: noqa -- ignore F401 for this import
UPDATEDB_PATH = '/etc/updatedb.conf'
CA_CERT_DIR = '/usr/local/share/ca-certificates'
def service_start(service_name, **kwargs):
@ -677,7 +679,7 @@ def check_hash(path, checksum, hash_type='md5'):
:param str checksum: Value of the checksum used to validate the file.
:param str hash_type: Hash algorithm used to generate `checksum`.
Can be any hash alrgorithm supported by :mod:`hashlib`,
Can be any hash algorithm supported by :mod:`hashlib`,
such as md5, sha1, sha256, sha512, etc.
:raises ChecksumError: If the file fails the checksum
@ -825,7 +827,8 @@ def list_nics(nic_type=None):
if nic_type:
for int_type in int_types:
cmd = ['ip', 'addr', 'show', 'label', int_type + '*']
ip_output = subprocess.check_output(cmd).decode('UTF-8')
ip_output = subprocess.check_output(
cmd).decode('UTF-8', errors='replace')
ip_output = ip_output.split('\n')
ip_output = (line for line in ip_output if line)
for line in ip_output:
@ -841,7 +844,8 @@ def list_nics(nic_type=None):
interfaces.append(iface)
else:
cmd = ['ip', 'a']
ip_output = subprocess.check_output(cmd).decode('UTF-8').split('\n')
ip_output = subprocess.check_output(
cmd).decode('UTF-8', errors='replace').split('\n')
ip_output = (line.strip() for line in ip_output if line)
key = re.compile(r'^[0-9]+:\s+(.+):')
@ -865,7 +869,8 @@ def set_nic_mtu(nic, mtu):
def get_nic_mtu(nic):
"""Return the Maximum Transmission Unit (MTU) for a network interface."""
cmd = ['ip', 'addr', 'show', nic]
ip_output = subprocess.check_output(cmd).decode('UTF-8').split('\n')
ip_output = subprocess.check_output(
cmd).decode('UTF-8', errors='replace').split('\n')
mtu = ""
for line in ip_output:
words = line.split()
@ -877,7 +882,7 @@ def get_nic_mtu(nic):
def get_nic_hwaddr(nic):
"""Return the Media Access Control (MAC) for a network interface."""
cmd = ['ip', '-o', '-0', 'addr', 'show', nic]
ip_output = subprocess.check_output(cmd).decode('UTF-8')
ip_output = subprocess.check_output(cmd).decode('UTF-8', errors='replace')
hwaddr = ""
words = ip_output.split()
if 'link/ether' in words:
@ -889,7 +894,7 @@ def get_nic_hwaddr(nic):
def chdir(directory):
"""Change the current working directory to a different directory for a code
block and return the previous directory after the block exits. Useful to
run commands from a specificed directory.
run commands from a specified directory.
:param str directory: The directory path to change to for this context.
"""
@ -924,9 +929,13 @@ def chownr(path, owner, group, follow_links=True, chowntopdir=False):
for root, dirs, files in os.walk(path, followlinks=follow_links):
for name in dirs + files:
full = os.path.join(root, name)
broken_symlink = os.path.lexists(full) and not os.path.exists(full)
if not broken_symlink:
try:
chown(full, uid, gid)
except (IOError, OSError) as e:
# Intended to ignore "file not found". Catching both to be
# compatible with both Python 2.7 and 3.x.
if e.errno == errno.ENOENT:
pass
def lchownr(path, owner, group):
@ -1074,7 +1083,7 @@ def install_ca_cert(ca_cert, name=None):
ca_cert = ca_cert.encode('utf8')
if not name:
name = 'juju-{}'.format(charm_name())
cert_file = '/usr/local/share/ca-certificates/{}.crt'.format(name)
cert_file = '{}/{}.crt'.format(CA_CERT_DIR, name)
new_hash = hashlib.md5(ca_cert).hexdigest()
if file_hash(cert_file) == new_hash:
return

3
charmhelpers/core/host_factory/ubuntu.py
View File

@ -25,7 +25,8 @@ UBUNTU_RELEASES = (
'cosmic',
'disco',
'eoan',
'focal'
'focal',
'groovy'
)

2
lib/charms_ceph/broker.py
View File

@ -750,7 +750,7 @@ def handle_create_cephfs(request, service):
"""
cephfs_name = request.get('mds_name')
data_pool = request.get('data_pool')
extra_pools = request.get('extra_pools', [])
extra_pools = request.get('extra_pools', None) or []
metadata_pool = request.get('metadata_pool')
# Check if the user params were provided
if not cephfs_name or not data_pool or not metadata_pool:

25
lib/charms_ceph/utils.py
View File

@ -2141,6 +2141,8 @@ def roll_monitor_cluster(new_version, upgrade_key):
# A sorted list of osd unit names
mon_sorted_list = sorted(monitor_list)
# Install packages immediately but defer restarts to when it's our time.
upgrade_monitor(new_version, restart_daemons=False)
try:
position = mon_sorted_list.index(my_name)
log("upgrade position: {}".format(position))
@ -2182,7 +2184,7 @@ def noop():
pass
def upgrade_monitor(new_version, kick_function=None):
def upgrade_monitor(new_version, kick_function=None, restart_daemons=True):
"""Upgrade the current ceph monitor to the new version
:param new_version: String version to upgrade to.
@ -2207,6 +2209,22 @@ def upgrade_monitor(new_version, kick_function=None):
status_set("blocked", "Upgrade to {} failed".format(new_version))
sys.exit(1)
kick_function()
try:
apt_install(packages=determine_packages(), fatal=True)
rm_packages = determine_packages_to_remove()
if rm_packages:
apt_purge(packages=rm_packages, fatal=True)
except subprocess.CalledProcessError as err:
log("Upgrading packages failed "
"with message: {}".format(err))
status_set("blocked", "Upgrade to {} failed".format(new_version))
sys.exit(1)
if not restart_daemons:
log("Packages upgraded but not restarting daemons yet.")
return
try:
if systemd():
service_stop('ceph-mon')
@ -2216,10 +2234,7 @@ def upgrade_monitor(new_version, kick_function=None):
service_stop('ceph-mgr.target')
else:
service_stop('ceph-mon-all')
apt_install(packages=determine_packages(), fatal=True)
rm_packages = determine_packages_to_remove()
if rm_packages:
apt_purge(packages=rm_packages, fatal=True)
kick_function()
owner = ceph_user()

4
tests/tests.yaml
View File

@ -24,6 +24,8 @@ gate_bundles:
- erasure-coded: focal-ussuri-ec
- focal-victoria
- erasure-coded: focal-victoria-ec
- groovy-victoria
- erasure-coded: groovy-victoria-ec
dev_bundles:
# Icehouse
@ -32,8 +34,6 @@ dev_bundles:
- xenial-ocata
# Pike
- xenial-pike
- groovy-victoria
- erasure-coded: groovy-victoria-ec
smoke_bundles:
- focal-ussuri