openstack/charm-keystone
Code Issues Proposed changes

Sync charm-helpers

Browse Source 
Change-Id: Ieb4af8c1d48c009232905506c102f5e0df67c24f
This commit is contained in:
Ryan Beisner 2018-11-07 15:34:41 -06:00
parent 32e4d822c1
commit eb0bbc965d
No known key found for this signature in database
GPG Key ID: 952BACDC1C1A05FB
10 changed files with 117 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
charmhelpers/__init__.py
View File

@ -23,22 +23,22 @@ import subprocess
import sys
try:
import six # flake8: noqa
import six # NOQA:F401
except ImportError:
if sys.version_info.major == 2:
subprocess.check_call(['apt-get', 'install', '-y', 'python-six'])
else:
subprocess.check_call(['apt-get', 'install', '-y', 'python3-six'])
import six # flake8: noqa
import six # NOQA:F401
try:
import yaml # flake8: noqa
import yaml # NOQA:F401
except ImportError:
if sys.version_info.major == 2:
subprocess.check_call(['apt-get', 'install', '-y', 'python-yaml'])
else:
subprocess.check_call(['apt-get', 'install', '-y', 'python3-yaml'])
import yaml # flake8: noqa
import yaml # NOQA:F401
# Holds a list of mapping of mangled function names that have been deprecated

18
charmhelpers/contrib/hardening/harden.py
View File

@ -27,6 +27,8 @@ from charmhelpers.contrib.hardening.ssh.checks import run_ssh_checks
from charmhelpers.contrib.hardening.mysql.checks import run_mysql_checks
from charmhelpers.contrib.hardening.apache.checks import run_apache_checks
_DISABLE_HARDENING_FOR_UNIT_TEST = False
def harden(overrides=None):
"""Hardening decorator.
@ -47,16 +49,28 @@ def harden(overrides=None):
provided with 'harden' config.
:returns: Returns value returned by decorated function once executed.
"""
if overrides is None:
overrides = []
def _harden_inner1(f):
log("Hardening function '%s'" % (f.__name__), level=DEBUG)
# As this has to be py2.7 compat, we can't use nonlocal. Use a trick
# to capture the dictionary that can then be updated.
_logged = {'done': False}
def _harden_inner2(*args, **kwargs):
# knock out hardening via a config var; normally it won't get
# disabled.
if _DISABLE_HARDENING_FOR_UNIT_TEST:
return f(*args, **kwargs)
if not _logged['done']:
log("Hardening function '%s'" % (f.__name__), level=DEBUG)
_logged['done'] = True
RUN_CATALOG = OrderedDict([('os', run_os_checks),
('ssh', run_ssh_checks),
('mysql', run_mysql_checks),
('apache', run_apache_checks)])
enabled = overrides or (config("harden") or "").split()
enabled = overrides[:] or (config("harden") or "").split()
if enabled:
modules_to_run = []
# modules will always be performed in the following order

48
charmhelpers/contrib/openstack/cert_utils.py
View File

@ -25,7 +25,9 @@ from charmhelpers.core.hookenv import (
local_unit,
network_get_primary_address,
config,
related_units,
relation_get,
relation_ids,
unit_get,
NoNetworkBinding,
log,
@ -225,3 +227,49 @@ def process_certificates(service_name, relation_id, unit,
create_ip_cert_links(
ssl_dir,
custom_hostname_link=custom_hostname_link)
def get_requests_for_local_unit(relation_name=None):
"""Extract any certificates data targeted at this unit down relation_name.
:param relation_name: str Name of relation to check for data.
:returns: List of bundles of certificates.
:rtype: List of dicts
"""
local_name = local_unit().replace('/', '_')
raw_certs_key = '{}.processed_requests'.format(local_name)
relation_name = relation_name or 'certificates'
bundles = []
for rid in relation_ids(relation_name):
for unit in related_units(rid):
data = relation_get(rid=rid, unit=unit)
if data.get(raw_certs_key):
bundles.append({
'ca': data['ca'],
'chain': data.get('chain'),
'certs': json.loads(data[raw_certs_key])})
return bundles
def get_bundle_for_cn(cn, relation_name=None):
"""Extract certificates for the given cn.
:param cn: str Canonical Name on certificate.
:param relation_name: str Relation to check for certificates down.
:returns: Dictionary of certificate data,
:rtype: dict.
"""
entries = get_requests_for_local_unit(relation_name)
cert_bundle = {}
for entry in entries:
for _cn, bundle in entry['certs'].items():
if _cn == cn:
cert_bundle = {
'cert': bundle['cert'],
'key': bundle['key'],
'chain': entry['chain'],
'ca': entry['ca']}
break
if cert_bundle:
break
return cert_bundle

2
charmhelpers/contrib/openstack/context.py
View File

@ -642,7 +642,7 @@ class HAProxyContext(OSContextGenerator):
return {}
l_unit = local_unit().replace('/', '-')
cluster_hosts = {}
cluster_hosts = collections.OrderedDict()
# NOTE(jamespage): build out map of configured network endpoints
# and associated backends

27
charmhelpers/contrib/openstack/utils.py
View File

@ -375,7 +375,7 @@ def get_swift_codename(version):
return codenames[0]
# NOTE: fallback - attempt to match with just major.minor version
match = re.match('^(\d+)\.(\d+)', version)
match = re.match(r'^(\d+)\.(\d+)', version)
if match:
major_minor_version = match.group(0)
for codename, versions in six.iteritems(SWIFT_CODENAMES):
@ -395,7 +395,7 @@ def get_os_codename_package(package, fatal=True):
out = subprocess.check_output(cmd)
if six.PY3:
out = out.decode('UTF-8')
except subprocess.CalledProcessError as e:
except subprocess.CalledProcessError:
return None
lines = out.split('\n')
for line in lines:
@ -427,11 +427,11 @@ def get_os_codename_package(package, fatal=True):
vers = apt.upstream_version(pkg.current_ver.ver_str)
if 'swift' in pkg.name:
# Fully x.y.z match for swift versions
match = re.match('^(\d+)\.(\d+)\.(\d+)', vers)
match = re.match(r'^(\d+)\.(\d+)\.(\d+)', vers)
else:
# x.y match only for 20XX.X
# and ignore patch level for other packages
match = re.match('^(\d+)\.(\d+)', vers)
match = re.match(r'^(\d+)\.(\d+)', vers)
if match:
vers = match.group(0)
@ -1450,20 +1450,33 @@ def pausable_restart_on_change(restart_map, stopstart=False,
see core.utils.restart_on_change() for more details.
Note restart_map can be a callable, in which case, restart_map is only
evaluated at runtime. This means that it is lazy and the underlying
function won't be called if the decorated function is never called. Note,
retains backwards compatibility for passing a non-callable dictionary.
@param f: the function to decorate
@param restart_map: the restart map {conf_file: [services]}
@param restart_map: (optionally callable, which then returns the
restart_map) the restart map {conf_file: [services]}
@param stopstart: DEFAULT false; whether to stop, start or just restart
@returns decorator to use a restart_on_change with pausability
"""
def wrap(f):
# py27 compatible nonlocal variable. When py3 only, replace with
# nonlocal keyword
__restart_map_cache = {'cache': None}
@functools.wraps(f)
def wrapped_f(*args, **kwargs):
if is_unit_paused_set():
return f(*args, **kwargs)
if __restart_map_cache['cache'] is None:
__restart_map_cache['cache'] = restart_map() \
if callable(restart_map) else restart_map
# otherwise, normal restart_on_change functionality
return restart_on_change_helper(
(lambda: f(*args, **kwargs)), restart_map, stopstart,
restart_functions)
(lambda: f(*args, **kwargs)), __restart_map_cache['cache'],
stopstart, restart_functions)
return wrapped_f
return wrap

2
charmhelpers/contrib/storage/linux/loopback.py
View File

@ -39,7 +39,7 @@ def loopback_devices():
devs = [d.strip().split(' ') for d in
check_output(cmd).splitlines() if d != '']
for dev, _, f in devs:
loopbacks[dev.replace(':', '')] = re.search('\((\S+)\)', f).groups()[0]
loopbacks[dev.replace(':', '')] = re.search(r'\((\S+)\)', f).groups()[0]
return loopbacks

4
charmhelpers/core/hookenv.py
View File

@ -525,7 +525,7 @@ def expected_peer_units():
This function will raise NotImplementedError if used with juju versions
without goal-state support.
:returns iterator
:returns: iterator
:rtype: types.GeneratorType
:raises: NotImplementedError
"""
@ -550,7 +550,7 @@ def expected_related_units(reltype=None):
support.
Example usage:
log('consumer {} of {} joined relation {}'
log('participant {} of {} joined relation {}'
.format(len(related_units()),
len(list(expected_related_units())),
relation_type()))

29
charmhelpers/core/host.py
View File

@ -40,7 +40,7 @@ from charmhelpers.osplatform import get_platform
__platform__ = get_platform()
if __platform__ == "ubuntu":
from charmhelpers.core.host_factory.ubuntu import (
from charmhelpers.core.host_factory.ubuntu import ( # NOQA:F401
service_available,
add_new_group,
lsb_release,
@ -48,7 +48,7 @@ if __platform__ == "ubuntu":
CompareHostReleases,
) # flake8: noqa -- ignore F401 for this import
elif __platform__ == "centos":
from charmhelpers.core.host_factory.centos import (
from charmhelpers.core.host_factory.centos import ( # NOQA:F401
service_available,
add_new_group,
lsb_release,
@ -58,6 +58,7 @@ elif __platform__ == "centos":
UPDATEDB_PATH = '/etc/updatedb.conf'
def service_start(service_name, **kwargs):
"""Start a system service.
@ -287,8 +288,8 @@ def service_running(service_name, **kwargs):
for key, value in six.iteritems(kwargs):
parameter = '%s=%s' % (key, value)
cmd.append(parameter)
output = subprocess.check_output(cmd,
stderr=subprocess.STDOUT).decode('UTF-8')
output = subprocess.check_output(
cmd, stderr=subprocess.STDOUT).decode('UTF-8')
except subprocess.CalledProcessError:
return False
else:
@ -442,7 +443,7 @@ def add_user_to_group(username, group):
def chage(username, lastday=None, expiredate=None, inactive=None,
mindays=None, maxdays=None, root=None, warndays=None):
mindays=None, maxdays=None, root=None, warndays=None):
"""Change user password expiry information
:param str username: User to update
@ -482,8 +483,10 @@ def chage(username, lastday=None, expiredate=None, inactive=None,
cmd.append(username)
subprocess.check_call(cmd)
remove_password_expiry = functools.partial(chage, expiredate='-1', inactive='-1', mindays='0', maxdays='-1')
def rsync(from_path, to_path, flags='-r', options=None, timeout=None):
"""Replicate the contents of a path"""
options = options or ['--delete', '--executability']
@ -535,13 +538,15 @@ def write_file(path, content, owner='root', group='root', perms=0o444):
# lets see if we can grab the file and compare the context, to avoid doing
# a write.
existing_content = None
existing_uid, existing_gid = None, None
existing_uid, existing_gid, existing_perms = None, None, None
try:
with open(path, 'rb') as target:
existing_content = target.read()
stat = os.stat(path)
existing_uid, existing_gid = stat.st_uid, stat.st_gid
except:
existing_uid, existing_gid, existing_perms = (
stat.st_uid, stat.st_gid, stat.st_mode
)
except Exception:
pass
if content != existing_content:
log("Writing file {} {}:{} {:o}".format(path, owner, group, perms),
@ -554,7 +559,7 @@ def write_file(path, content, owner='root', group='root', perms=0o444):
target.write(content)
return
# the contents were the same, but we might still need to change the
# ownership.
# ownership or permissions.
if existing_uid != uid:
log("Changing uid on already existing content: {} -> {}"
.format(existing_uid, uid), level=DEBUG)
@ -563,6 +568,10 @@ def write_file(path, content, owner='root', group='root', perms=0o444):
log("Changing gid on already existing content: {} -> {}"
.format(existing_gid, gid), level=DEBUG)
os.chown(path, -1, gid)
if existing_perms != perms:
log("Changing permissions on existing content: {} -> {}"
.format(existing_perms, perms), level=DEBUG)
os.chmod(path, perms)
def fstab_remove(mp):
@ -827,7 +836,7 @@ def list_nics(nic_type=None):
ip_output = subprocess.check_output(cmd).decode('UTF-8').split('\n')
ip_output = (line.strip() for line in ip_output if line)
key = re.compile('^[0-9]+:\s+(.+):')
key = re.compile(r'^[0-9]+:\s+(.+):')
for line in ip_output:
matched = re.search(key, line)
if matched:

4
charmhelpers/core/kernel.py
View File

@ -26,12 +26,12 @@ from charmhelpers.core.hookenv import (
__platform__ = get_platform()
if __platform__ == "ubuntu":
from charmhelpers.core.kernel_factory.ubuntu import (
from charmhelpers.core.kernel_factory.ubuntu import ( # NOQA:F401
persistent_modprobe,
update_initramfs,
) # flake8: noqa -- ignore F401 for this import
elif __platform__ == "centos":
from charmhelpers.core.kernel_factory.centos import (
from charmhelpers.core.kernel_factory.centos import ( # NOQA:F401
persistent_modprobe,
update_initramfs,
) # flake8: noqa -- ignore F401 for this import

5
charmhelpers/fetch/ubuntu.py
View File

@ -294,7 +294,7 @@ def apt_unhold(packages, fatal=False):
def import_key(key):
"""Import an ASCII Armor key.
/!\ A Radix64 format keyid is also supported for backwards
A Radix64 format keyid is also supported for backwards
compatibility, but should never be used; the key retrieval
mechanism is insecure and subject to man-in-the-middle attacks
voiding all signature checks using that key.
@ -454,6 +454,9 @@ def _add_apt_repository(spec):
:param spec: the parameter to pass to add_apt_repository
"""
if '{series}' in spec:
series = lsb_release()['DISTRIB_CODENAME']
spec = spec.replace('{series}', series)
_run_with_retries(['add-apt-repository', '--yes', spec])