openstack/charm-nova-compute
Code Issues Proposed changes

Charmhelpers sync to get vaultlocker fixes

Browse Source 
Also gates checking vaultlocker status until it is installed

Change-Id: I07f92132b0340b538ee472887c7fd0e0cc911453
Closes-Bug: #1849323
This commit is contained in:
Edward Hope-Morley 2019-12-10 13:38:51 +00:00
parent 83b2201285
commit 3e67cc5387
11 changed files with 498 additions and 163 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
hooks/charmhelpers/contrib/network/ufw.py
View File

@ -35,8 +35,9 @@ Examples:
>>> ufw.enable()
>>> ufw.service('4949', 'close') # munin
"""
import re
import os
import re
import six
import subprocess
from charmhelpers.core import hookenv
@ -236,29 +237,45 @@ def default_policy(policy='deny', direction='incoming'):
def modify_access(src, dst='any', port=None, proto=None, action='allow',
index=None):
index=None, prepend=False, comment=None):
"""
Grant access to an address or subnet
:param src: address (e.g. 192.168.1.234) or subnet
(e.g. 192.168.1.0/24).
:type src: Optional[str]
:param dst: destiny of the connection, if the machine has multiple IPs and
connections to only one of those have to accepted this is the
field has to be set.
:type dst: Optional[str]
:param port: destiny port
:type port: Optional[int]
:param proto: protocol (tcp or udp)
:type proto: Optional[str]
:param action: `allow` or `delete`
:type action: str
:param index: if different from None the rule is inserted at the given
`index`.
:type index: Optional[int]
:param prepend: Whether to insert the rule before all other rules matching
the rule's IP type.
:type prepend: bool
:param comment: Create the rule with a comment
:type comment: Optional[str]
"""
if not is_enabled():
hookenv.log('ufw is disabled, skipping modify_access()', level='WARN')
return
if action == 'delete':
if index is not None:
cmd = ['ufw', '--force', 'delete', str(index)]
else:
cmd = ['ufw', 'delete', 'allow']
elif index is not None:
cmd = ['ufw', 'insert', str(index), action]
elif prepend:
cmd = ['ufw', 'prepend', action]
else:
cmd = ['ufw', action]
@ -274,6 +291,9 @@ def modify_access(src, dst='any', port=None, proto=None, action='allow',
if proto is not None:
cmd += ['proto', proto]
if comment:
cmd.extend(['comment', comment])
hookenv.log('ufw {}: {}'.format(action, ' '.join(cmd)), level='DEBUG')
p = subprocess.Popen(cmd, stdout=subprocess.PIPE)
(stdout, stderr) = p.communicate()
@ -337,3 +357,33 @@ def service(name, action):
else:
raise UFWError(("'{}' not supported, use 'allow' "
"or 'delete'").format(action))
def status():
"""Retrieve firewall rules as represented by UFW.
:returns: Tuples with rule number and data
(1, {'to': '', 'action':, 'from':, '', ipv6: True, 'comment': ''})
:rtype: Iterator[Tuple[int, Dict[str, Union[bool, str]]]]
"""
if six.PY2:
raise RuntimeError('Call to function not supported on Python2')
cp = subprocess.run(('ufw', 'status', 'numbered',),
stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
check=True, universal_newlines=True)
for line in cp.stdout.splitlines():
if not line.startswith('['):
continue
ipv6 = True if '(v6)' in line else False
line = line.replace('(v6)', '')
line = line.replace('[', '')
line = line.replace(']', '')
line = line.replace('Anywhere', 'any')
row = line.split()
yield (int(row[0]), {
'to': row[1],
'action': ' '.join(row[2:4]).lower(),
'from': row[4],
'ipv6': ipv6,
'comment': row[6] if len(row) > 5 and row[5] == '#' else '',
})

4
hooks/charmhelpers/contrib/openstack/context.py
View File

@ -1940,7 +1940,7 @@ class VolumeAPIContext(InternalEndpointContext):
as well as the catalog_info string that would be supplied. Returns
a dict containing the volume_api_version and the volume_catalog_info.
"""
rel = os_release(self.pkg, base='icehouse')
rel = os_release(self.pkg)
version = '2'
if CompareOpenStackReleases(rel) >= 'pike':
version = '3'
@ -2140,7 +2140,7 @@ class VersionsContext(OSContextGenerator):
self.pkg = pkg
def __call__(self):
ostack = os_release(self.pkg, base='icehouse')
ostack = os_release(self.pkg)
osystem = lsb_release()['DISTRIB_CODENAME'].lower()
return {
'openstack_release': ostack,

3
hooks/charmhelpers/contrib/openstack/ha/utils.py
View File

@ -157,10 +157,11 @@ def generate_ha_relation_data(service,
_relation_data = {'resources': {}, 'resource_params': {}}
if haproxy_enabled:
_meta = 'meta migration-threshold="INFINITY" failure-timeout="5s"'
_haproxy_res = 'res_{}_haproxy'.format(service)
_relation_data['resources'] = {_haproxy_res: 'lsb:haproxy'}
_relation_data['resource_params'] = {
_haproxy_res: 'op monitor interval="5s"'
_haproxy_res: '{} op monitor interval="5s"'.format(_meta)
}
_relation_data['init_services'] = {_haproxy_res: 'haproxy'}
_relation_data['clones'] = {

304
hooks/charmhelpers/contrib/openstack/policyd.py
View File

@ -17,9 +17,11 @@ import contextlib
import os
import six
import shutil
import sys
import yaml
import zipfile
import charmhelpers
import charmhelpers.core.hookenv as hookenv
import charmhelpers.core.host as ch_host
@ -115,8 +117,8 @@ library for further details).
default: False
description: |
If True then use the resource file named 'policyd-override' to install
override yaml files in the service's policy.d directory. The resource
file should be a zip file containing at least one yaml file with a .yaml
override YAML files in the service's policy.d directory. The resource
file should be a ZIP file containing at least one yaml file with a .yaml
or .yml extension. If False then remove the overrides.
"""
@ -134,14 +136,14 @@ resources:
Policy Overrides
----------------
This service allows for policy overrides using the `policy.d` directory. This
is an **advanced** feature and the policies that the service supports should be
clearly and unambiguously understood before trying to override, or add to, the
default policies that the service uses.
This feature allows for policy overrides using the `policy.d` directory. This
is an **advanced** feature and the policies that the OpenStack service supports
should be clearly and unambiguously understood before trying to override, or
add to, the default policies that the service uses. The charm also has some
policy defaults. They should also be understood before being overridden.
The charm also has some policy defaults. They should also be understood before
being overridden. It is possible to break the system (for tenants and other
services) if policies are incorrectly applied to the service.
> **Caution**: It is possible to break the system (for tenants and other
services) if policies are incorrectly applied to the service.
Policy overrides are YAML files that contain rules that will add to, or
override, existing policy rules in the service. The `policy.d` directory is
@ -149,30 +151,16 @@ a place to put the YAML override files. This charm owns the
`/etc/keystone/policy.d` directory, and as such, any manual changes to it will
be overwritten on charm upgrades.
Policy overrides are provided to the charm using a resource file called
`policyd-override`. This is attached to the charm using (for example):
Overrides are provided to the charm using a Juju resource called
`policyd-override`. The resource is a ZIP file. This file, say
`overrides.zip`, is attached to the charm by:
juju attach-resource <charm-name> policyd-override=<some-file>
The `<charm-name>` is the name that this charm is deployed as, with
`<some-file>` being the resource file containing the policy overrides.
juju attach-resource <charm-name> policyd-override=overrides.zip
The format of the resource file is a ZIP file (.zip extension) containing at
least one YAML file with an extension of `.yaml` or `.yml`. Note that any
directories in the ZIP file are ignored; all of the files are flattened into a
single directory. There must not be any duplicated filenames; this will cause
an error and nothing in the resource file will be applied.
The policy override is enabled in the charm using:
(ed. next part is optional is the charm supports some form of
template/substitution on a read file)
If a (ed. "one or more of") [`.j2`, `.tmpl`, `.tpl`] file is found in the
resource file then the charm will perform a substitution with charm variables
taken from the config or relations. (ed. edit as appropriate to include the
variable).
To enable the policy overrides the config option `use-policyd-override` must be
set to `True`.
juju config <charm-name> use-policyd-override=true
When `use-policyd-override` is `True` the status line of the charm will be
prefixed with `PO:` indicating that policies have been overridden. If the
@ -180,12 +168,8 @@ installation of the policy override YAML files failed for any reason then the
status line will be prefixed with `PO (broken):`. The log file for the charm
will indicate the reason. No policy override files are installed if the `PO
(broken):` is shown. The status line indicates that the overrides are broken,
not that the policy for the service has failed - they will be the defaults for
the charm and service.
If the policy overrides did not install then *either* attach a new, corrected,
resource file *or* disable the policy overrides by setting
`use-policyd-override` to False.
not that the policy for the service has failed. The policy will be the defaults
for the charm and service.
Policy overrides on one service may affect the functionality of another
service. Therefore, it may be necessary to provide policy overrides for
@ -251,7 +235,10 @@ def maybe_do_policyd_overrides(openstack_release,
blacklist_paths=None,
blacklist_keys=None,
template_function=None,
restart_handler=None):
restart_handler=None,
user=None,
group=None,
config_changed=False):
"""If the config option is set, get the resource file and process it to
enable the policy.d overrides for the service passed.
@ -280,6 +267,11 @@ def maybe_do_policyd_overrides(openstack_release,
directory. However, for any services where this is buggy then a
restart_handler can be used to force the policy.d files to be read.
If the config_changed param is True, then the handling is slightly
different: It will only perform the policyd overrides if the config is True
and the success file doesn't exist. Otherwise, it does nothing as the
resource file has already been processed.
:param openstack_release: The openstack release that is installed.
:type openstack_release: str
:param service: the service name to construct the policy.d directory for.
@ -295,16 +287,43 @@ def maybe_do_policyd_overrides(openstack_release,
:param restart_handler: The function to call if the service should be
restarted.
:type restart_handler: Union[None, Callable[]]
:param user: The user to create/write files/directories as
:type user: Union[None, str]
:param group: the group to create/write files/directories as
:type group: Union[None, str]
:param config_changed: Set to True for config_changed hook.
:type config_changed: bool
"""
_user = service if user is None else user
_group = service if group is None else group
if not is_policyd_override_valid_on_this_release(openstack_release):
return
hookenv.log("Running maybe_do_policyd_overrides",
level=POLICYD_LOG_LEVEL_DEFAULT)
config = hookenv.config()
try:
if not config.get(POLICYD_CONFIG_NAME, False):
clean_policyd_dir_for(service,
blacklist_paths,
user=_user,
group=_group)
if (os.path.isfile(_policy_success_file())
and restart_handler is not None
and callable(restart_handler)):
restart_handler()
remove_policy_success_file()
clean_policyd_dir_for(service, blacklist_paths)
return
except Exception:
except Exception as e:
hookenv.log("... ERROR: Exception is: {}".format(str(e)),
level=POLICYD_CONFIG_NAME)
import traceback
hookenv.log(traceback.format_exc(), level=POLICYD_LOG_LEVEL_DEFAULT)
return
if not is_policyd_override_valid_on_this_release(openstack_release):
# if the policyd overrides have been performed when doing config_changed
# just return
if config_changed and is_policy_success_file_set():
hookenv.log("... already setup, so skipping.",
level=POLICYD_LOG_LEVEL_DEFAULT)
return
# from now on it should succeed; if it doesn't then status line will show
# broken.
@ -316,49 +335,18 @@ def maybe_do_policyd_overrides(openstack_release,
restart_handler()
def maybe_do_policyd_overrides_on_config_changed(openstack_release,
service,
blacklist_paths=None,
blacklist_keys=None,
template_function=None,
restart_handler=None):
"""This function is designed to be called from the config changed hook
handler. It will only perform the policyd overrides if the config is True
and the success file doesn't exist. Otherwise, it does nothing as the
resource file has already been processed.
@charmhelpers.deprecate("Use maybe_do_poliyd_overrrides instead")
def maybe_do_policyd_overrides_on_config_changed(*args, **kwargs):
"""This function is designed to be called from the config changed hook.
DEPRECATED: please use maybe_do_policyd_overrides() with the param
`config_changed` as `True`.
See maybe_do_policyd_overrides() for more details on the params.
:param openstack_release: The openstack release that is installed.
:type openstack_release: str
:param service: the service name to construct the policy.d directory for.
:type service: str
:param blacklist_paths: optional list of paths to leave alone
:type blacklist_paths: Union[None, List[str]]
:param blacklist_keys: optional list of keys that mustn't appear in the
yaml file's
:type blacklist_keys: Union[None, List[str]]
:param template_function: Optional function that can modify the string
prior to being processed as a Yaml document.
:type template_function: Union[None, Callable[[str], str]]
:param restart_handler: The function to call if the service should be
restarted.
:type restart_handler: Union[None, Callable[]]
"""
config = hookenv.config()
try:
if not config.get(POLICYD_CONFIG_NAME, False):
remove_policy_success_file()
clean_policyd_dir_for(service, blacklist_paths)
return
except Exception:
return
# if the policyd overrides have been performed just return
if os.path.isfile(_policy_success_file()):
return
maybe_do_policyd_overrides(
openstack_release, service, blacklist_paths, blacklist_keys,
template_function, restart_handler)
if 'config_changed' not in kwargs.keys():
kwargs['config_changed'] = True
return maybe_do_policyd_overrides(*args, **kwargs)
def get_policy_resource_filename():
@ -375,13 +363,16 @@ def get_policy_resource_filename():
@contextlib.contextmanager
def open_and_filter_yaml_files(filepath):
def open_and_filter_yaml_files(filepath, has_subdirs=False):
"""Validate that the filepath provided is a zip file and contains at least
one (.yaml|.yml) file, and that the files are not duplicated when the zip
file is flattened. Note that the yaml files are not checked. This is the
first stage in validating the policy zipfile; individual yaml files are not
checked for validity or black listed keys.
If the has_subdirs param is True, then the files are flattened to the first
directory, and the files in the root are ignored.
An example of use is:
with open_and_filter_yaml_files(some_path) as zfp, g:
@ -390,6 +381,8 @@ def open_and_filter_yaml_files(filepath):
:param filepath: a filepath object that can be opened by zipfile
:type filepath: Union[AnyStr, os.PathLike[AntStr]]
:param has_subdirs: Keep first level of subdirectories in yaml file.
:type has_subdirs: bool
:returns: (zfp handle,
a generator of the (name, filename, ZipInfo object) tuples) as a
tuple.
@ -402,7 +395,7 @@ def open_and_filter_yaml_files(filepath):
with zipfile.ZipFile(filepath, 'r') as zfp:
# first pass through; check for duplicates and at least one yaml file.
names = collections.defaultdict(int)
yamlfiles = _yamlfiles(zfp)
yamlfiles = _yamlfiles(zfp, has_subdirs)
for name, _, _, _ in yamlfiles:
names[name] += 1
# There must be at least 1 yaml file.
@ -418,26 +411,49 @@ def open_and_filter_yaml_files(filepath):
yield (zfp, yamlfiles)
def _yamlfiles(zipfile):
def _yamlfiles(zipfile, has_subdirs=False):
"""Helper to get a yaml file (according to POLICYD_VALID_EXTS extensions)
and the infolist item from a zipfile.
If the `has_subdirs` param is True, the the only yaml files that have a
directory component are read, and then first part of the directory
component is kept, along with the filename in the name. e.g. an entry with
a filename of:
compute/someotherdir/override.yaml
is returned as:
compute/override, yaml, override.yaml, <ZipInfo object>
This is to help with the special, additional, processing that the dashboard
charm requires.
:param zipfile: the zipfile to read zipinfo items from
:type zipfile: zipfile.ZipFile
:returns: generator of (name, ext, filename, info item) for each self-identified
yaml file.
:param has_subdirs: Keep first level of subdirectories in yaml file.
:type has_subdirs: bool
:returns: generator of (name, ext, filename, info item) for each
self-identified yaml file.
:rtype: List[(str, str, str, zipfile.ZipInfo)]
"""
l = []
files = []
for infolist_item in zipfile.infolist():
try:
if infolist_item.is_dir():
continue
_, name_ext = os.path.split(infolist_item.filename)
except AttributeError:
# fallback to "old" way to determine dir entry for pre-py36
if infolist_item.filename.endswith('/'):
continue
_dir, name_ext = os.path.split(infolist_item.filename)
name, ext = os.path.splitext(name_ext)
if has_subdirs and _dir != "":
name = os.path.join(_dir.split(os.path.sep)[0], name)
ext = ext.lower()
if ext and ext in POLICYD_VALID_EXTS:
l.append((name, ext, name_ext, infolist_item))
return l
files.append((name, ext, name_ext, infolist_item))
return files
def read_and_validate_yaml(stream_or_doc, blacklist_keys=None):
@ -483,9 +499,6 @@ def read_and_validate_yaml(stream_or_doc, blacklist_keys=None):
def policyd_dir_for(service):
"""Return the policy directory for the named service.
This assumes the default name of "policy.d" which is kept across all
charms.
:param service: str
:returns: the policy.d override directory.
:rtype: os.PathLike[str]
@ -493,7 +506,7 @@ def policyd_dir_for(service):
return os.path.join("/", "etc", service, "policy.d")
def clean_policyd_dir_for(service, keep_paths=None):
def clean_policyd_dir_for(service, keep_paths=None, user=None, group=None):
"""Clean out the policyd directory except for items that should be kept.
The keep_paths, if used, should be set to the full path of the files that
@ -506,12 +519,19 @@ def clean_policyd_dir_for(service, keep_paths=None):
:type service: str
:param keep_paths: optional list of paths to not delete.
:type keep_paths: Union[None, List[str]]
:param user: The user to create/write files/directories as
:type user: Union[None, str]
:param group: the group to create/write files/directories as
:type group: Union[None, str]
"""
_user = service if user is None else user
_group = service if group is None else group
keep_paths = keep_paths or []
path = policyd_dir_for(service)
hookenv.log("Cleaning path: {}".format(path), level=hookenv.DEBUG)
if not os.path.exists(path):
ch_host.mkdir(path, owner=service, group=service, perms=0o775)
_scanner = os.scandir if six.PY3 else _py2_scandir
ch_host.mkdir(path, owner=_user, group=_group, perms=0o775)
_scanner = os.scandir if sys.version_info > (3, 4) else _py2_scandir
for direntry in _scanner(path):
# see if the path should be kept.
if direntry.path in keep_paths:
@ -523,6 +543,22 @@ def clean_policyd_dir_for(service, keep_paths=None):
os.remove(direntry.path)
def maybe_create_directory_for(path, user, group):
"""For the filename 'path', ensure that the directory for that path exists.
Note that if the directory already exists then the permissions are NOT
changed.
:param path: the filename including the path to it.
:type path: str
:param user: the user to create the directory as
:param group: the group to create the directory as
"""
_dir, _ = os.path.split(path)
if not os.path.exists(_dir):
ch_host.mkdir(_dir, owner=user, group=group, perms=0o775)
@contextlib.contextmanager
def _py2_scandir(path):
"""provide a py2 implementation of os.scandir if this module ever gets used
@ -558,6 +594,11 @@ def path_for_policy_file(service, name):
It is constructed using policyd_dir_for(), the name and the ".yaml"
extension.
For horizon, for example, it's a bit more complicated. The name param is
actually "override_service_dir/a_name", where target_service needs to be
one the allowed horizon override services. This translation and check is
done in the _yamlfiles() function.
:param service: the service name
:type service: str
:param name: the name for the policy override
@ -585,6 +626,22 @@ def remove_policy_success_file():
pass
def set_policy_success_file():
"""Set the file that indicates successful policyd override."""
open(_policy_success_file(), "w").close()
def is_policy_success_file_set():
"""Returns True if the policy success file has been set.
This indicates that policies are overridden and working properly.
:returns: True if the policy file is set
:rtype: bool
"""
return os.path.isfile(_policy_success_file())
def policyd_status_message_prefix():
"""Return the prefix str for the status line.
@ -594,7 +651,7 @@ def policyd_status_message_prefix():
:returns: the prefix
:rtype: str
"""
if os.path.isfile(_policy_success_file()):
if is_policy_success_file_set():
return "PO:"
return "PO (broken):"
@ -603,7 +660,11 @@ def process_policy_resource_file(resource_file,
service,
blacklist_paths=None,
blacklist_keys=None,
template_function=None):
template_function=None,
preserve_topdir=False,
preprocess_filename=None,
user=None,
group=None):
"""Process the resource file (which should contain at least one yaml file)
and write those files to the service's policy.d directory.
@ -623,6 +684,16 @@ def process_policy_resource_file(resource_file,
its file path reconstructed. This, also, must not match any path in the
black list.
The yaml filename can be modified in two ways. If the `preserve_topdir`
param is True, then files will be flattened to the top dir. This allows
for creating sets of files that can be grouped into a single level tree
structure.
Secondly, if the `preprocess_filename` param is not None and callable()
then the name is passed to that function for preprocessing before being
converted to the end location. This is to allow munging of the filename
prior to being tested for a blacklist path.
If any error occurs, then the policy.d directory is cleared, the error is
written to the log, and the status line will eventually show as failed.
@ -638,17 +709,39 @@ def process_policy_resource_file(resource_file,
:param template_function: Optional function that can modify the yaml
document.
:type template_function: Union[None, Callable[[AnyStr], AnyStr]]
:param preserve_topdir: Keep the toplevel subdir
:type preserve_topdir: bool
:param preprocess_filename: Optional function to use to process filenames
extracted from the resource file.
:type preprocess_filename: Union[None, Callable[[AnyStr]. AnyStr]]
:param user: The user to create/write files/directories as
:type user: Union[None, str]
:param group: the group to create/write files/directories as
:type group: Union[None, str]
:returns: True if the processing was successful, False if not.
:rtype: boolean
"""
hookenv.log("Running process_policy_resource_file", level=hookenv.DEBUG)
blacklist_paths = blacklist_paths or []
completed = False
_preprocess = None
if preprocess_filename is not None and callable(preprocess_filename):
_preprocess = preprocess_filename
_user = service if user is None else user
_group = service if group is None else group
try:
with open_and_filter_yaml_files(resource_file) as (zfp, gen):
with open_and_filter_yaml_files(
resource_file, preserve_topdir) as (zfp, gen):
# first clear out the policy.d directory and clear success
remove_policy_success_file()
clean_policyd_dir_for(service, blacklist_paths)
clean_policyd_dir_for(service,
blacklist_paths,
user=_user,
group=_group)
for name, ext, filename, zipinfo in gen:
# See if the name should be preprocessed.
if _preprocess is not None:
name = _preprocess(name)
# construct a name for the output file.
yaml_filename = path_for_policy_file(service, name)
if yaml_filename in blacklist_paths:
@ -666,8 +759,12 @@ def process_policy_resource_file(resource_file,
"available".format(filename))
doc = template_function(doc)
yaml_doc = read_and_validate_yaml(doc, blacklist_keys)
with open(yaml_filename, "wt") as f:
yaml.dump(yaml_doc, f)
# we may have to create the directory
maybe_create_directory_for(yaml_filename, _user, _group)
ch_host.write_file(yaml_filename,
yaml.dump(yaml_doc).encode('utf-8'),
_user,
_group)
# Every thing worked, so we mark up a success.
completed = True
except (BadZipFile, BadPolicyZipFile, BadPolicyYamlFile) as e:
@ -691,10 +788,13 @@ def process_policy_resource_file(resource_file,
hookenv.log("Processing {} failed: cleaning policy.d directory"
.format(resource_file),
level=POLICYD_LOG_LEVEL_DEFAULT)
clean_policyd_dir_for(service, blacklist_paths)
clean_policyd_dir_for(service,
blacklist_paths,
user=_user,
group=_group)
else:
# touch the success filename
hookenv.log("policy.d overrides installed.",
level=POLICYD_LOG_LEVEL_DEFAULT)
open(_policy_success_file(), "w").close()
set_policy_success_file()
return completed

11
hooks/charmhelpers/contrib/openstack/utils.py
View File

@ -204,7 +204,7 @@ SWIFT_CODENAMES = OrderedDict([
('stein',
['2.20.0', '2.21.0']),
('train',
['2.22.0']),
['2.22.0', '2.23.0']),
])
# >= Liberty version->codename mapping
@ -531,7 +531,7 @@ def reset_os_release():
_os_rel = None
def os_release(package, base='essex', reset_cache=False):
def os_release(package, base=None, reset_cache=False):
'''
Returns OpenStack release codename from a cached global.
@ -542,6 +542,8 @@ def os_release(package, base='essex', reset_cache=False):
the installation source, the earliest release supported by the charm should
be returned.
'''
if not base:
base = UBUNTU_OPENSTACK_RELEASE[lsb_release()['DISTRIB_CODENAME']]
global _os_rel
if reset_cache:
reset_os_release()
@ -670,7 +672,10 @@ def openstack_upgrade_available(package):
codename = get_os_codename_install_source(src)
avail_vers = get_os_version_codename_swift(codename)
else:
try:
avail_vers = get_os_version_install_source(src)
except:
avail_vers = cur_vers
apt.init()
return apt.version_compare(avail_vers, cur_vers) >= 1
@ -1693,7 +1698,7 @@ def enable_memcache(source=None, release=None, package=None):
if release:
_release = release
else:
_release = os_release(package, base='icehouse')
_release = os_release(package)
if not _release:
_release = get_os_codename_install_source(source)

45
hooks/charmhelpers/contrib/openstack/vaultlocker.py
View File

@ -37,9 +37,13 @@ class VaultKVContext(context.OSContextGenerator):
)
def __call__(self):
import hvac
ctxt = {}
# NOTE(hopem): see https://bugs.launchpad.net/charm-helpers/+bug/1849323
db = unitdata.kv()
last_token = db.get('last-token')
# currently known-good secret-id
secret_id = db.get('secret-id')
for relation_id in hookenv.relation_ids(self.interfaces[0]):
for unit in hookenv.related_units(relation_id):
data = hookenv.relation_get(unit=unit,
@ -54,27 +58,48 @@ class VaultKVContext(context.OSContextGenerator):
# Tokens may change when secret_id's are being
# reissued - if so use token to get new secret_id
if token != last_token:
token_success = False
try:
secret_id = retrieve_secret_id(
url=vault_url,
token=token
)
token_success = True
except hvac.exceptions.InvalidRequest:
# Try next
pass
if token_success:
db.set('secret-id', secret_id)
db.set('last-token', token)
db.flush()
ctxt = {
'vault_url': vault_url,
'role_id': json.loads(role_id),
'secret_id': secret_id,
'secret_backend': self.secret_backend,
}
ctxt['vault_url'] = vault_url
ctxt['role_id'] = json.loads(role_id)
ctxt['secret_id'] = secret_id
ctxt['secret_backend'] = self.secret_backend
vault_ca = data.get('vault_ca')
if vault_ca:
ctxt['vault_ca'] = json.loads(vault_ca)
self.complete = True
break
else:
if secret_id:
ctxt['vault_url'] = vault_url
ctxt['role_id'] = json.loads(role_id)
ctxt['secret_id'] = secret_id
ctxt['secret_backend'] = self.secret_backend
vault_ca = data.get('vault_ca')
if vault_ca:
ctxt['vault_ca'] = json.loads(vault_ca)
if self.complete:
break
if ctxt:
self.complete = True
return ctxt
return {}
def write_vaultlocker_conf(context, priority=100):

17
hooks/charmhelpers/contrib/storage/linux/ceph.py
View File

@ -422,6 +422,8 @@ def enabled_manager_modules():
cmd = ['ceph', 'mgr', 'module', 'ls']
try:
modules = check_output(cmd)
if six.PY3:
modules = modules.decode('UTF-8')
except CalledProcessError as e:
log("Failed to list ceph modules: {}".format(e), WARNING)
return []
@ -1185,6 +1187,15 @@ class CephBrokerRq(object):
self.request_id = str(uuid.uuid1())
self.ops = []
def add_op(self, op):
"""Add an op if it is not already in the list.
:param op: Operation to add.
:type op: dict
"""
if op not in self.ops:
self.ops.append(op)
def add_op_request_access_to_group(self, name, namespace=None,
permission=None, key_name=None,
object_prefix_permissions=None):
@ -1198,7 +1209,7 @@ class CephBrokerRq(object):
'rwx': ['prefix1', 'prefix2'],
'class-read': ['prefix3']}
"""
self.ops.append({
self.add_op({
'op': 'add-permissions-to-key', 'group': name,
'namespace': namespace,
'name': key_name or service_name(),
@ -1251,7 +1262,7 @@ class CephBrokerRq(object):
if pg_num and weight:
raise ValueError('pg_num and weight are mutually exclusive')
self.ops.append({'op': 'create-pool', 'name': name,
self.add_op({'op': 'create-pool', 'name': name,
'replicas': replica_count, 'pg_num': pg_num,
'weight': weight, 'group': group,
'group-namespace': namespace, 'app-name': app_name,
@ -1283,7 +1294,7 @@ class CephBrokerRq(object):
:param max_objects: Maximum objects quota to apply
:type max_objects: int
"""
self.ops.append({'op': 'create-pool', 'name': name,
self.add_op({'op': 'create-pool', 'name': name,
'pool-type': 'erasure',
'erasure-profile': erasure_profile,
'weight': weight,

111
hooks/charmhelpers/core/hookenv.py
View File

@ -34,6 +34,8 @@ import errno
import tempfile
from subprocess import CalledProcessError
from charmhelpers import deprecate
import six
if not six.PY3:
from UserDict import UserDict
@ -119,6 +121,24 @@ def log(message, level=None):
raise
def function_log(message):
"""Write a function progress message"""
command = ['function-log']
if not isinstance(message, six.string_types):
message = repr(message)
command += [message[:SH_MAX_ARG]]
# Missing function-log should not cause failures in unit tests
# Send function_log output to stderr
try:
subprocess.call(command)
except OSError as e:
if e.errno == errno.ENOENT:
message = "function-log: {}".format(message)
print(message, file=sys.stderr)
else:
raise
class Serializable(UserDict):
"""Wrapper, an object that can be serialized to yaml or json"""
@ -946,9 +966,23 @@ def charm_dir():
return os.environ.get('CHARM_DIR')
def cmd_exists(cmd):
"""Return True if the specified cmd exists in the path"""
return any(
os.access(os.path.join(path, cmd), os.X_OK)
for path in os.environ["PATH"].split(os.pathsep)
)
@cached
@deprecate("moved to function_get()", log=log)
def action_get(key=None):
"""Gets the value of an action parameter, or all key/value param pairs"""
"""
.. deprecated:: 0.20.7
Alias for :func:`function_get`.
Gets the value of an action parameter, or all key/value param pairs.
"""
cmd = ['action-get']
if key is not None:
cmd.append(key)
@ -957,36 +991,103 @@ def action_get(key=None):
return action_data
@cached
def function_get(key=None):
"""Gets the value of an action parameter, or all key/value param pairs"""
cmd = ['function-get']
# Fallback for older charms.
if not cmd_exists('function-get'):
cmd = ['action-get']
if key is not None:
cmd.append(key)
cmd.append('--format=json')
function_data = json.loads(subprocess.check_output(cmd).decode('UTF-8'))
return function_data
@deprecate("moved to function_set()", log=log)
def action_set(values):
"""Sets the values to be returned after the action finishes"""
"""
.. deprecated:: 0.20.7
Alias for :func:`function_set`.
Sets the values to be returned after the action finishes.
"""
cmd = ['action-set']
for k, v in list(values.items()):
cmd.append('{}={}'.format(k, v))
subprocess.check_call(cmd)
def action_fail(message):
"""Sets the action status to failed and sets the error message.
def function_set(values):
"""Sets the values to be returned after the function finishes"""
cmd = ['function-set']
# Fallback for older charms.
if not cmd_exists('function-get'):
cmd = ['action-set']
The results set by action_set are preserved."""
for k, v in list(values.items()):
cmd.append('{}={}'.format(k, v))
subprocess.check_call(cmd)
@deprecate("moved to function_fail()", log=log)
def action_fail(message):
"""
.. deprecated:: 0.20.7
Alias for :func:`function_fail`.
Sets the action status to failed and sets the error message.
The results set by action_set are preserved.
"""
subprocess.check_call(['action-fail', message])
def function_fail(message):
"""Sets the function status to failed and sets the error message.
The results set by function_set are preserved."""
cmd = ['function-fail']
# Fallback for older charms.
if not cmd_exists('function-fail'):
cmd = ['action-fail']
cmd.append(message)
subprocess.check_call(cmd)
def action_name():
"""Get the name of the currently executing action."""
return os.environ.get('JUJU_ACTION_NAME')
def function_name():
"""Get the name of the currently executing function."""
return os.environ.get('JUJU_FUNCTION_NAME') or action_name()
def action_uuid():
"""Get the UUID of the currently executing action."""
return os.environ.get('JUJU_ACTION_UUID')
def function_id():
"""Get the ID of the currently executing function."""
return os.environ.get('JUJU_FUNCTION_ID') or action_uuid()
def action_tag():
"""Get the tag for the currently executing action."""
return os.environ.get('JUJU_ACTION_TAG')
def function_tag():
"""Get the tag for the currently executing function."""
return os.environ.get('JUJU_FUNCTION_TAG') or action_tag()
def status_set(workload_state, message):
"""Set the workload state with a message

2
hooks/nova_compute_hooks.py
View File

@ -166,6 +166,8 @@ def install():
db.set('nova-compute-charm-use-fqdn', True)
db.flush()
install_vaultlocker()
@hooks.hook('config-changed')
@restart_on_change(restart_map())

52
hooks/nova_compute_utils.py
View File

@ -36,6 +36,7 @@ from charmhelpers.fetch import (
apt_autoremove,
apt_mark,
filter_missing_packages,
filter_installed_packages,
)
from charmhelpers.core.fstab import Fstab
@ -306,6 +307,10 @@ def libvirt_daemon():
return LIBVIRT_BIN_DAEMON
def vaultlocker_installed():
return len(filter_installed_packages(['vaultlocker'])) == 0
def resource_map():
'''
Dynamically generate a map of resources that will be managed for a single
@ -316,6 +321,18 @@ def resource_map():
resource_map = deepcopy(BASE_RESOURCE_MAP)
else:
resource_map = deepcopy(LIBVIRT_RESOURCE_MAP)
# if vault deps are not installed it is not yet possible to check the vault
# context status since it requires the hvac dependency.
if not vaultlocker_installed():
to_delete = []
for item in resource_map[NOVA_CONF]['contexts']:
if isinstance(item, type(vaultlocker.VaultKVContext())):
to_delete.append(item)
for item in to_delete:
resource_map[NOVA_CONF]['contexts'].remove(item)
net_manager = network_manager()
# Network manager gets set late by the cloud-compute interface.
@ -882,7 +899,17 @@ def assess_status_func(configs, services_=None):
@return f() -> None : a function that assesses the unit's workload status
"""
required_interfaces = REQUIRED_INTERFACES.copy()
required_interfaces.update(get_optional_relations())
optional_relations = get_optional_relations()
if 'vault' in optional_relations:
# skip check if hvac dependency not installed yet
if not vaultlocker_installed():
log("Vault dependencies not yet met so removing from status check")
del optional_relations['vault']
else:
log("Vault dependencies met so including in status check")
required_interfaces.update(optional_relations)
return make_assess_status_func(
configs, required_interfaces,
services=services_ or services(), ports=None)
@ -952,21 +979,28 @@ def determine_block_device():
def configure_local_ephemeral_storage():
"""Configure local block device for use as ephemeral instance storage"""
# Preflight check vault relation if encryption is enabled
encrypt = config('encrypt')
if encrypt:
if not vaultlocker_installed():
log("Encryption requested but vaultlocker not yet installed",
level=DEBUG)
return
vault_kv = vaultlocker.VaultKVContext(
secret_backend=vaultlocker.VAULTLOCKER_BACKEND
)
context = vault_kv()
encrypt = config('encrypt')
if encrypt and not vault_kv.complete:
if vault_kv.complete:
# NOTE: only write vaultlocker configuration once relation is
# complete otherwise we run the chance of an empty
# configuration file being installed on a machine with other
# vaultlocker based services
vaultlocker.write_vaultlocker_conf(context, priority=80)
else:
log("Encryption requested but vault relation not complete",
level=DEBUG)
return
elif encrypt and vault_kv.complete:
# NOTE: only write vaultlocker configuration once relation is complete
# otherwise we run the chance of an empty configuration file
# being installed on a machine with other vaultlocker based
# services
vaultlocker.write_vaultlocker_conf(context, priority=80)
db = kv()
storage_configured = db.get('storage-configured', False)

10
unit_tests/test_nova_compute_utils.py
View File

@ -947,12 +947,15 @@ class NovaComputeUtilsTests(CharmTestCase):
self.config.assert_called_with('ephemeral-device')
self.storage_list.assert_called_with('ephemeral-device')
@patch.object(utils, 'filter_installed_packages')
@patch.object(utils, 'uuid')
@patch.object(utils, 'determine_block_device')
def test_configure_local_ephemeral_storage_encrypted(
self,
determine_block_device,
uuid):
uuid,
filter_installed_packages):
filter_installed_packages.return_value = []
determine_block_device.return_value = '/dev/sdb'
uuid.uuid4.return_value = 'test'
@ -1039,7 +1042,10 @@ class NovaComputeUtilsTests(CharmTestCase):
self.assertTrue(self.test_kv.get('storage-configured'))
self.vaultlocker.write_vaultlocker_conf.assert_not_called()
def test_configure_local_ephemeral_storage_done(self):
@patch.object(utils, 'filter_installed_packages')
def test_configure_local_ephemeral_storage_done(self,
filter_installed_packages):
filter_installed_packages.return_value = []
self.test_kv.set('storage-configured', True)
mock_context = MagicMock()