diff --git a/charmhelpers/contrib/openstack/context.py b/charmhelpers/contrib/openstack/context.py index a3d48c4..9b80b6d 100644 --- a/charmhelpers/contrib/openstack/context.py +++ b/charmhelpers/contrib/openstack/context.py @@ -1940,7 +1940,7 @@ class VolumeAPIContext(InternalEndpointContext): as well as the catalog_info string that would be supplied. Returns a dict containing the volume_api_version and the volume_catalog_info. """ - rel = os_release(self.pkg, base='icehouse') + rel = os_release(self.pkg) version = '2' if CompareOpenStackReleases(rel) >= 'pike': version = '3' @@ -2140,7 +2140,7 @@ class VersionsContext(OSContextGenerator): self.pkg = pkg def __call__(self): - ostack = os_release(self.pkg, base='icehouse') + ostack = os_release(self.pkg) osystem = lsb_release()['DISTRIB_CODENAME'].lower() return { 'openstack_release': ostack, diff --git a/charmhelpers/contrib/openstack/policyd.py b/charmhelpers/contrib/openstack/policyd.py index 1adf247..83ca4ab 100644 --- a/charmhelpers/contrib/openstack/policyd.py +++ b/charmhelpers/contrib/openstack/policyd.py @@ -17,6 +17,7 @@ import contextlib import os import six import shutil +import sys import yaml import zipfile @@ -115,8 +116,8 @@ library for further details). default: False description: | If True then use the resource file named 'policyd-override' to install - override yaml files in the service's policy.d directory. The resource - file should be a zip file containing at least one yaml file with a .yaml + override YAML files in the service's policy.d directory. The resource + file should be a ZIP file containing at least one yaml file with a .yaml or .yml extension. If False then remove the overrides. """ @@ -134,14 +135,14 @@ resources: Policy Overrides ---------------- -This service allows for policy overrides using the `policy.d` directory. This -is an **advanced** feature and the policies that the service supports should be -clearly and unambiguously understood before trying to override, or add to, the -default policies that the service uses. +This feature allows for policy overrides using the `policy.d` directory. This +is an **advanced** feature and the policies that the OpenStack service supports +should be clearly and unambiguously understood before trying to override, or +add to, the default policies that the service uses. The charm also has some +policy defaults. They should also be understood before being overridden. -The charm also has some policy defaults. They should also be understood before -being overridden. It is possible to break the system (for tenants and other -services) if policies are incorrectly applied to the service. +> **Caution**: It is possible to break the system (for tenants and other + services) if policies are incorrectly applied to the service. Policy overrides are YAML files that contain rules that will add to, or override, existing policy rules in the service. The `policy.d` directory is @@ -149,30 +150,16 @@ a place to put the YAML override files. This charm owns the `/etc/keystone/policy.d` directory, and as such, any manual changes to it will be overwritten on charm upgrades. -Policy overrides are provided to the charm using a resource file called -`policyd-override`. This is attached to the charm using (for example): +Overrides are provided to the charm using a Juju resource called +`policyd-override`. The resource is a ZIP file. This file, say +`overrides.zip`, is attached to the charm by: - juju attach-resource policyd-override= -The `` is the name that this charm is deployed as, with -`` being the resource file containing the policy overrides. + juju attach-resource policyd-override=overrides.zip -The format of the resource file is a ZIP file (.zip extension) containing at -least one YAML file with an extension of `.yaml` or `.yml`. Note that any -directories in the ZIP file are ignored; all of the files are flattened into a -single directory. There must not be any duplicated filenames; this will cause -an error and nothing in the resource file will be applied. +The policy override is enabled in the charm using: -(ed. next part is optional is the charm supports some form of -template/substitution on a read file) - -If a (ed. "one or more of") [`.j2`, `.tmpl`, `.tpl`] file is found in the -resource file then the charm will perform a substitution with charm variables -taken from the config or relations. (ed. edit as appropriate to include the -variable). - -To enable the policy overrides the config option `use-policyd-override` must be -set to `True`. + juju config use-policyd-override=true When `use-policyd-override` is `True` the status line of the charm will be prefixed with `PO:` indicating that policies have been overridden. If the @@ -180,12 +167,8 @@ installation of the policy override YAML files failed for any reason then the status line will be prefixed with `PO (broken):`. The log file for the charm will indicate the reason. No policy override files are installed if the `PO (broken):` is shown. The status line indicates that the overrides are broken, -not that the policy for the service has failed - they will be the defaults for -the charm and service. - -If the policy overrides did not install then *either* attach a new, corrected, -resource file *or* disable the policy overrides by setting -`use-policyd-override` to False. +not that the policy for the service has failed. The policy will be the defaults +for the charm and service. Policy overrides on one service may affect the functionality of another service. Therefore, it may be necessary to provide policy overrides for @@ -296,15 +279,28 @@ def maybe_do_policyd_overrides(openstack_release, restarted. :type restart_handler: Union[None, Callable[]] """ + hookenv.log("Running maybe_do_policyd_overrides", + level=POLICYD_LOG_LEVEL_DEFAULT) + if not is_policyd_override_valid_on_this_release(openstack_release): + hookenv.log("... policy overrides not valid on this release: {}" + .format(openstack_release), + level=POLICYD_LOG_LEVEL_DEFAULT) + return config = hookenv.config() try: if not config.get(POLICYD_CONFIG_NAME, False): - remove_policy_success_file() clean_policyd_dir_for(service, blacklist_paths) + if (os.path.isfile(_policy_success_file()) and + restart_handler is not None and + callable(restart_handler)): + restart_handler() + remove_policy_success_file() return - except Exception: - return - if not is_policyd_override_valid_on_this_release(openstack_release): + except Exception as e: + hookenv.log("... ERROR: Exception is: {}".format(str(e)), + level=POLICYD_CONFIG_NAME) + import traceback + hookenv.log(traceback.format_exc(), level=POLICYD_LOG_LEVEL_DEFAULT) return # from now on it should succeed; if it doesn't then status line will show # broken. @@ -345,16 +341,30 @@ def maybe_do_policyd_overrides_on_config_changed(openstack_release, restarted. :type restart_handler: Union[None, Callable[]] """ + if not is_policyd_override_valid_on_this_release(openstack_release): + return + hookenv.log("Running maybe_do_policyd_overrides_on_config_changed", + level=POLICYD_LOG_LEVEL_DEFAULT) config = hookenv.config() try: if not config.get(POLICYD_CONFIG_NAME, False): - remove_policy_success_file() clean_policyd_dir_for(service, blacklist_paths) + if (os.path.isfile(_policy_success_file()) and + restart_handler is not None and + callable(restart_handler)): + restart_handler() + remove_policy_success_file() return - except Exception: + except Exception as e: + hookenv.log("... ERROR: Exception is: {}".format(str(e)), + level=POLICYD_CONFIG_NAME) + import traceback + hookenv.log(traceback.format_exc(), level=POLICYD_LOG_LEVEL_DEFAULT) return # if the policyd overrides have been performed just return if os.path.isfile(_policy_success_file()): + hookenv.log("... already setup, so skipping.", + level=POLICYD_LOG_LEVEL_DEFAULT) return maybe_do_policyd_overrides( openstack_release, service, blacklist_paths, blacklist_keys, @@ -430,8 +440,13 @@ def _yamlfiles(zipfile): """ l = [] for infolist_item in zipfile.infolist(): - if infolist_item.is_dir(): - continue + try: + if infolist_item.is_dir(): + continue + except AttributeError: + # fallback to "old" way to determine dir entry for pre-py36 + if infolist_item.filename.endswith('/'): + continue _, name_ext = os.path.split(infolist_item.filename) name, ext = os.path.splitext(name_ext) ext = ext.lower() @@ -511,7 +526,7 @@ def clean_policyd_dir_for(service, keep_paths=None): path = policyd_dir_for(service) if not os.path.exists(path): ch_host.mkdir(path, owner=service, group=service, perms=0o775) - _scanner = os.scandir if six.PY3 else _py2_scandir + _scanner = os.scandir if sys.version_info > (3, 4) else _py2_scandir for direntry in _scanner(path): # see if the path should be kept. if direntry.path in keep_paths: @@ -641,6 +656,7 @@ def process_policy_resource_file(resource_file, :returns: True if the processing was successful, False if not. :rtype: boolean """ + hookenv.log("Running process_policy_resource_file", level=hookenv.DEBUG) blacklist_paths = blacklist_paths or [] completed = False try: diff --git a/charmhelpers/contrib/openstack/utils.py b/charmhelpers/contrib/openstack/utils.py index ac96f84..0219026 100644 --- a/charmhelpers/contrib/openstack/utils.py +++ b/charmhelpers/contrib/openstack/utils.py @@ -204,7 +204,7 @@ SWIFT_CODENAMES = OrderedDict([ ('stein', ['2.20.0', '2.21.0']), ('train', - ['2.22.0']), + ['2.22.0', '2.23.0']), ]) # >= Liberty version->codename mapping @@ -531,7 +531,7 @@ def reset_os_release(): _os_rel = None -def os_release(package, base='essex', reset_cache=False): +def os_release(package, base=None, reset_cache=False): ''' Returns OpenStack release codename from a cached global. @@ -542,6 +542,8 @@ def os_release(package, base='essex', reset_cache=False): the installation source, the earliest release supported by the charm should be returned. ''' + if not base: + base = UBUNTU_OPENSTACK_RELEASE[lsb_release()['DISTRIB_CODENAME']] global _os_rel if reset_cache: reset_os_release() @@ -670,7 +672,10 @@ def openstack_upgrade_available(package): codename = get_os_codename_install_source(src) avail_vers = get_os_version_codename_swift(codename) else: - avail_vers = get_os_version_install_source(src) + try: + avail_vers = get_os_version_install_source(src) + except: + avail_vers = cur_vers apt.init() return apt.version_compare(avail_vers, cur_vers) >= 1 @@ -1693,7 +1698,7 @@ def enable_memcache(source=None, release=None, package=None): if release: _release = release else: - _release = os_release(package, base='icehouse') + _release = os_release(package) if not _release: _release = get_os_codename_install_source(source) diff --git a/charmhelpers/contrib/python/__init__.py b/charmhelpers/contrib/python/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/charmhelpers/contrib/storage/linux/ceph.py b/charmhelpers/contrib/storage/linux/ceph.py index e13dfa8..104977a 100644 --- a/charmhelpers/contrib/storage/linux/ceph.py +++ b/charmhelpers/contrib/storage/linux/ceph.py @@ -422,6 +422,8 @@ def enabled_manager_modules(): cmd = ['ceph', 'mgr', 'module', 'ls'] try: modules = check_output(cmd) + if six.PY3: + modules = modules.decode('UTF-8') except CalledProcessError as e: log("Failed to list ceph modules: {}".format(e), WARNING) return [] @@ -1185,6 +1187,15 @@ class CephBrokerRq(object): self.request_id = str(uuid.uuid1()) self.ops = [] + def add_op(self, op): + """Add an op if it is not already in the list. + + :param op: Operation to add. + :type op: dict + """ + if op not in self.ops: + self.ops.append(op) + def add_op_request_access_to_group(self, name, namespace=None, permission=None, key_name=None, object_prefix_permissions=None): @@ -1198,7 +1209,7 @@ class CephBrokerRq(object): 'rwx': ['prefix1', 'prefix2'], 'class-read': ['prefix3']} """ - self.ops.append({ + self.add_op({ 'op': 'add-permissions-to-key', 'group': name, 'namespace': namespace, 'name': key_name or service_name(), @@ -1251,11 +1262,11 @@ class CephBrokerRq(object): if pg_num and weight: raise ValueError('pg_num and weight are mutually exclusive') - self.ops.append({'op': 'create-pool', 'name': name, - 'replicas': replica_count, 'pg_num': pg_num, - 'weight': weight, 'group': group, - 'group-namespace': namespace, 'app-name': app_name, - 'max-bytes': max_bytes, 'max-objects': max_objects}) + self.add_op({'op': 'create-pool', 'name': name, + 'replicas': replica_count, 'pg_num': pg_num, + 'weight': weight, 'group': group, + 'group-namespace': namespace, 'app-name': app_name, + 'max-bytes': max_bytes, 'max-objects': max_objects}) def add_op_create_erasure_pool(self, name, erasure_profile=None, weight=None, group=None, app_name=None, @@ -1283,12 +1294,12 @@ class CephBrokerRq(object): :param max_objects: Maximum objects quota to apply :type max_objects: int """ - self.ops.append({'op': 'create-pool', 'name': name, - 'pool-type': 'erasure', - 'erasure-profile': erasure_profile, - 'weight': weight, - 'group': group, 'app-name': app_name, - 'max-bytes': max_bytes, 'max-objects': max_objects}) + self.add_op({'op': 'create-pool', 'name': name, + 'pool-type': 'erasure', + 'erasure-profile': erasure_profile, + 'weight': weight, + 'group': group, 'app-name': app_name, + 'max-bytes': max_bytes, 'max-objects': max_objects}) def set_ops(self, ops): """Set request ops to provided value. diff --git a/charmhelpers/core/hookenv.py b/charmhelpers/core/hookenv.py index 4744eb4..39b1cd0 100644 --- a/charmhelpers/core/hookenv.py +++ b/charmhelpers/core/hookenv.py @@ -119,6 +119,24 @@ def log(message, level=None): raise +def action_log(message): + """Write an action progress message""" + command = ['action-log'] + if not isinstance(message, six.string_types): + message = repr(message) + command += [message[:SH_MAX_ARG]] + # Missing action-log should not cause failures in unit tests + # Send action_log output to stderr + try: + subprocess.call(command) + except OSError as e: + if e.errno == errno.ENOENT: + message = "action-log: {}".format(message) + print(message, file=sys.stderr) + else: + raise + + class Serializable(UserDict): """Wrapper, an object that can be serialized to yaml or json"""