Merge "21.04 libraries freeze for charms on master branch"

charm-helpers-hooks.yaml

@@ -1,4 +1,4 @@
-repo: https://github.com/juju/charm-helpers
+repo: https://github.com/juju/charm-helpers@stable/21.04
 destination: hooks/charmhelpers
 include:
     - core

hooks/charmhelpers/contrib/openstack/deferred_events.py

@@ -0,0 +1,410 @@
+# Copyright 2021 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Module for managing deferred service events.
+
+This module is used to manage deferred service events from both charm actions
+and package actions.
+"""
+
+import datetime
+import glob
+import yaml
+import os
+import time
+import uuid
+
+import charmhelpers.contrib.openstack.policy_rcd as policy_rcd
+import charmhelpers.core.hookenv as hookenv
+import charmhelpers.core.host as host
+import charmhelpers.core.unitdata as unitdata
+
+import subprocess
+
+
+# Deferred events generated from the charm are stored along side those
+# generated from packaging.
+DEFERRED_EVENTS_DIR = policy_rcd.POLICY_DEFERRED_EVENTS_DIR
+
+
+class ServiceEvent():
+
+    def __init__(self, timestamp, service, reason, action,
+                 policy_requestor_name=None, policy_requestor_type=None):
+        self.timestamp = timestamp
+        self.service = service
+        self.reason = reason
+        self.action = action
+        if not policy_requestor_name:
+            self.policy_requestor_name = hookenv.service_name()
+        if not policy_requestor_type:
+            self.policy_requestor_type = 'charm'
+
+    def __eq__(self, other):
+        for attr in vars(self):
+            if getattr(self, attr) != getattr(other, attr):
+                return False
+        return True
+
+    def matching_request(self, other):
+        for attr in ['service', 'action', 'reason']:
+            if getattr(self, attr) != getattr(other, attr):
+                return False
+        return True
+
+    @classmethod
+    def from_dict(cls, data):
+        return cls(
+            data['timestamp'],
+            data['service'],
+            data['reason'],
+            data['action'],
+            data.get('policy_requestor_name'),
+            data.get('policy_requestor_type'))
+
+
+def deferred_events_files():
+    """Deferred event files
+
+    Deferred event files that were generated by service_name() policy.
+
+    :returns: Deferred event files
+    :rtype: List[str]
+    """
+    return glob.glob('{}/*.deferred'.format(DEFERRED_EVENTS_DIR))
+
+
+def read_event_file(file_name):
+    """Read a file and return the corresponding objects.
+
+    :param file_name: Name of file to read.
+    :type file_name: str
+    :returns: ServiceEvent from file.
+    :rtype: ServiceEvent
+    """
+    with open(file_name, 'r') as f:
+        contents = yaml.safe_load(f)
+    event = ServiceEvent(
+        contents['timestamp'],
+        contents['service'],
+        contents['reason'],
+        contents['action'])
+    return event
+
+
+def deferred_events():
+    """Get list of deferred events.
+
+    List of deferred events. Events are represented by dicts of the form:
+
+       {
+           action: restart,
+           policy_requestor_name: neutron-openvswitch,
+           policy_requestor_type: charm,
+           reason: 'Pkg update',
+           service: openvswitch-switch,
+           time: 1614328743}
+
+    :returns: List of deferred events.
+    :rtype: List[ServiceEvent]
+    """
+    events = []
+    for defer_file in deferred_events_files():
+        events.append((defer_file, read_event_file(defer_file)))
+    return events
+
+
+def duplicate_event_files(event):
+    """Get list of event files that have equivalent deferred events.
+
+    :param event: Event to compare
+    :type event: ServiceEvent
+    :returns: List of event files
+    :rtype: List[str]
+    """
+    duplicates = []
+    for event_file, existing_event in deferred_events():
+        if event.matching_request(existing_event):
+            duplicates.append(event_file)
+    return duplicates
+
+
+def get_event_record_file(policy_requestor_type, policy_requestor_name):
+    """Generate filename for storing a new event.
+
+    :param policy_requestor_type: System that blocked event
+    :type policy_requestor_type: str
+    :param policy_requestor_name: Name of application that blocked event
+    :type policy_requestor_name: str
+    :returns: File name
+    :rtype: str
+    """
+    file_name = '{}/{}-{}-{}.deferred'.format(
+        DEFERRED_EVENTS_DIR,
+        policy_requestor_type,
+        policy_requestor_name,
+        uuid.uuid1())
+    return file_name
+
+
+def save_event(event):
+    """Write deferred events to backend.
+
+    :param event: Event to save
+    :type event: ServiceEvent
+    """
+    requestor_name = hookenv.service_name()
+    requestor_type = 'charm'
+    init_policy_log_dir()
+    if duplicate_event_files(event):
+        hookenv.log(
+            "Not writing new event, existing event found. {} {} {}".format(
+                event.service,
+                event.action,
+                event.reason),
+            level="DEBUG")
+    else:
+        record_file = get_event_record_file(
+            policy_requestor_type=requestor_type,
+            policy_requestor_name=requestor_name)
+
+        with open(record_file, 'w') as f:
+            data = {
+                'timestamp': event.timestamp,
+                'service': event.service,
+                'action': event.action,
+                'reason': event.reason,
+                'policy_requestor_type': requestor_type,
+                'policy_requestor_name': requestor_name}
+            yaml.dump(data, f)
+
+
+def clear_deferred_events(svcs, action):
+    """Remove any outstanding deferred events.
+
+    Remove a deferred event if its service is in the services list and its
+    action matches.
+
+    :param svcs: List of services to remove.
+    :type svcs: List[str]
+    :param action: Action to remove
+    :type action: str
+    """
+    # XXX This function is not currently processing the action. It needs to
+    #     match the action and also take account of try-restart and the
+    #     equivalnce of stop-start and restart.
+    for defer_file in deferred_events_files():
+        deferred_event = read_event_file(defer_file)
+        if deferred_event.service in svcs:
+            os.remove(defer_file)
+
+
+def init_policy_log_dir():
+    """Ensure directory to store events exists."""
+    if not os.path.exists(DEFERRED_EVENTS_DIR):
+        os.mkdir(DEFERRED_EVENTS_DIR)
+
+
+def get_deferred_events():
+    """Return a list of deferred events requested by the charm and packages.
+
+    :returns: List of deferred events
+    :rtype: List[ServiceEvent]
+    """
+    events = []
+    for _, event in deferred_events():
+        events.append(event)
+    return events
+
+
+def get_deferred_restarts():
+    """List of deferred restart events requested by the charm and packages.
+
+    :returns: List of deferred restarts
+    :rtype: List[ServiceEvent]
+    """
+    return [e for e in get_deferred_events() if e.action == 'restart']
+
+
+def clear_deferred_restarts(services):
+    """Clear deferred restart events targetted at `services`.
+
+    :param services: Services with deferred actions to clear.
+    :type services: List[str]
+    """
+    clear_deferred_events(services, 'restart')
+
+
+def process_svc_restart(service):
+    """Respond to a service restart having occured.
+
+    :param service: Services that the action was performed against.
+    :type service: str
+    """
+    clear_deferred_restarts([service])
+
+
+def is_restart_permitted():
+    """Check whether restarts are permitted.
+
+    :returns: Whether restarts are permitted
+    :rtype: bool
+    """
+    if hookenv.config('enable-auto-restarts') is None:
+        return True
+    return hookenv.config('enable-auto-restarts')
+
+
+def check_and_record_restart_request(service, changed_files):
+    """Check if restarts are permitted, if they are not log the request.
+
+    :param service: Service to be restarted
+    :type service: str
+    :param changed_files: Files that have changed to trigger restarts.
+    :type changed_files: List[str]
+    :returns: Whether restarts are permitted
+    :rtype: bool
+    """
+    changed_files = sorted(list(set(changed_files)))
+    permitted = is_restart_permitted()
+    if not permitted:
+        save_event(ServiceEvent(
+            timestamp=round(time.time()),
+            service=service,
+            reason='File(s) changed: {}'.format(
+                ', '.join(changed_files)),
+            action='restart'))
+    return permitted
+
+
+def deferrable_svc_restart(service, reason=None):
+    """Restarts service if permitted, if not defer it.
+
+    :param service: Service to be restarted
+    :type service: str
+    :param reason: Reason for restart
+    :type reason: Union[str, None]
+    """
+    if is_restart_permitted():
+        host.service_restart(service)
+    else:
+        save_event(ServiceEvent(
+            timestamp=round(time.time()),
+            service=service,
+            reason=reason,
+            action='restart'))
+
+
+def configure_deferred_restarts(services):
+    """Setup deferred restarts.
+
+    :param services: Services to block restarts of.
+    :type services: List[str]
+    """
+    policy_rcd.install_policy_rcd()
+    if is_restart_permitted():
+        policy_rcd.remove_policy_file()
+    else:
+        blocked_actions = ['stop', 'restart', 'try-restart']
+        for svc in services:
+            policy_rcd.add_policy_block(svc, blocked_actions)
+
+
+def get_service_start_time(service):
+    """Find point in time when the systemd unit transitioned to active state.
+
+    :param service: Services to check timetsamp of.
+    :type service: str
+    """
+    start_time = None
+    out = subprocess.check_output(
+        [
+            'systemctl',
+            'show',
+            service,
+            '--property=ActiveEnterTimestamp'])
+    str_time = out.decode().rstrip().replace('ActiveEnterTimestamp=', '')
+    if str_time:
+        start_time = datetime.datetime.strptime(
+            str_time,
+            '%a %Y-%m-%d %H:%M:%S %Z')
+    return start_time
+
+
+def check_restart_timestamps():
+    """Check deferred restarts against systemd units start time.
+
+    Check if a service has a deferred event and clear it if it has been
+    subsequently restarted.
+    """
+    for event in get_deferred_restarts():
+        start_time = get_service_start_time(event.service)
+        deferred_restart_time = datetime.datetime.fromtimestamp(
+            event.timestamp)
+        if start_time and start_time < deferred_restart_time:
+            hookenv.log(
+                ("Restart still required, {} was started at {}, restart was "
+                 "requested after that at {}").format(
+                    event.service,
+                    start_time,
+                    deferred_restart_time),
+                level='DEBUG')
+        else:
+            clear_deferred_restarts([event.service])
+
+
+def set_deferred_hook(hookname):
+    """Record that a hook has been deferred.
+
+    :param hookname: Name of hook that was deferred.
+    :type hookname: str
+    """
+    with unitdata.HookData()() as t:
+        kv = t[0]
+        deferred_hooks = kv.get('deferred-hooks', [])
+        if hookname not in deferred_hooks:
+            deferred_hooks.append(hookname)
+            kv.set('deferred-hooks', sorted(list(set(deferred_hooks))))
+
+
+def get_deferred_hooks():
+    """Get a list of deferred hooks.
+
+    :returns: List of hook names.
+    :rtype: List[str]
+    """
+    with unitdata.HookData()() as t:
+        kv = t[0]
+        return kv.get('deferred-hooks', [])
+
+
+def clear_deferred_hooks():
+    """Clear any deferred hooks."""
+    with unitdata.HookData()() as t:
+        kv = t[0]
+        kv.set('deferred-hooks', [])
+
+
+def clear_deferred_hook(hookname):
+    """Clear a specific deferred hooks.
+
+    :param hookname: Name of hook to remove.
+    :type hookname: str
+    """
+    with unitdata.HookData()() as t:
+        kv = t[0]
+        deferred_hooks = kv.get('deferred-hooks', [])
+        if hookname in deferred_hooks:
+            deferred_hooks.remove(hookname)
+            kv.set('deferred-hooks', deferred_hooks)

hooks/charmhelpers/contrib/openstack/exceptions.py

@@ -19,3 +19,8 @@ class OSContextError(Exception):
     This exception is principally used in contrib.openstack.context
     """
     pass
+
+
+class ServiceActionError(Exception):
+    """Raised when a service action (stop/start/ etc) failed."""
+    pass

hooks/charmhelpers/contrib/openstack/files/policy_rc_d_script.py

@@ -0,0 +1,196 @@
+#!/usr/bin/env python3
+
+"""This script is an implemenation of policy-rc.d
+
+For further information on policy-rc.d see *1
+
+*1 https://people.debian.org/~hmh/invokerc.d-policyrc.d-specification.txt
+"""
+import collections
+import glob
+import os
+import logging
+import sys
+import time
+import uuid
+import yaml
+
+
+SystemPolicy = collections.namedtuple(
+    'SystemPolicy',
+    [
+        'policy_requestor_name',
+        'policy_requestor_type',
+        'service',
+        'blocked_actions'])
+
+DEFAULT_POLICY_CONFIG_DIR = '/etc/policy-rc.d'
+DEFAULT_POLICY_LOG_DIR = '/var/lib/policy-rc.d'
+
+
+def read_policy_file(policy_file):
+    """Return system policies from given file.
+
+    :param file_name: Name of file to read.
+    :type file_name: str
+    :returns: Policy
+    :rtype: List[SystemPolicy]
+    """
+    policies = []
+    if os.path.exists(policy_file):
+        with open(policy_file, 'r') as f:
+            policy = yaml.safe_load(f)
+        for service, actions in policy['blocked_actions'].items():
+            service = service.replace('.service', '')
+            policies.append(SystemPolicy(
+                policy_requestor_name=policy['policy_requestor_name'],
+                policy_requestor_type=policy['policy_requestor_type'],
+                service=service,
+                blocked_actions=actions))
+    return policies
+
+
+def get_policies(policy_config_dir):
+    """Return all system policies in policy_config_dir.
+
+    :param policy_config_dir: Name of file to read.
+    :type policy_config_dir: str
+    :returns: Policy
+    :rtype: List[SystemPolicy]
+    """
+    _policy = []
+    for f in glob.glob('{}/*.policy'.format(policy_config_dir)):
+        _policy.extend(read_policy_file(f))
+    return _policy
+
+
+def record_blocked_action(service, action, blocking_policies, policy_log_dir):
+    """Record that an action was requested but deniedl
+
+    :param service: Service that was blocked
+    :type service: str
+    :param action: Action that was blocked.
+    :type action: str
+    :param blocking_policies: Policies that blocked the action on the service.
+    :type blocking_policies: List[SystemPolicy]
+    :param policy_log_dir: Directory to place the blocking action record.
+    :type policy_log_dir: str
+    """
+    if not os.path.exists(policy_log_dir):
+        os.mkdir(policy_log_dir)
+    seconds = round(time.time())
+    for policy in blocking_policies:
+        if not os.path.exists(policy_log_dir):
+            os.mkdir(policy_log_dir)
+        file_name = '{}/{}-{}-{}.deferred'.format(
+            policy_log_dir,
+            policy.policy_requestor_type,
+            policy.policy_requestor_name,
+            uuid.uuid1())
+        with open(file_name, 'w') as f:
+            data = {
+                'timestamp': seconds,
+                'service': service,
+                'action': action,
+                'reason': 'Package update',
+                'policy_requestor_type': policy.policy_requestor_type,
+                'policy_requestor_name': policy.policy_requestor_name}
+            yaml.dump(data, f)
+
+
+def get_blocking_policies(service, action, policy_config_dir):
+    """Record that an action was requested but deniedl
+
+    :param service: Service that action is requested against.
+    :type service: str
+    :param action: Action that is requested.
+    :type action: str
+    :param policy_config_dir: Directory that stores policy files.
+    :type policy_config_dir: str
+    :returns: Policies
+    :rtype: List[SystemPolicy]
+    """
+    service = service.replace('.service', '')
+    blocking_policies = [
+        policy
+        for policy in get_policies(policy_config_dir)
+        if policy.service == service and action in policy.blocked_actions]
+    return blocking_policies
+
+
+def process_action_request(service, action, policy_config_dir, policy_log_dir):
+    """Take the requested action against service and check if it is permitted.
+
+    :param service: Service that action is requested against.
+    :type service: str
+    :param action: Action that is requested.
+    :type action: str
+    :param policy_config_dir: Directory that stores policy files.
+    :type policy_config_dir: str
+    :param policy_log_dir: Directory that stores policy files.
+    :type policy_log_dir: str
+    :returns: Tuple of whether the action is permitted and explanation.
+    :rtype: (boolean, str)
+    """
+    blocking_policies = get_blocking_policies(
+        service,
+        action,
+        policy_config_dir)
+    if blocking_policies:
+        policy_msg = [
+            '{} {}'.format(p.policy_requestor_type, p.policy_requestor_name)
+            for p in sorted(blocking_policies)]
+        message = '{} of {} blocked by {}'.format(
+            action,
+            service,
+            ', '.join(policy_msg))
+        record_blocked_action(
+            service,
+            action,
+            blocking_policies,
+            policy_log_dir)
+        action_permitted = False
+    else:
+        message = "Permitting {} {}".format(service, action)
+        action_permitted = True
+    return action_permitted, message
+
+
+def main():
+    logging.basicConfig(
+        filename='/var/log/policy-rc.d.log',
+        level=logging.DEBUG,
+        format='%(asctime)s %(message)s')
+
+    service = sys.argv[1]
+    action = sys.argv[2]
+
+    permitted, message = process_action_request(
+        service,
+        action,
+        DEFAULT_POLICY_CONFIG_DIR,
+        DEFAULT_POLICY_LOG_DIR)
+    logging.info(message)
+
+    # https://people.debian.org/~hmh/invokerc.d-policyrc.d-specification.txt
+    # Exit status codes:
+    #  0 - action allowed
+    #  1 - unknown action (therefore, undefined policy)
+    # 100 - unknown initscript id
+    # 101 - action forbidden by policy
+    # 102 - subsystem error
+    # 103 - syntax error
+    # 104 - [reserved]
+    # 105 - behaviour uncertain, policy undefined.
+    # 106 - action not allowed. Use the returned fallback actions
+    #       (which are implied to be "allowed") instead.
+
+    if permitted:
+        return 0
+    else:
+        return 101
+
+
+if __name__ == "__main__":
+    rc = main()
+    sys.exit(rc)

hooks/charmhelpers/contrib/openstack/policy_rcd.py

@@ -0,0 +1,173 @@
+# Copyright 2021 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Module for managing policy-rc.d script and associated files.
+
+This module manages the installation of /usr/sbin/policy-rc.d, the
+policy files and the event files. When a package update occurs the
+packaging system calls:
+
+policy-rc.d [options] <initscript ID> <actions>
+
+The return code of the script determines if the packaging system
+will perform that action on the given service. The policy-rc.d
+implementation installed by this module checks if an action is
+permitted by checking policy files placed in /etc/policy-rc.d.
+If a policy file exists which denies the requested action then
+this is recorded in an event file which is placed in
+/var/lib/policy-rc.d.
+"""
+
+import os
+import shutil
+import tempfile
+import yaml
+
+import charmhelpers.contrib.openstack.files as os_files
+import charmhelpers.contrib.openstack.alternatives as alternatives
+import charmhelpers.core.hookenv as hookenv
+import charmhelpers.core.host as host
+
+POLICY_HEADER = """# Managed by juju\n"""
+POLICY_DEFERRED_EVENTS_DIR = '/var/lib/policy-rc.d'
+POLICY_CONFIG_DIR = '/etc/policy-rc.d'
+
+
+def get_policy_file_name():
+    """Get the name of the policy file for this application.
+
+    :returns: Policy file name
+    :rtype: str
+    """
+    application_name = hookenv.service_name()
+    return '{}/charm-{}.policy'.format(POLICY_CONFIG_DIR, application_name)
+
+
+def read_default_policy_file():
+    """Return the policy file.
+
+    A policy is in the form:
+        blocked_actions:
+            neutron-dhcp-agent: [restart, stop, try-restart]
+            neutron-l3-agent: [restart, stop, try-restart]
+            neutron-metadata-agent: [restart, stop, try-restart]
+            neutron-openvswitch-agent: [restart, stop, try-restart]
+            openvswitch-switch: [restart, stop, try-restart]
+            ovs-vswitchd: [restart, stop, try-restart]
+            ovs-vswitchd-dpdk: [restart, stop, try-restart]
+            ovsdb-server: [restart, stop, try-restart]
+        policy_requestor_name: neutron-openvswitch
+        policy_requestor_type: charm
+
+    :returns: Policy
+    :rtype: Dict[str, Union[str, Dict[str, List[str]]]
+    """
+    policy = {}
+    policy_file = get_policy_file_name()
+    if os.path.exists(policy_file):
+        with open(policy_file, 'r') as f:
+            policy = yaml.safe_load(f)
+    return policy
+
+
+def write_policy_file(policy_file, policy):
+    """Write policy to disk.
+
+    :param policy_file: Name of policy file
+    :type policy_file: str
+    :param policy: Policy
+    :type policy: Dict[str, Union[str, Dict[str, List[str]]]]
+    """
+    with tempfile.NamedTemporaryFile('w', delete=False) as f:
+        f.write(POLICY_HEADER)
+        yaml.dump(policy, f)
+        tmp_file_name = f.name
+    shutil.move(tmp_file_name, policy_file)
+
+
+def remove_policy_file():
+    """Remove policy file."""
+    try:
+        os.remove(get_policy_file_name())
+    except FileNotFoundError:
+        pass
+
+
+def install_policy_rcd():
+    """Install policy-rc.d components."""
+    source_file_dir = os.path.dirname(os.path.abspath(os_files.__file__))
+    policy_rcd_exec = "/var/lib/charm/{}/policy-rc.d".format(
+        hookenv.service_name())
+    host.mkdir(os.path.dirname(policy_rcd_exec))
+    shutil.copy2(
+        '{}/policy_rc_d_script.py'.format(source_file_dir),
+        policy_rcd_exec)
+    # policy-rc.d must be installed via the alternatives system:
+    # https://people.debian.org/~hmh/invokerc.d-policyrc.d-specification.txt
+    if not os.path.exists('/usr/sbin/policy-rc.d'):
+        alternatives.install_alternative(
+            'policy-rc.d',
+            '/usr/sbin/policy-rc.d',
+            policy_rcd_exec)
+    host.mkdir(POLICY_CONFIG_DIR)
+
+
+def get_default_policy():
+    """Return the default policy structure.
+
+    :returns: Policy
+    :rtype: Dict[str, Union[str, Dict[str, List[str]]]
+    """
+    policy = {
+        'policy_requestor_name': hookenv.service_name(),
+        'policy_requestor_type': 'charm',
+        'blocked_actions': {}}
+    return policy
+
+
+def add_policy_block(service, blocked_actions):
+    """Update a policy file with new list of actions.
+
+    :param service: Service name
+    :type service: str
+    :param blocked_actions: Action to block
+    :type blocked_actions: List[str]
+    """
+    policy = read_default_policy_file() or get_default_policy()
+    policy_file = get_policy_file_name()
+    if policy['blocked_actions'].get(service):
+        policy['blocked_actions'][service].extend(blocked_actions)
+    else:
+        policy['blocked_actions'][service] = blocked_actions
+    policy['blocked_actions'][service] = sorted(
+        list(set(policy['blocked_actions'][service])))
+    write_policy_file(policy_file, policy)
+
+
+def remove_policy_block(service, unblocked_actions):
+    """Remove list of actions from policy file.
+
+    :param service: Service name
+    :type service: str
+    :param unblocked_actions: Action to unblock
+    :type unblocked_actions: List[str]
+    """
+    policy_file = get_policy_file_name()
+    policy = read_default_policy_file()
+    for action in unblocked_actions:
+        try:
+            policy['blocked_actions'][service].remove(action)
+        except (KeyError, ValueError):
+            continue
+    write_policy_file(policy_file, policy)

hooks/charmhelpers/contrib/openstack/utils.py

@@ -14,7 +14,7 @@
 
 # Common python helper functions used for OpenStack charms.
 from collections import OrderedDict, namedtuple
-from functools import wraps
+from functools import partial, wraps
 
 import subprocess
 import json
@@ -36,9 +36,12 @@ from charmhelpers.contrib.network import ip
 
 from charmhelpers.core import decorators, unitdata
 
+import charmhelpers.contrib.openstack.deferred_events as deferred_events
+
 from charmhelpers.core.hookenv import (
     WORKLOAD_STATES,
     action_fail,
+    action_get,
     action_set,
     config,
     expected_peer_units,
@@ -112,7 +115,7 @@ from charmhelpers.fetch.snap import (
 
 from charmhelpers.contrib.storage.linux.utils import is_block_device, zap_disk
 from charmhelpers.contrib.storage.linux.loopback import ensure_loopback_device
-from charmhelpers.contrib.openstack.exceptions import OSContextError
+from charmhelpers.contrib.openstack.exceptions import OSContextError, ServiceActionError
 from charmhelpers.contrib.openstack.policyd import (
     policyd_status_message_prefix,
     POLICYD_CONFIG_NAME,
@@ -148,6 +151,7 @@ OPENSTACK_RELEASES = (
     'train',
     'ussuri',
     'victoria',
+    'wallaby',
 )
 
 UBUNTU_OPENSTACK_RELEASE = OrderedDict([
@@ -170,6 +174,7 @@ UBUNTU_OPENSTACK_RELEASE = OrderedDict([
     ('eoan', 'train'),
     ('focal', 'ussuri'),
     ('groovy', 'victoria'),
+    ('hirsute', 'wallaby'),
 ])
 
 
@@ -193,6 +198,7 @@ OPENSTACK_CODENAMES = OrderedDict([
     ('2019.2', 'train'),
     ('2020.1', 'ussuri'),
     ('2020.2', 'victoria'),
+    ('2021.1', 'wallaby'),
 ])
 
 # The ugly duckling - must list releases oldest to newest
@@ -301,8 +307,8 @@ PACKAGE_CODENAMES = {
         ('14', 'rocky'),
         ('15', 'stein'),
         ('16', 'train'),
-        ('18', 'ussuri'),
+        ('18', 'ussuri'),  # Note this was actually 17.0 - 18.3
-        ('19', 'victoria'),
+        ('19', 'victoria'),  # Note this is really 18.6
     ]),
     'ceilometer-common': OrderedDict([
         ('5', 'liberty'),
@@ -1083,6 +1089,18 @@ def _determine_os_workload_status(
     try:
         if config(POLICYD_CONFIG_NAME):
             message = "{} {}".format(policyd_status_message_prefix(), message)
+        deferred_restarts = list(set(
+            [e.service for e in deferred_events.get_deferred_restarts()]))
+        if deferred_restarts:
+            svc_msg = "Services queued for restart: {}".format(
+                ', '.join(sorted(deferred_restarts)))
+            message = "{}. {}".format(message, svc_msg)
+        deferred_hooks = deferred_events.get_deferred_hooks()
+        if deferred_hooks:
+            svc_msg = "Hooks skipped due to disabled auto restarts: {}".format(
+                ', '.join(sorted(deferred_hooks)))
+            message = "{}. {}".format(message, svc_msg)
+
     except Exception:
         pass
 
@@ -1567,6 +1585,33 @@ def is_unit_paused_set():
         return False
 
 
+def is_hook_allowed(hookname, check_deferred_restarts=True):
+    """Check if hook can run.
+
+    :param hookname: Name of hook to check..
+    :type hookname: str
+    :param check_deferred_restarts: Whether to check deferred restarts.
+    :type check_deferred_restarts: bool
+    """
+    permitted = True
+    reasons = []
+    if is_unit_paused_set():
+        reasons.append(
+            "Unit is pause or upgrading. Skipping {}".format(hookname))
+        permitted = False
+
+    if check_deferred_restarts:
+        if deferred_events.is_restart_permitted():
+            permitted = True
+            deferred_events.clear_deferred_hook(hookname)
+        else:
+            if not config().changed('enable-auto-restarts'):
+                deferred_events.set_deferred_hook(hookname)
+            reasons.append("auto restarts are disabled")
+            permitted = False
+    return permitted, " and ".join(reasons)
+
+
 def manage_payload_services(action, services=None, charm_func=None):
     """Run an action against all services.
 
@@ -1727,6 +1772,43 @@ def resume_unit(assess_status_func, services=None, ports=None,
         raise Exception("Couldn't resume: {}".format("; ".join(messages)))
 
 
+def restart_services_action(services=None, when_all_stopped_func=None,
+                            deferred_only=None):
+    """Manage a service restart request via charm action.
+
+    :param services: Services to be restarted
+    :type model_name: List[str]
+    :param when_all_stopped_func: Function to call when all services are
+                                  stopped.
+    :type when_all_stopped_func: Callable[]
+    :param model_name: Only restart services which have a deferred restart
+                       event.
+    :type model_name: bool
+    """
+    if services and deferred_only:
+        raise ValueError(
+            "services and deferred_only are mutually exclusive")
+    if deferred_only:
+        services = list(set(
+            [a.service for a in deferred_events.get_deferred_restarts()]))
+    _, messages = manage_payload_services(
+        'stop',
+        services=services,
+        charm_func=when_all_stopped_func)
+    if messages:
+        raise ServiceActionError(
+            "Error processing service stop request: {}".format(
+                "; ".join(messages)))
+    _, messages = manage_payload_services(
+        'start',
+        services=services)
+    if messages:
+        raise ServiceActionError(
+            "Error processing service start request: {}".format(
+                "; ".join(messages)))
+    deferred_events.clear_deferred_restarts(services)
+
+
 def make_assess_status_func(*args, **kwargs):
     """Creates an assess_status_func() suitable for handing to pause_unit()
     and resume_unit().
@@ -2201,6 +2283,23 @@ def container_scoped_relations():
     return relations
 
 
+def container_scoped_relation_get(attribute=None):
+    """Get relation data from all container scoped relations.
+
+    :param attribute: Name of attribute to get
+    :type attribute: Optional[str]
+    :returns: Iterator with relation data
+    :rtype: Iterator[Optional[any]]
+    """
+    for endpoint_name in container_scoped_relations():
+        for rid in relation_ids(endpoint_name):
+            for unit in related_units(rid):
+                yield relation_get(
+                    attribute=attribute,
+                    unit=unit,
+                    rid=rid)
+
+
 def is_db_ready(use_current_context=False, rel_name=None):
     """Check remote database is ready to be used.
 
@@ -2497,3 +2596,84 @@ def sequence_status_check_functions(*functions):
         return state, message
 
     return _inner_sequenced_functions
+
+
+SubordinatePackages = namedtuple('SubordinatePackages', ['install', 'purge'])
+
+
+def get_subordinate_release_packages(os_release, package_type='deb'):
+    """Iterate over subordinate relations and get package information.
+
+    :param os_release: OpenStack release to look for
+    :type os_release: str
+    :param package_type: Package type (one of 'deb' or 'snap')
+    :type package_type: str
+    :returns: Packages to install and packages to purge or None
+    :rtype: SubordinatePackages[set,set]
+    """
+    install = set()
+    purge = set()
+
+    for rdata in container_scoped_relation_get('releases-packages-map'):
+        rp_map = json.loads(rdata or '{}')
+        # The map provided by subordinate has OpenStack release name as key.
+        # Find package information from subordinate matching requested release
+        # or the most recent release prior to requested release by sorting the
+        # keys in reverse order. This follows established patterns in our
+        # charms for templates and reactive charm implementations, i.e. as long
+        # as nothing has changed the definitions for the prior OpenStack
+        # release is still valid.
+        for release in sorted(rp_map.keys(), reverse=True):
+            if (CompareOpenStackReleases(release) <= os_release and
+                    package_type in rp_map[release]):
+                for name, container in (
+                        ('install', install),
+                        ('purge', purge)):
+                    for pkg in rp_map[release][package_type].get(name, []):
+                        container.add(pkg)
+                break
+    return SubordinatePackages(install, purge)
+
+
+os_restart_on_change = partial(
+    pausable_restart_on_change,
+    can_restart_now_f=deferred_events.check_and_record_restart_request,
+    post_svc_restart_f=deferred_events.process_svc_restart)
+
+
+def restart_services_action_helper(all_services):
+    """Helper to run the restart-services action.
+
+    NOTE: all_services is all services that could be restarted but
+          depending on the action arguments it may be a subset of
+          these that are actually restarted.
+
+    :param all_services: All services that could be restarted
+    :type all_services: List[str]
+    """
+    deferred_only = action_get("deferred-only")
+    services = action_get("services")
+    if services:
+        services = services.split()
+    else:
+        services = all_services
+    if deferred_only:
+        restart_services_action(deferred_only=True)
+    else:
+        restart_services_action(services=services)
+
+
+def show_deferred_events_action_helper():
+    """Helper to run the show-deferred-restarts action."""
+    restarts = []
+    for event in deferred_events.get_deferred_events():
+        restarts.append('{} {} {}'.format(
+            str(event.timestamp),
+            event.service.ljust(40),
+            event.reason))
+    restarts.sort()
+    output = {
+        'restarts': restarts,
+        'hooks': deferred_events.get_deferred_hooks()}
+    action_set({'output': "{}".format(
+        yaml.dump(output, default_flow_style=False))})

hooks/charmhelpers/core/hookenv.py

@@ -1622,3 +1622,12 @@ def _contains_range(addresses):
         addresses.startswith(".") or
         ",." in addresses or
         " ." in addresses)
+
+
+def is_subordinate():
+    """Check whether charm is subordinate in unit metadata.
+
+    :returns: True if unit is subordniate, False otherwise.
+    :rtype: bool
+    """
+    return metadata().get('subordinate') is True

hooks/charmhelpers/core/host.py

@@ -694,39 +694,93 @@ class ChecksumError(ValueError):
     pass
 
 
-def restart_on_change(restart_map, stopstart=False, restart_functions=None):
+class restart_on_change(object):
-    """Restart services based on configuration files changing
+    """Decorator and context manager to handle restarts.
 
-    This function is used a decorator, for example::
+    Usage:
 
-        @restart_on_change({
+       @restart_on_change(restart_map, ...)
-            '/etc/ceph/ceph.conf': [ 'cinder-api', 'cinder-volume' ]
+       def function_that_might_trigger_a_restart(...)
-            '/etc/apache/sites-enabled/*': [ 'apache2' ]
+           ...
-            })
-        def config_changed():
-            pass  # your code here
 
-    In this example, the cinder-api and cinder-volume services
+    Or:
-    would be restarted if /etc/ceph/ceph.conf is changed by the
-    ceph_client_changed function. The apache2 service would be
-    restarted if any file matching the pattern got changed, created
-    or removed. Standard wildcards are supported, see documentation
-    for the 'glob' module for more information.
 
-    @param restart_map: {path_file_name: [service_name, ...]
+       with restart_on_change(restart_map, ...):
-    @param stopstart: DEFAULT false; whether to stop, start OR restart
+           do_stuff_that_might_trigger_a_restart()
-    @param restart_functions: nonstandard functions to use to restart services
+           ...
-                              {svc: func, ...}
-    @returns result from decorated function
     """
-    def wrap(f):
+
+    def __init__(self, restart_map, stopstart=False, restart_functions=None,
+                 can_restart_now_f=None, post_svc_restart_f=None,
+                 pre_restarts_wait_f=None):
+        """
+        :param restart_map: {file: [service, ...]}
+        :type restart_map: Dict[str, List[str,]]
+        :param stopstart: whether to stop, start or restart a service
+        :type stopstart: booleean
+        :param restart_functions: nonstandard functions to use to restart
+                                  services {svc: func, ...}
+        :type restart_functions: Dict[str, Callable[[str], None]]
+        :param can_restart_now_f: A function used to check if the restart is
+                                  permitted.
+        :type can_restart_now_f: Callable[[str, List[str]], boolean]
+        :param post_svc_restart_f: A function run after a service has
+                                   restarted.
+        :type post_svc_restart_f: Callable[[str], None]
+        :param pre_restarts_wait_f: A function callled before any restarts.
+        :type pre_restarts_wait_f: Callable[None, None]
+        """
+        self.restart_map = restart_map
+        self.stopstart = stopstart
+        self.restart_functions = restart_functions
+        self.can_restart_now_f = can_restart_now_f
+        self.post_svc_restart_f = post_svc_restart_f
+        self.pre_restarts_wait_f = pre_restarts_wait_f
+
+    def __call__(self, f):
+        """Work like a decorator.
+
+        Returns a wrapped function that performs the restart if triggered.
+
+        :param f: The function that is being wrapped.
+        :type f: Callable[[Any], Any]
+        :returns: the wrapped function
+        :rtype: Callable[[Any], Any]
+        """
         @functools.wraps(f)
         def wrapped_f(*args, **kwargs):
             return restart_on_change_helper(
-                (lambda: f(*args, **kwargs)), restart_map, stopstart,
+                (lambda: f(*args, **kwargs)),
-                restart_functions)
+                self.restart_map,
+                stopstart=self.stopstart,
+                restart_functions=self.restart_functions,
+                can_restart_now_f=self.can_restart_now_f,
+                post_svc_restart_f=self.post_svc_restart_f,
+                pre_restarts_wait_f=self.pre_restarts_wait_f)
         return wrapped_f
-    return wrap
+
+    def __enter__(self):
+        """Enter the runtime context related to this object. """
+        self.checksums = _pre_restart_on_change_helper(self.restart_map)
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        """Exit the runtime context related to this object.
+
+        The parameters describe the exception that caused the context to be
+        exited. If the context was exited without an exception, all three
+        arguments will be None.
+        """
+        if exc_type is None:
+            _post_restart_on_change_helper(
+                self.checksums,
+                self.restart_map,
+                stopstart=self.stopstart,
+                restart_functions=self.restart_functions,
+                can_restart_now_f=self.can_restart_now_f,
+                post_svc_restart_f=self.post_svc_restart_f,
+                pre_restarts_wait_f=self.pre_restarts_wait_f)
+        # All is good, so return False; any exceptions will propagate.
+        return False
 
 
 def restart_on_change_helper(lambda_f, restart_map, stopstart=False,
@@ -747,8 +801,8 @@ def restart_on_change_helper(lambda_f, restart_map, stopstart=False,
     `my_restart_func` should a function which takes one argument which is the
     service name to be retstarted.
 
-    `can_restart_now_f` is a function which checks that a restart is permitted. It
+    `can_restart_now_f` is a function which checks that a restart is permitted.
-    should returna bool which indicates if a restart is allowed and should
+    It should return a bool which indicates if a restart is allowed and should
     take a service name (str) and a list of changed files (List[str]) as
     arguments.
 
@@ -779,10 +833,58 @@ def restart_on_change_helper(lambda_f, restart_map, stopstart=False,
     :returns: result of lambda_f()
     :rtype: ANY
     """
+    checksums = _pre_restart_on_change_helper(restart_map)
+    r = lambda_f()
+    _post_restart_on_change_helper(checksums,
+                                   restart_map,
+                                   stopstart,
+                                   restart_functions,
+                                   can_restart_now_f,
+                                   post_svc_restart_f,
+                                   pre_restarts_wait_f)
+    return r
+
+
+def _pre_restart_on_change_helper(restart_map):
+    """Take a snapshot of file hashes.
+
+    :param restart_map: {file: [service, ...]}
+    :type restart_map: Dict[str, List[str,]]
+    :returns: Dictionary of file paths and the files checksum.
+    :rtype: Dict[str, str]
+    """
+    return {path: path_hash(path) for path in restart_map}
+
+
+def _post_restart_on_change_helper(checksums,
+                                   restart_map,
+                                   stopstart=False,
+                                   restart_functions=None,
+                                   can_restart_now_f=None,
+                                   post_svc_restart_f=None,
+                                   pre_restarts_wait_f=None):
+    """Check whether files have changed.
+
+    :param checksums: Dictionary of file paths and the files checksum.
+    :type checksums: Dict[str, str]
+    :param restart_map: {file: [service, ...]}
+    :type restart_map: Dict[str, List[str,]]
+    :param stopstart: whether to stop, start or restart a service
+    :type stopstart: booleean
+    :param restart_functions: nonstandard functions to use to restart services
+                              {svc: func, ...}
+    :type restart_functions: Dict[str, Callable[[str], None]]
+    :param can_restart_now_f: A function used to check if the restart is
+                              permitted.
+    :type can_restart_now_f: Callable[[str, List[str]], boolean]
+    :param post_svc_restart_f: A function run after a service has
+                               restarted.
+    :type post_svc_restart_f: Callable[[str], None]
+    :param pre_restarts_wait_f: A function callled before any restarts.
+    :type pre_restarts_wait_f: Callable[None, None]
+    """
     if restart_functions is None:
         restart_functions = {}
-    checksums = {path: path_hash(path) for path in restart_map}
-    r = lambda_f()
     changed_files = defaultdict(list)
     restarts = []
     # create a list of lists of the services to restart
@@ -799,7 +901,8 @@ def restart_on_change_helper(lambda_f, restart_map, stopstart=False,
         actions = ('stop', 'start') if stopstart else ('restart',)
         for service_name in services_list:
             if can_restart_now_f:
-                if not can_restart_now_f(service_name, changed_files[service_name]):
+                if not can_restart_now_f(service_name,
+                                         changed_files[service_name]):
                     continue
             if service_name in restart_functions:
                 restart_functions[service_name](service_name)
@@ -808,7 +911,6 @@ def restart_on_change_helper(lambda_f, restart_map, stopstart=False,
                     service(action, service_name)
             if post_svc_restart_f:
                 post_svc_restart_f(service_name)
-    return r
 
 
 def pwgen(length=None):

hooks/charmhelpers/core/host_factory/ubuntu.py

@@ -96,12 +96,14 @@ def cmp_pkgrevno(package, revno, pkgcache=None):
     the pkgcache argument is None. Be sure to add charmhelpers.fetch if
     you call this function, or pass an apt_pkg.Cache() instance.
     """
-    from charmhelpers.fetch import apt_pkg
+    from charmhelpers.fetch import apt_pkg, get_installed_version
     if not pkgcache:
-        from charmhelpers.fetch import apt_cache
+        current_ver = get_installed_version(package)
-        pkgcache = apt_cache()
+    else:
-    pkg = pkgcache[package]
+        pkg = pkgcache[package]
-    return apt_pkg.version_compare(pkg.current_ver.ver_str, revno)
+        current_ver = pkg.current_ver
+
+    return apt_pkg.version_compare(current_ver.ver_str, revno)
 
 
 @cached

hooks/charmhelpers/fetch/__init__.py

@@ -105,6 +105,7 @@ if __platform__ == "ubuntu":
     get_upstream_version = fetch.get_upstream_version
     apt_pkg = fetch.ubuntu_apt_pkg
     get_apt_dpkg_env = fetch.get_apt_dpkg_env
+    get_installed_version = fetch.get_installed_version
 elif __platform__ == "centos":
     yum_search = fetch.yum_search
 

hooks/charmhelpers/fetch/ubuntu.py

@@ -200,6 +200,14 @@ CLOUD_ARCHIVE_POCKETS = {
     'victoria/proposed': 'focal-proposed/victoria',
     'focal-victoria/proposed': 'focal-proposed/victoria',
     'focal-proposed/victoria': 'focal-proposed/victoria',
+    # Wallaby
+    'wallaby': 'focal-updates/wallaby',
+    'focal-wallaby': 'focal-updates/wallaby',
+    'focal-wallaby/updates': 'focal-updates/wallaby',
+    'focal-updates/wallaby': 'focal-updates/wallaby',
+    'wallaby/proposed': 'focal-proposed/wallaby',
+    'focal-wallaby/proposed': 'focal-proposed/wallaby',
+    'focal-proposed/wallaby': 'focal-proposed/wallaby',
 }
 
 
@@ -831,6 +839,22 @@ def get_upstream_version(package):
     return ubuntu_apt_pkg.upstream_version(pkg.current_ver.ver_str)
 
 
+def get_installed_version(package):
+    """Determine installed version of a package
+
+    @returns None (if not installed) or the installed version as
+    Version object
+    """
+    cache = apt_cache()
+    dpkg_result = cache._dpkg_list([package]).get(package, {})
+    current_ver = None
+    installed_version = dpkg_result.get('version')
+
+    if installed_version:
+        current_ver = ubuntu_apt_pkg.Version({'ver_str': installed_version})
+    return current_ver
+
+
 def get_apt_dpkg_env():
     """Get environment suitable for execution of APT and DPKG tools.
 

test-requirements.txt

@@ -42,8 +42,8 @@ oslo.utils<=3.41.0;python_version<'3.6'
 
 coverage>=4.5.2
 pyudev              # for ceph-* charm unit tests (need to fix the ceph-* charm unit tests/mocking)
-git+https://github.com/openstack-charmers/zaza.git#egg=zaza;python_version>='3.0'
+git+https://github.com/openstack-charmers/zaza.git@stable/21.04#egg=zaza;python_version>='3.0'
-git+https://github.com/openstack-charmers/zaza-openstack-tests.git#egg=zaza.openstack
+git+https://github.com/openstack-charmers/zaza-openstack-tests.git@stable/21.04#egg=zaza.openstack
 
 # Needed for charm-glance:
 git+https://opendev.org/openstack/tempest.git#egg=tempest;python_version>='3.6'