21.04 libraries freeze for charms on master branch

* charm-helpers sync for classic charms
* build.lock file for reactive charms
* ensure tox.ini is from release-tools
* ensure requirements.txt files are from release-tools
* On reactive charms:
  - ensure stable/21.04 branch for charms.openstack
  - ensure stable/21.04 branch for charm-helpers

Change-Id: I95f8da0789b7f165677b031284acc4fadfdfeab4
This commit is contained in:
Alex Kavanagh 2021-04-03 20:18:03 +01:00
parent 997987e791
commit c59fa98451
14 changed files with 1348 additions and 83 deletions

View File

@ -1,4 +1,4 @@
repo: https://github.com/juju/charm-helpers repo: https://github.com/juju/charm-helpers@stable/21.04
destination: charmhelpers destination: charmhelpers
include: include:
- core - core

View File

@ -42,6 +42,7 @@ import pika
import swiftclient import swiftclient
from charmhelpers.core.decorators import retry_on_exception from charmhelpers.core.decorators import retry_on_exception
from charmhelpers.contrib.amulet.utils import ( from charmhelpers.contrib.amulet.utils import (
AmuletUtils AmuletUtils
) )

View File

@ -74,7 +74,6 @@ from charmhelpers.core.host import (
pwgen, pwgen,
lsb_release, lsb_release,
CompareHostReleases, CompareHostReleases,
is_container,
) )
from charmhelpers.contrib.hahelpers.cluster import ( from charmhelpers.contrib.hahelpers.cluster import (
determine_apache_port, determine_apache_port,
@ -1596,16 +1595,21 @@ def _calculate_workers():
@returns int: number of worker processes to use @returns int: number of worker processes to use
''' '''
multiplier = config('worker-multiplier') or DEFAULT_MULTIPLIER multiplier = config('worker-multiplier')
# distinguish an empty config and an explicit config as 0.0
if multiplier is None:
multiplier = DEFAULT_MULTIPLIER
count = int(_num_cpus() * multiplier) count = int(_num_cpus() * multiplier)
if multiplier > 0 and count == 0: if count <= 0:
# assign at least one worker
count = 1 count = 1
if config('worker-multiplier') is None and is_container(): if config('worker-multiplier') is None:
# NOTE(jamespage): Limit unconfigured worker-multiplier # NOTE(jamespage): Limit unconfigured worker-multiplier
# to MAX_DEFAULT_WORKERS to avoid insane # to MAX_DEFAULT_WORKERS to avoid insane
# worker configuration in LXD containers # worker configuration on large servers
# on large servers
# Reference: https://pad.lv/1665270 # Reference: https://pad.lv/1665270
count = min(count, MAX_DEFAULT_WORKERS) count = min(count, MAX_DEFAULT_WORKERS)

View File

@ -0,0 +1,410 @@
# Copyright 2021 Canonical Limited.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Module for managing deferred service events.
This module is used to manage deferred service events from both charm actions
and package actions.
"""
import datetime
import glob
import yaml
import os
import time
import uuid
import charmhelpers.contrib.openstack.policy_rcd as policy_rcd
import charmhelpers.core.hookenv as hookenv
import charmhelpers.core.host as host
import charmhelpers.core.unitdata as unitdata
import subprocess
# Deferred events generated from the charm are stored along side those
# generated from packaging.
DEFERRED_EVENTS_DIR = policy_rcd.POLICY_DEFERRED_EVENTS_DIR
class ServiceEvent():
def __init__(self, timestamp, service, reason, action,
policy_requestor_name=None, policy_requestor_type=None):
self.timestamp = timestamp
self.service = service
self.reason = reason
self.action = action
if not policy_requestor_name:
self.policy_requestor_name = hookenv.service_name()
if not policy_requestor_type:
self.policy_requestor_type = 'charm'
def __eq__(self, other):
for attr in vars(self):
if getattr(self, attr) != getattr(other, attr):
return False
return True
def matching_request(self, other):
for attr in ['service', 'action', 'reason']:
if getattr(self, attr) != getattr(other, attr):
return False
return True
@classmethod
def from_dict(cls, data):
return cls(
data['timestamp'],
data['service'],
data['reason'],
data['action'],
data.get('policy_requestor_name'),
data.get('policy_requestor_type'))
def deferred_events_files():
"""Deferred event files
Deferred event files that were generated by service_name() policy.
:returns: Deferred event files
:rtype: List[str]
"""
return glob.glob('{}/*.deferred'.format(DEFERRED_EVENTS_DIR))
def read_event_file(file_name):
"""Read a file and return the corresponding objects.
:param file_name: Name of file to read.
:type file_name: str
:returns: ServiceEvent from file.
:rtype: ServiceEvent
"""
with open(file_name, 'r') as f:
contents = yaml.safe_load(f)
event = ServiceEvent(
contents['timestamp'],
contents['service'],
contents['reason'],
contents['action'])
return event
def deferred_events():
"""Get list of deferred events.
List of deferred events. Events are represented by dicts of the form:
{
action: restart,
policy_requestor_name: neutron-openvswitch,
policy_requestor_type: charm,
reason: 'Pkg update',
service: openvswitch-switch,
time: 1614328743}
:returns: List of deferred events.
:rtype: List[ServiceEvent]
"""
events = []
for defer_file in deferred_events_files():
events.append((defer_file, read_event_file(defer_file)))
return events
def duplicate_event_files(event):
"""Get list of event files that have equivalent deferred events.
:param event: Event to compare
:type event: ServiceEvent
:returns: List of event files
:rtype: List[str]
"""
duplicates = []
for event_file, existing_event in deferred_events():
if event.matching_request(existing_event):
duplicates.append(event_file)
return duplicates
def get_event_record_file(policy_requestor_type, policy_requestor_name):
"""Generate filename for storing a new event.
:param policy_requestor_type: System that blocked event
:type policy_requestor_type: str
:param policy_requestor_name: Name of application that blocked event
:type policy_requestor_name: str
:returns: File name
:rtype: str
"""
file_name = '{}/{}-{}-{}.deferred'.format(
DEFERRED_EVENTS_DIR,
policy_requestor_type,
policy_requestor_name,
uuid.uuid1())
return file_name
def save_event(event):
"""Write deferred events to backend.
:param event: Event to save
:type event: ServiceEvent
"""
requestor_name = hookenv.service_name()
requestor_type = 'charm'
init_policy_log_dir()
if duplicate_event_files(event):
hookenv.log(
"Not writing new event, existing event found. {} {} {}".format(
event.service,
event.action,
event.reason),
level="DEBUG")
else:
record_file = get_event_record_file(
policy_requestor_type=requestor_type,
policy_requestor_name=requestor_name)
with open(record_file, 'w') as f:
data = {
'timestamp': event.timestamp,
'service': event.service,
'action': event.action,
'reason': event.reason,
'policy_requestor_type': requestor_type,
'policy_requestor_name': requestor_name}
yaml.dump(data, f)
def clear_deferred_events(svcs, action):
"""Remove any outstanding deferred events.
Remove a deferred event if its service is in the services list and its
action matches.
:param svcs: List of services to remove.
:type svcs: List[str]
:param action: Action to remove
:type action: str
"""
# XXX This function is not currently processing the action. It needs to
# match the action and also take account of try-restart and the
# equivalnce of stop-start and restart.
for defer_file in deferred_events_files():
deferred_event = read_event_file(defer_file)
if deferred_event.service in svcs:
os.remove(defer_file)
def init_policy_log_dir():
"""Ensure directory to store events exists."""
if not os.path.exists(DEFERRED_EVENTS_DIR):
os.mkdir(DEFERRED_EVENTS_DIR)
def get_deferred_events():
"""Return a list of deferred events requested by the charm and packages.
:returns: List of deferred events
:rtype: List[ServiceEvent]
"""
events = []
for _, event in deferred_events():
events.append(event)
return events
def get_deferred_restarts():
"""List of deferred restart events requested by the charm and packages.
:returns: List of deferred restarts
:rtype: List[ServiceEvent]
"""
return [e for e in get_deferred_events() if e.action == 'restart']
def clear_deferred_restarts(services):
"""Clear deferred restart events targetted at `services`.
:param services: Services with deferred actions to clear.
:type services: List[str]
"""
clear_deferred_events(services, 'restart')
def process_svc_restart(service):
"""Respond to a service restart having occured.
:param service: Services that the action was performed against.
:type service: str
"""
clear_deferred_restarts([service])
def is_restart_permitted():
"""Check whether restarts are permitted.
:returns: Whether restarts are permitted
:rtype: bool
"""
if hookenv.config('enable-auto-restarts') is None:
return True
return hookenv.config('enable-auto-restarts')
def check_and_record_restart_request(service, changed_files):
"""Check if restarts are permitted, if they are not log the request.
:param service: Service to be restarted
:type service: str
:param changed_files: Files that have changed to trigger restarts.
:type changed_files: List[str]
:returns: Whether restarts are permitted
:rtype: bool
"""
changed_files = sorted(list(set(changed_files)))
permitted = is_restart_permitted()
if not permitted:
save_event(ServiceEvent(
timestamp=round(time.time()),
service=service,
reason='File(s) changed: {}'.format(
', '.join(changed_files)),
action='restart'))
return permitted
def deferrable_svc_restart(service, reason=None):
"""Restarts service if permitted, if not defer it.
:param service: Service to be restarted
:type service: str
:param reason: Reason for restart
:type reason: Union[str, None]
"""
if is_restart_permitted():
host.service_restart(service)
else:
save_event(ServiceEvent(
timestamp=round(time.time()),
service=service,
reason=reason,
action='restart'))
def configure_deferred_restarts(services):
"""Setup deferred restarts.
:param services: Services to block restarts of.
:type services: List[str]
"""
policy_rcd.install_policy_rcd()
if is_restart_permitted():
policy_rcd.remove_policy_file()
else:
blocked_actions = ['stop', 'restart', 'try-restart']
for svc in services:
policy_rcd.add_policy_block(svc, blocked_actions)
def get_service_start_time(service):
"""Find point in time when the systemd unit transitioned to active state.
:param service: Services to check timetsamp of.
:type service: str
"""
start_time = None
out = subprocess.check_output(
[
'systemctl',
'show',
service,
'--property=ActiveEnterTimestamp'])
str_time = out.decode().rstrip().replace('ActiveEnterTimestamp=', '')
if str_time:
start_time = datetime.datetime.strptime(
str_time,
'%a %Y-%m-%d %H:%M:%S %Z')
return start_time
def check_restart_timestamps():
"""Check deferred restarts against systemd units start time.
Check if a service has a deferred event and clear it if it has been
subsequently restarted.
"""
for event in get_deferred_restarts():
start_time = get_service_start_time(event.service)
deferred_restart_time = datetime.datetime.fromtimestamp(
event.timestamp)
if start_time and start_time < deferred_restart_time:
hookenv.log(
("Restart still required, {} was started at {}, restart was "
"requested after that at {}").format(
event.service,
start_time,
deferred_restart_time),
level='DEBUG')
else:
clear_deferred_restarts([event.service])
def set_deferred_hook(hookname):
"""Record that a hook has been deferred.
:param hookname: Name of hook that was deferred.
:type hookname: str
"""
with unitdata.HookData()() as t:
kv = t[0]
deferred_hooks = kv.get('deferred-hooks', [])
if hookname not in deferred_hooks:
deferred_hooks.append(hookname)
kv.set('deferred-hooks', sorted(list(set(deferred_hooks))))
def get_deferred_hooks():
"""Get a list of deferred hooks.
:returns: List of hook names.
:rtype: List[str]
"""
with unitdata.HookData()() as t:
kv = t[0]
return kv.get('deferred-hooks', [])
def clear_deferred_hooks():
"""Clear any deferred hooks."""
with unitdata.HookData()() as t:
kv = t[0]
kv.set('deferred-hooks', [])
def clear_deferred_hook(hookname):
"""Clear a specific deferred hooks.
:param hookname: Name of hook to remove.
:type hookname: str
"""
with unitdata.HookData()() as t:
kv = t[0]
deferred_hooks = kv.get('deferred-hooks', [])
if hookname in deferred_hooks:
deferred_hooks.remove(hookname)
kv.set('deferred-hooks', deferred_hooks)

View File

@ -19,3 +19,8 @@ class OSContextError(Exception):
This exception is principally used in contrib.openstack.context This exception is principally used in contrib.openstack.context
""" """
pass pass
class ServiceActionError(Exception):
"""Raised when a service action (stop/start/ etc) failed."""
pass

View File

@ -0,0 +1,196 @@
#!/usr/bin/env python3
"""This script is an implemenation of policy-rc.d
For further information on policy-rc.d see *1
*1 https://people.debian.org/~hmh/invokerc.d-policyrc.d-specification.txt
"""
import collections
import glob
import os
import logging
import sys
import time
import uuid
import yaml
SystemPolicy = collections.namedtuple(
'SystemPolicy',
[
'policy_requestor_name',
'policy_requestor_type',
'service',
'blocked_actions'])
DEFAULT_POLICY_CONFIG_DIR = '/etc/policy-rc.d'
DEFAULT_POLICY_LOG_DIR = '/var/lib/policy-rc.d'
def read_policy_file(policy_file):
"""Return system policies from given file.
:param file_name: Name of file to read.
:type file_name: str
:returns: Policy
:rtype: List[SystemPolicy]
"""
policies = []
if os.path.exists(policy_file):
with open(policy_file, 'r') as f:
policy = yaml.safe_load(f)
for service, actions in policy['blocked_actions'].items():
service = service.replace('.service', '')
policies.append(SystemPolicy(
policy_requestor_name=policy['policy_requestor_name'],
policy_requestor_type=policy['policy_requestor_type'],
service=service,
blocked_actions=actions))
return policies
def get_policies(policy_config_dir):
"""Return all system policies in policy_config_dir.
:param policy_config_dir: Name of file to read.
:type policy_config_dir: str
:returns: Policy
:rtype: List[SystemPolicy]
"""
_policy = []
for f in glob.glob('{}/*.policy'.format(policy_config_dir)):
_policy.extend(read_policy_file(f))
return _policy
def record_blocked_action(service, action, blocking_policies, policy_log_dir):
"""Record that an action was requested but deniedl
:param service: Service that was blocked
:type service: str
:param action: Action that was blocked.
:type action: str
:param blocking_policies: Policies that blocked the action on the service.
:type blocking_policies: List[SystemPolicy]
:param policy_log_dir: Directory to place the blocking action record.
:type policy_log_dir: str
"""
if not os.path.exists(policy_log_dir):
os.mkdir(policy_log_dir)
seconds = round(time.time())
for policy in blocking_policies:
if not os.path.exists(policy_log_dir):
os.mkdir(policy_log_dir)
file_name = '{}/{}-{}-{}.deferred'.format(
policy_log_dir,
policy.policy_requestor_type,
policy.policy_requestor_name,
uuid.uuid1())
with open(file_name, 'w') as f:
data = {
'timestamp': seconds,
'service': service,
'action': action,
'reason': 'Package update',
'policy_requestor_type': policy.policy_requestor_type,
'policy_requestor_name': policy.policy_requestor_name}
yaml.dump(data, f)
def get_blocking_policies(service, action, policy_config_dir):
"""Record that an action was requested but deniedl
:param service: Service that action is requested against.
:type service: str
:param action: Action that is requested.
:type action: str
:param policy_config_dir: Directory that stores policy files.
:type policy_config_dir: str
:returns: Policies
:rtype: List[SystemPolicy]
"""
service = service.replace('.service', '')
blocking_policies = [
policy
for policy in get_policies(policy_config_dir)
if policy.service == service and action in policy.blocked_actions]
return blocking_policies
def process_action_request(service, action, policy_config_dir, policy_log_dir):
"""Take the requested action against service and check if it is permitted.
:param service: Service that action is requested against.
:type service: str
:param action: Action that is requested.
:type action: str
:param policy_config_dir: Directory that stores policy files.
:type policy_config_dir: str
:param policy_log_dir: Directory that stores policy files.
:type policy_log_dir: str
:returns: Tuple of whether the action is permitted and explanation.
:rtype: (boolean, str)
"""
blocking_policies = get_blocking_policies(
service,
action,
policy_config_dir)
if blocking_policies:
policy_msg = [
'{} {}'.format(p.policy_requestor_type, p.policy_requestor_name)
for p in sorted(blocking_policies)]
message = '{} of {} blocked by {}'.format(
action,
service,
', '.join(policy_msg))
record_blocked_action(
service,
action,
blocking_policies,
policy_log_dir)
action_permitted = False
else:
message = "Permitting {} {}".format(service, action)
action_permitted = True
return action_permitted, message
def main():
logging.basicConfig(
filename='/var/log/policy-rc.d.log',
level=logging.DEBUG,
format='%(asctime)s %(message)s')
service = sys.argv[1]
action = sys.argv[2]
permitted, message = process_action_request(
service,
action,
DEFAULT_POLICY_CONFIG_DIR,
DEFAULT_POLICY_LOG_DIR)
logging.info(message)
# https://people.debian.org/~hmh/invokerc.d-policyrc.d-specification.txt
# Exit status codes:
# 0 - action allowed
# 1 - unknown action (therefore, undefined policy)
# 100 - unknown initscript id
# 101 - action forbidden by policy
# 102 - subsystem error
# 103 - syntax error
# 104 - [reserved]
# 105 - behaviour uncertain, policy undefined.
# 106 - action not allowed. Use the returned fallback actions
# (which are implied to be "allowed") instead.
if permitted:
return 0
else:
return 101
if __name__ == "__main__":
rc = main()
sys.exit(rc)

View File

@ -0,0 +1,173 @@
# Copyright 2021 Canonical Limited.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Module for managing policy-rc.d script and associated files.
This module manages the installation of /usr/sbin/policy-rc.d, the
policy files and the event files. When a package update occurs the
packaging system calls:
policy-rc.d [options] <initscript ID> <actions>
The return code of the script determines if the packaging system
will perform that action on the given service. The policy-rc.d
implementation installed by this module checks if an action is
permitted by checking policy files placed in /etc/policy-rc.d.
If a policy file exists which denies the requested action then
this is recorded in an event file which is placed in
/var/lib/policy-rc.d.
"""
import os
import shutil
import tempfile
import yaml
import charmhelpers.contrib.openstack.files as os_files
import charmhelpers.contrib.openstack.alternatives as alternatives
import charmhelpers.core.hookenv as hookenv
import charmhelpers.core.host as host
POLICY_HEADER = """# Managed by juju\n"""
POLICY_DEFERRED_EVENTS_DIR = '/var/lib/policy-rc.d'
POLICY_CONFIG_DIR = '/etc/policy-rc.d'
def get_policy_file_name():
"""Get the name of the policy file for this application.
:returns: Policy file name
:rtype: str
"""
application_name = hookenv.service_name()
return '{}/charm-{}.policy'.format(POLICY_CONFIG_DIR, application_name)
def read_default_policy_file():
"""Return the policy file.
A policy is in the form:
blocked_actions:
neutron-dhcp-agent: [restart, stop, try-restart]
neutron-l3-agent: [restart, stop, try-restart]
neutron-metadata-agent: [restart, stop, try-restart]
neutron-openvswitch-agent: [restart, stop, try-restart]
openvswitch-switch: [restart, stop, try-restart]
ovs-vswitchd: [restart, stop, try-restart]
ovs-vswitchd-dpdk: [restart, stop, try-restart]
ovsdb-server: [restart, stop, try-restart]
policy_requestor_name: neutron-openvswitch
policy_requestor_type: charm
:returns: Policy
:rtype: Dict[str, Union[str, Dict[str, List[str]]]
"""
policy = {}
policy_file = get_policy_file_name()
if os.path.exists(policy_file):
with open(policy_file, 'r') as f:
policy = yaml.safe_load(f)
return policy
def write_policy_file(policy_file, policy):
"""Write policy to disk.
:param policy_file: Name of policy file
:type policy_file: str
:param policy: Policy
:type policy: Dict[str, Union[str, Dict[str, List[str]]]]
"""
with tempfile.NamedTemporaryFile('w', delete=False) as f:
f.write(POLICY_HEADER)
yaml.dump(policy, f)
tmp_file_name = f.name
shutil.move(tmp_file_name, policy_file)
def remove_policy_file():
"""Remove policy file."""
try:
os.remove(get_policy_file_name())
except FileNotFoundError:
pass
def install_policy_rcd():
"""Install policy-rc.d components."""
source_file_dir = os.path.dirname(os.path.abspath(os_files.__file__))
policy_rcd_exec = "/var/lib/charm/{}/policy-rc.d".format(
hookenv.service_name())
host.mkdir(os.path.dirname(policy_rcd_exec))
shutil.copy2(
'{}/policy_rc_d_script.py'.format(source_file_dir),
policy_rcd_exec)
# policy-rc.d must be installed via the alternatives system:
# https://people.debian.org/~hmh/invokerc.d-policyrc.d-specification.txt
if not os.path.exists('/usr/sbin/policy-rc.d'):
alternatives.install_alternative(
'policy-rc.d',
'/usr/sbin/policy-rc.d',
policy_rcd_exec)
host.mkdir(POLICY_CONFIG_DIR)
def get_default_policy():
"""Return the default policy structure.
:returns: Policy
:rtype: Dict[str, Union[str, Dict[str, List[str]]]
"""
policy = {
'policy_requestor_name': hookenv.service_name(),
'policy_requestor_type': 'charm',
'blocked_actions': {}}
return policy
def add_policy_block(service, blocked_actions):
"""Update a policy file with new list of actions.
:param service: Service name
:type service: str
:param blocked_actions: Action to block
:type blocked_actions: List[str]
"""
policy = read_default_policy_file() or get_default_policy()
policy_file = get_policy_file_name()
if policy['blocked_actions'].get(service):
policy['blocked_actions'][service].extend(blocked_actions)
else:
policy['blocked_actions'][service] = blocked_actions
policy['blocked_actions'][service] = sorted(
list(set(policy['blocked_actions'][service])))
write_policy_file(policy_file, policy)
def remove_policy_block(service, unblocked_actions):
"""Remove list of actions from policy file.
:param service: Service name
:type service: str
:param unblocked_actions: Action to unblock
:type unblocked_actions: List[str]
"""
policy_file = get_policy_file_name()
policy = read_default_policy_file()
for action in unblocked_actions:
try:
policy['blocked_actions'][service].remove(action)
except (KeyError, ValueError):
continue
write_policy_file(policy_file, policy)

View File

@ -14,7 +14,7 @@
# Common python helper functions used for OpenStack charms. # Common python helper functions used for OpenStack charms.
from collections import OrderedDict, namedtuple from collections import OrderedDict, namedtuple
from functools import wraps from functools import partial, wraps
import subprocess import subprocess
import json import json
@ -36,9 +36,12 @@ from charmhelpers.contrib.network import ip
from charmhelpers.core import decorators, unitdata from charmhelpers.core import decorators, unitdata
import charmhelpers.contrib.openstack.deferred_events as deferred_events
from charmhelpers.core.hookenv import ( from charmhelpers.core.hookenv import (
WORKLOAD_STATES, WORKLOAD_STATES,
action_fail, action_fail,
action_get,
action_set, action_set,
config, config,
expected_peer_units, expected_peer_units,
@ -112,7 +115,7 @@ from charmhelpers.fetch.snap import (
from charmhelpers.contrib.storage.linux.utils import is_block_device, zap_disk from charmhelpers.contrib.storage.linux.utils import is_block_device, zap_disk
from charmhelpers.contrib.storage.linux.loopback import ensure_loopback_device from charmhelpers.contrib.storage.linux.loopback import ensure_loopback_device
from charmhelpers.contrib.openstack.exceptions import OSContextError from charmhelpers.contrib.openstack.exceptions import OSContextError, ServiceActionError
from charmhelpers.contrib.openstack.policyd import ( from charmhelpers.contrib.openstack.policyd import (
policyd_status_message_prefix, policyd_status_message_prefix,
POLICYD_CONFIG_NAME, POLICYD_CONFIG_NAME,
@ -148,6 +151,7 @@ OPENSTACK_RELEASES = (
'train', 'train',
'ussuri', 'ussuri',
'victoria', 'victoria',
'wallaby',
) )
UBUNTU_OPENSTACK_RELEASE = OrderedDict([ UBUNTU_OPENSTACK_RELEASE = OrderedDict([
@ -170,6 +174,7 @@ UBUNTU_OPENSTACK_RELEASE = OrderedDict([
('eoan', 'train'), ('eoan', 'train'),
('focal', 'ussuri'), ('focal', 'ussuri'),
('groovy', 'victoria'), ('groovy', 'victoria'),
('hirsute', 'wallaby'),
]) ])
@ -193,6 +198,7 @@ OPENSTACK_CODENAMES = OrderedDict([
('2019.2', 'train'), ('2019.2', 'train'),
('2020.1', 'ussuri'), ('2020.1', 'ussuri'),
('2020.2', 'victoria'), ('2020.2', 'victoria'),
('2021.1', 'wallaby'),
]) ])
# The ugly duckling - must list releases oldest to newest # The ugly duckling - must list releases oldest to newest
@ -301,8 +307,8 @@ PACKAGE_CODENAMES = {
('14', 'rocky'), ('14', 'rocky'),
('15', 'stein'), ('15', 'stein'),
('16', 'train'), ('16', 'train'),
('18', 'ussuri'), ('18', 'ussuri'), # Note this was actually 17.0 - 18.3
('19', 'victoria'), ('19', 'victoria'), # Note this is really 18.6
]), ]),
'ceilometer-common': OrderedDict([ 'ceilometer-common': OrderedDict([
('5', 'liberty'), ('5', 'liberty'),
@ -483,9 +489,26 @@ def get_swift_codename(version):
return None return None
@deprecate("moved to charmhelpers.contrib.openstack.utils.get_installed_os_version()", "2021-01", log=juju_log)
def get_os_codename_package(package, fatal=True): def get_os_codename_package(package, fatal=True):
'''Derive OpenStack release codename from an installed package.''' """Derive OpenStack release codename from an installed package.
Initially, see if the openstack-release pkg is available (by trying to
install it) and use it instead.
If it isn't then it falls back to the existing method of checking the
version of the package passed and then resolving the version from that
using lookup tables.
Note: if possible, charms should use get_installed_os_version() to
determine the version of the "openstack-release" pkg.
:param package: the package to test for version information.
:type package: str
:param fatal: If True (default), then die via error_out()
:type fatal: bool
:returns: the OpenStack release codename (e.g. ussuri)
:rtype: str
"""
codename = get_installed_os_version() codename = get_installed_os_version()
if codename: if codename:
@ -579,8 +602,22 @@ def get_os_version_package(pkg, fatal=True):
def get_installed_os_version(): def get_installed_os_version():
apt_install(filter_installed_packages(['openstack-release']), fatal=False) """Determine the OpenStack release code name from openstack-release pkg.
print("OpenStack Release: {}".format(openstack_release()))
This uses the "openstack-release" pkg (if it exists) to return the
OpenStack release codename (e.g. usurri, mitaka, ocata, etc.)
Note, it caches the result so that it is only done once per hook.
:returns: the OpenStack release codename, if available
:rtype: Optional[str]
"""
@cached
def _do_install():
apt_install(filter_installed_packages(['openstack-release']),
fatal=False, quiet=True)
_do_install()
return openstack_release().get('OPENSTACK_CODENAME') return openstack_release().get('OPENSTACK_CODENAME')
@ -1052,6 +1089,18 @@ def _determine_os_workload_status(
try: try:
if config(POLICYD_CONFIG_NAME): if config(POLICYD_CONFIG_NAME):
message = "{} {}".format(policyd_status_message_prefix(), message) message = "{} {}".format(policyd_status_message_prefix(), message)
deferred_restarts = list(set(
[e.service for e in deferred_events.get_deferred_restarts()]))
if deferred_restarts:
svc_msg = "Services queued for restart: {}".format(
', '.join(sorted(deferred_restarts)))
message = "{}. {}".format(message, svc_msg)
deferred_hooks = deferred_events.get_deferred_hooks()
if deferred_hooks:
svc_msg = "Hooks skipped due to disabled auto restarts: {}".format(
', '.join(sorted(deferred_hooks)))
message = "{}. {}".format(message, svc_msg)
except Exception: except Exception:
pass pass
@ -1536,6 +1585,33 @@ def is_unit_paused_set():
return False return False
def is_hook_allowed(hookname, check_deferred_restarts=True):
"""Check if hook can run.
:param hookname: Name of hook to check..
:type hookname: str
:param check_deferred_restarts: Whether to check deferred restarts.
:type check_deferred_restarts: bool
"""
permitted = True
reasons = []
if is_unit_paused_set():
reasons.append(
"Unit is pause or upgrading. Skipping {}".format(hookname))
permitted = False
if check_deferred_restarts:
if deferred_events.is_restart_permitted():
permitted = True
deferred_events.clear_deferred_hook(hookname)
else:
if not config().changed('enable-auto-restarts'):
deferred_events.set_deferred_hook(hookname)
reasons.append("auto restarts are disabled")
permitted = False
return permitted, " and ".join(reasons)
def manage_payload_services(action, services=None, charm_func=None): def manage_payload_services(action, services=None, charm_func=None):
"""Run an action against all services. """Run an action against all services.
@ -1696,6 +1772,43 @@ def resume_unit(assess_status_func, services=None, ports=None,
raise Exception("Couldn't resume: {}".format("; ".join(messages))) raise Exception("Couldn't resume: {}".format("; ".join(messages)))
def restart_services_action(services=None, when_all_stopped_func=None,
deferred_only=None):
"""Manage a service restart request via charm action.
:param services: Services to be restarted
:type model_name: List[str]
:param when_all_stopped_func: Function to call when all services are
stopped.
:type when_all_stopped_func: Callable[]
:param model_name: Only restart services which have a deferred restart
event.
:type model_name: bool
"""
if services and deferred_only:
raise ValueError(
"services and deferred_only are mutually exclusive")
if deferred_only:
services = list(set(
[a.service for a in deferred_events.get_deferred_restarts()]))
_, messages = manage_payload_services(
'stop',
services=services,
charm_func=when_all_stopped_func)
if messages:
raise ServiceActionError(
"Error processing service stop request: {}".format(
"; ".join(messages)))
_, messages = manage_payload_services(
'start',
services=services)
if messages:
raise ServiceActionError(
"Error processing service start request: {}".format(
"; ".join(messages)))
deferred_events.clear_deferred_restarts(services)
def make_assess_status_func(*args, **kwargs): def make_assess_status_func(*args, **kwargs):
"""Creates an assess_status_func() suitable for handing to pause_unit() """Creates an assess_status_func() suitable for handing to pause_unit()
and resume_unit(). and resume_unit().
@ -1717,7 +1830,10 @@ def make_assess_status_func(*args, **kwargs):
def pausable_restart_on_change(restart_map, stopstart=False, def pausable_restart_on_change(restart_map, stopstart=False,
restart_functions=None): restart_functions=None,
can_restart_now_f=None,
post_svc_restart_f=None,
pre_restarts_wait_f=None):
"""A restart_on_change decorator that checks to see if the unit is """A restart_on_change decorator that checks to see if the unit is
paused. If it is paused then the decorated function doesn't fire. paused. If it is paused then the decorated function doesn't fire.
@ -1743,11 +1859,28 @@ def pausable_restart_on_change(restart_map, stopstart=False,
function won't be called if the decorated function is never called. Note, function won't be called if the decorated function is never called. Note,
retains backwards compatibility for passing a non-callable dictionary. retains backwards compatibility for passing a non-callable dictionary.
@param f: the function to decorate :param f: function to decorate.
@param restart_map: (optionally callable, which then returns the :type f: Callable
restart_map) the restart map {conf_file: [services]} :param restart_map: Optionally callable, which then returns the restart_map or
@param stopstart: DEFAULT false; whether to stop, start or just restart the restart map {conf_file: [services]}
@returns decorator to use a restart_on_change with pausability :type restart_map: Union[Callable[[],], Dict[str, List[str,]]
:param stopstart: whether to stop, start or restart a service
:type stopstart: booleean
:param restart_functions: nonstandard functions to use to restart services
{svc: func, ...}
:type restart_functions: Dict[str, Callable[[str], None]]
:param can_restart_now_f: A function used to check if the restart is
permitted.
:type can_restart_now_f: Callable[[str, List[str]], boolean]
:param post_svc_restart_f: A function run after a service has
restarted.
:type post_svc_restart_f: Callable[[str], None]
:param pre_restarts_wait_f: A function callled before any restarts.
:type pre_restarts_wait_f: Callable[None, None]
:returns: decorator to use a restart_on_change with pausability
:rtype: decorator
""" """
def wrap(f): def wrap(f):
# py27 compatible nonlocal variable. When py3 only, replace with # py27 compatible nonlocal variable. When py3 only, replace with
@ -1763,8 +1896,13 @@ def pausable_restart_on_change(restart_map, stopstart=False,
if callable(restart_map) else restart_map if callable(restart_map) else restart_map
# otherwise, normal restart_on_change functionality # otherwise, normal restart_on_change functionality
return restart_on_change_helper( return restart_on_change_helper(
(lambda: f(*args, **kwargs)), __restart_map_cache['cache'], (lambda: f(*args, **kwargs)),
stopstart, restart_functions) __restart_map_cache['cache'],
stopstart,
restart_functions,
can_restart_now_f,
post_svc_restart_f,
pre_restarts_wait_f)
return wrapped_f return wrapped_f
return wrap return wrap
@ -2145,6 +2283,23 @@ def container_scoped_relations():
return relations return relations
def container_scoped_relation_get(attribute=None):
"""Get relation data from all container scoped relations.
:param attribute: Name of attribute to get
:type attribute: Optional[str]
:returns: Iterator with relation data
:rtype: Iterator[Optional[any]]
"""
for endpoint_name in container_scoped_relations():
for rid in relation_ids(endpoint_name):
for unit in related_units(rid):
yield relation_get(
attribute=attribute,
unit=unit,
rid=rid)
def is_db_ready(use_current_context=False, rel_name=None): def is_db_ready(use_current_context=False, rel_name=None):
"""Check remote database is ready to be used. """Check remote database is ready to be used.
@ -2418,3 +2573,107 @@ def get_api_application_status():
msg = 'Some units are not ready' msg = 'Some units are not ready'
juju_log(msg, 'DEBUG') juju_log(msg, 'DEBUG')
return app_state, msg return app_state, msg
def sequence_status_check_functions(*functions):
"""Sequence the functions passed so that they all get a chance to run as
the charm status check functions.
:param *functions: a list of functions that return (state, message)
:type *functions: List[Callable[[OSConfigRender], (str, str)]]
:returns: the Callable that takes configs and returns (state, message)
:rtype: Callable[[OSConfigRender], (str, str)]
"""
def _inner_sequenced_functions(configs):
state, message = 'unknown', ''
for f in functions:
new_state, new_message = f(configs)
state = workload_state_compare(state, new_state)
if message:
message = "{}, {}".format(message, new_message)
else:
message = new_message
return state, message
return _inner_sequenced_functions
SubordinatePackages = namedtuple('SubordinatePackages', ['install', 'purge'])
def get_subordinate_release_packages(os_release, package_type='deb'):
"""Iterate over subordinate relations and get package information.
:param os_release: OpenStack release to look for
:type os_release: str
:param package_type: Package type (one of 'deb' or 'snap')
:type package_type: str
:returns: Packages to install and packages to purge or None
:rtype: SubordinatePackages[set,set]
"""
install = set()
purge = set()
for rdata in container_scoped_relation_get('releases-packages-map'):
rp_map = json.loads(rdata or '{}')
# The map provided by subordinate has OpenStack release name as key.
# Find package information from subordinate matching requested release
# or the most recent release prior to requested release by sorting the
# keys in reverse order. This follows established patterns in our
# charms for templates and reactive charm implementations, i.e. as long
# as nothing has changed the definitions for the prior OpenStack
# release is still valid.
for release in sorted(rp_map.keys(), reverse=True):
if (CompareOpenStackReleases(release) <= os_release and
package_type in rp_map[release]):
for name, container in (
('install', install),
('purge', purge)):
for pkg in rp_map[release][package_type].get(name, []):
container.add(pkg)
break
return SubordinatePackages(install, purge)
os_restart_on_change = partial(
pausable_restart_on_change,
can_restart_now_f=deferred_events.check_and_record_restart_request,
post_svc_restart_f=deferred_events.process_svc_restart)
def restart_services_action_helper(all_services):
"""Helper to run the restart-services action.
NOTE: all_services is all services that could be restarted but
depending on the action arguments it may be a subset of
these that are actually restarted.
:param all_services: All services that could be restarted
:type all_services: List[str]
"""
deferred_only = action_get("deferred-only")
services = action_get("services")
if services:
services = services.split()
else:
services = all_services
if deferred_only:
restart_services_action(deferred_only=True)
else:
restart_services_action(services=services)
def show_deferred_events_action_helper():
"""Helper to run the show-deferred-restarts action."""
restarts = []
for event in deferred_events.get_deferred_events():
restarts.append('{} {} {}'.format(
str(event.timestamp),
event.service.ljust(40),
event.reason))
restarts.sort()
output = {
'restarts': restarts,
'hooks': deferred_events.get_deferred_hooks()}
action_set({'output': "{}".format(
yaml.dump(output, default_flow_style=False))})

View File

@ -1622,3 +1622,12 @@ def _contains_range(addresses):
addresses.startswith(".") or addresses.startswith(".") or
",." in addresses or ",." in addresses or
" ." in addresses) " ." in addresses)
def is_subordinate():
"""Check whether charm is subordinate in unit metadata.
:returns: True if unit is subordniate, False otherwise.
:rtype: bool
"""
return metadata().get('subordinate') is True

View File

@ -34,7 +34,7 @@ import itertools
import six import six
from contextlib import contextmanager from contextlib import contextmanager
from collections import OrderedDict from collections import OrderedDict, defaultdict
from .hookenv import log, INFO, DEBUG, local_unit, charm_name from .hookenv import log, INFO, DEBUG, local_unit, charm_name
from .fstab import Fstab from .fstab import Fstab
from charmhelpers.osplatform import get_platform from charmhelpers.osplatform import get_platform
@ -694,74 +694,223 @@ class ChecksumError(ValueError):
pass pass
def restart_on_change(restart_map, stopstart=False, restart_functions=None): class restart_on_change(object):
"""Restart services based on configuration files changing """Decorator and context manager to handle restarts.
This function is used a decorator, for example:: Usage:
@restart_on_change({ @restart_on_change(restart_map, ...)
'/etc/ceph/ceph.conf': [ 'cinder-api', 'cinder-volume' ] def function_that_might_trigger_a_restart(...)
'/etc/apache/sites-enabled/*': [ 'apache2' ] ...
})
def config_changed():
pass # your code here
In this example, the cinder-api and cinder-volume services Or:
would be restarted if /etc/ceph/ceph.conf is changed by the
ceph_client_changed function. The apache2 service would be
restarted if any file matching the pattern got changed, created
or removed. Standard wildcards are supported, see documentation
for the 'glob' module for more information.
@param restart_map: {path_file_name: [service_name, ...] with restart_on_change(restart_map, ...):
@param stopstart: DEFAULT false; whether to stop, start OR restart do_stuff_that_might_trigger_a_restart()
@param restart_functions: nonstandard functions to use to restart services ...
{svc: func, ...} """
@returns result from decorated function
def __init__(self, restart_map, stopstart=False, restart_functions=None,
can_restart_now_f=None, post_svc_restart_f=None,
pre_restarts_wait_f=None):
"""
:param restart_map: {file: [service, ...]}
:type restart_map: Dict[str, List[str,]]
:param stopstart: whether to stop, start or restart a service
:type stopstart: booleean
:param restart_functions: nonstandard functions to use to restart
services {svc: func, ...}
:type restart_functions: Dict[str, Callable[[str], None]]
:param can_restart_now_f: A function used to check if the restart is
permitted.
:type can_restart_now_f: Callable[[str, List[str]], boolean]
:param post_svc_restart_f: A function run after a service has
restarted.
:type post_svc_restart_f: Callable[[str], None]
:param pre_restarts_wait_f: A function callled before any restarts.
:type pre_restarts_wait_f: Callable[None, None]
"""
self.restart_map = restart_map
self.stopstart = stopstart
self.restart_functions = restart_functions
self.can_restart_now_f = can_restart_now_f
self.post_svc_restart_f = post_svc_restart_f
self.pre_restarts_wait_f = pre_restarts_wait_f
def __call__(self, f):
"""Work like a decorator.
Returns a wrapped function that performs the restart if triggered.
:param f: The function that is being wrapped.
:type f: Callable[[Any], Any]
:returns: the wrapped function
:rtype: Callable[[Any], Any]
""" """
def wrap(f):
@functools.wraps(f) @functools.wraps(f)
def wrapped_f(*args, **kwargs): def wrapped_f(*args, **kwargs):
return restart_on_change_helper( return restart_on_change_helper(
(lambda: f(*args, **kwargs)), restart_map, stopstart, (lambda: f(*args, **kwargs)),
restart_functions) self.restart_map,
stopstart=self.stopstart,
restart_functions=self.restart_functions,
can_restart_now_f=self.can_restart_now_f,
post_svc_restart_f=self.post_svc_restart_f,
pre_restarts_wait_f=self.pre_restarts_wait_f)
return wrapped_f return wrapped_f
return wrap
def __enter__(self):
"""Enter the runtime context related to this object. """
self.checksums = _pre_restart_on_change_helper(self.restart_map)
def __exit__(self, exc_type, exc_val, exc_tb):
"""Exit the runtime context related to this object.
The parameters describe the exception that caused the context to be
exited. If the context was exited without an exception, all three
arguments will be None.
"""
if exc_type is None:
_post_restart_on_change_helper(
self.checksums,
self.restart_map,
stopstart=self.stopstart,
restart_functions=self.restart_functions,
can_restart_now_f=self.can_restart_now_f,
post_svc_restart_f=self.post_svc_restart_f,
pre_restarts_wait_f=self.pre_restarts_wait_f)
# All is good, so return False; any exceptions will propagate.
return False
def restart_on_change_helper(lambda_f, restart_map, stopstart=False, def restart_on_change_helper(lambda_f, restart_map, stopstart=False,
restart_functions=None): restart_functions=None,
can_restart_now_f=None,
post_svc_restart_f=None,
pre_restarts_wait_f=None):
"""Helper function to perform the restart_on_change function. """Helper function to perform the restart_on_change function.
This is provided for decorators to restart services if files described This is provided for decorators to restart services if files described
in the restart_map have changed after an invocation of lambda_f(). in the restart_map have changed after an invocation of lambda_f().
@param lambda_f: function to call. This functions allows for a number of helper functions to be passed.
@param restart_map: {file: [service, ...]}
@param stopstart: whether to stop, start or restart a service `restart_functions` is a map with a service as the key and the
@param restart_functions: nonstandard functions to use to restart services corresponding value being the function to call to restart the service. For
example if `restart_functions={'some-service': my_restart_func}` then
`my_restart_func` should a function which takes one argument which is the
service name to be retstarted.
`can_restart_now_f` is a function which checks that a restart is permitted.
It should return a bool which indicates if a restart is allowed and should
take a service name (str) and a list of changed files (List[str]) as
arguments.
`post_svc_restart_f` is a function which runs after a service has been
restarted. It takes the service name that was restarted as an argument.
`pre_restarts_wait_f` is a function which is called before any restarts
occur. The use case for this is an application which wants to try and
stagger restarts between units.
:param lambda_f: function to call.
:type lambda_f: Callable[[], ANY]
:param restart_map: {file: [service, ...]}
:type restart_map: Dict[str, List[str,]]
:param stopstart: whether to stop, start or restart a service
:type stopstart: booleean
:param restart_functions: nonstandard functions to use to restart services
{svc: func, ...} {svc: func, ...}
@returns result of lambda_f() :type restart_functions: Dict[str, Callable[[str], None]]
:param can_restart_now_f: A function used to check if the restart is
permitted.
:type can_restart_now_f: Callable[[str, List[str]], boolean]
:param post_svc_restart_f: A function run after a service has
restarted.
:type post_svc_restart_f: Callable[[str], None]
:param pre_restarts_wait_f: A function callled before any restarts.
:type pre_restarts_wait_f: Callable[None, None]
:returns: result of lambda_f()
:rtype: ANY
"""
checksums = _pre_restart_on_change_helper(restart_map)
r = lambda_f()
_post_restart_on_change_helper(checksums,
restart_map,
stopstart,
restart_functions,
can_restart_now_f,
post_svc_restart_f,
pre_restarts_wait_f)
return r
def _pre_restart_on_change_helper(restart_map):
"""Take a snapshot of file hashes.
:param restart_map: {file: [service, ...]}
:type restart_map: Dict[str, List[str,]]
:returns: Dictionary of file paths and the files checksum.
:rtype: Dict[str, str]
"""
return {path: path_hash(path) for path in restart_map}
def _post_restart_on_change_helper(checksums,
restart_map,
stopstart=False,
restart_functions=None,
can_restart_now_f=None,
post_svc_restart_f=None,
pre_restarts_wait_f=None):
"""Check whether files have changed.
:param checksums: Dictionary of file paths and the files checksum.
:type checksums: Dict[str, str]
:param restart_map: {file: [service, ...]}
:type restart_map: Dict[str, List[str,]]
:param stopstart: whether to stop, start or restart a service
:type stopstart: booleean
:param restart_functions: nonstandard functions to use to restart services
{svc: func, ...}
:type restart_functions: Dict[str, Callable[[str], None]]
:param can_restart_now_f: A function used to check if the restart is
permitted.
:type can_restart_now_f: Callable[[str, List[str]], boolean]
:param post_svc_restart_f: A function run after a service has
restarted.
:type post_svc_restart_f: Callable[[str], None]
:param pre_restarts_wait_f: A function callled before any restarts.
:type pre_restarts_wait_f: Callable[None, None]
""" """
if restart_functions is None: if restart_functions is None:
restart_functions = {} restart_functions = {}
checksums = {path: path_hash(path) for path in restart_map} changed_files = defaultdict(list)
r = lambda_f() restarts = []
# create a list of lists of the services to restart # create a list of lists of the services to restart
restarts = [restart_map[path] for path, services in restart_map.items():
for path in restart_map if path_hash(path) != checksums[path]:
if path_hash(path) != checksums[path]] restarts.append(services)
for svc in services:
changed_files[svc].append(path)
# create a flat list of ordered services without duplicates from lists # create a flat list of ordered services without duplicates from lists
services_list = list(OrderedDict.fromkeys(itertools.chain(*restarts))) services_list = list(OrderedDict.fromkeys(itertools.chain(*restarts)))
if services_list: if services_list:
if pre_restarts_wait_f:
pre_restarts_wait_f()
actions = ('stop', 'start') if stopstart else ('restart',) actions = ('stop', 'start') if stopstart else ('restart',)
for service_name in services_list: for service_name in services_list:
if can_restart_now_f:
if not can_restart_now_f(service_name,
changed_files[service_name]):
continue
if service_name in restart_functions: if service_name in restart_functions:
restart_functions[service_name](service_name) restart_functions[service_name](service_name)
else: else:
for action in actions: for action in actions:
service(action, service_name) service(action, service_name)
return r if post_svc_restart_f:
post_svc_restart_f(service_name)
def pwgen(length=None): def pwgen(length=None):

View File

@ -96,12 +96,14 @@ def cmp_pkgrevno(package, revno, pkgcache=None):
the pkgcache argument is None. Be sure to add charmhelpers.fetch if the pkgcache argument is None. Be sure to add charmhelpers.fetch if
you call this function, or pass an apt_pkg.Cache() instance. you call this function, or pass an apt_pkg.Cache() instance.
""" """
from charmhelpers.fetch import apt_pkg from charmhelpers.fetch import apt_pkg, get_installed_version
if not pkgcache: if not pkgcache:
from charmhelpers.fetch import apt_cache current_ver = get_installed_version(package)
pkgcache = apt_cache() else:
pkg = pkgcache[package] pkg = pkgcache[package]
return apt_pkg.version_compare(pkg.current_ver.ver_str, revno) current_ver = pkg.current_ver
return apt_pkg.version_compare(current_ver.ver_str, revno)
@cached @cached

View File

@ -105,6 +105,7 @@ if __platform__ == "ubuntu":
get_upstream_version = fetch.get_upstream_version get_upstream_version = fetch.get_upstream_version
apt_pkg = fetch.ubuntu_apt_pkg apt_pkg = fetch.ubuntu_apt_pkg
get_apt_dpkg_env = fetch.get_apt_dpkg_env get_apt_dpkg_env = fetch.get_apt_dpkg_env
get_installed_version = fetch.get_installed_version
elif __platform__ == "centos": elif __platform__ == "centos":
yum_search = fetch.yum_search yum_search = fetch.yum_search

View File

@ -13,6 +13,7 @@
# limitations under the License. # limitations under the License.
from collections import OrderedDict from collections import OrderedDict
import os
import platform import platform
import re import re
import six import six
@ -20,6 +21,7 @@ import subprocess
import sys import sys
import time import time
from charmhelpers import deprecate
from charmhelpers.core.host import get_distrib_codename, get_system_env from charmhelpers.core.host import get_distrib_codename, get_system_env
from charmhelpers.core.hookenv import ( from charmhelpers.core.hookenv import (
@ -198,6 +200,14 @@ CLOUD_ARCHIVE_POCKETS = {
'victoria/proposed': 'focal-proposed/victoria', 'victoria/proposed': 'focal-proposed/victoria',
'focal-victoria/proposed': 'focal-proposed/victoria', 'focal-victoria/proposed': 'focal-proposed/victoria',
'focal-proposed/victoria': 'focal-proposed/victoria', 'focal-proposed/victoria': 'focal-proposed/victoria',
# Wallaby
'wallaby': 'focal-updates/wallaby',
'focal-wallaby': 'focal-updates/wallaby',
'focal-wallaby/updates': 'focal-updates/wallaby',
'focal-updates/wallaby': 'focal-updates/wallaby',
'wallaby/proposed': 'focal-proposed/wallaby',
'focal-wallaby/proposed': 'focal-proposed/wallaby',
'focal-proposed/wallaby': 'focal-proposed/wallaby',
} }
@ -251,13 +261,19 @@ def apt_cache(*_, **__):
# Detect this situation, log a warning and make the call to # Detect this situation, log a warning and make the call to
# ``apt_pkg.init()`` to avoid the consumer Python interpreter from # ``apt_pkg.init()`` to avoid the consumer Python interpreter from
# crashing with a segmentation fault. # crashing with a segmentation fault.
log('Support for use of upstream ``apt_pkg`` module in conjunction' @deprecate(
'with charm-helpers is deprecated since 2019-06-25', level=WARNING) 'Support for use of upstream ``apt_pkg`` module in conjunction'
'with charm-helpers is deprecated since 2019-06-25',
date=None, log=lambda x: log(x, level=WARNING))
def one_shot_log():
pass
one_shot_log()
sys.modules['apt_pkg'].init() sys.modules['apt_pkg'].init()
return ubuntu_apt_pkg.Cache() return ubuntu_apt_pkg.Cache()
def apt_install(packages, options=None, fatal=False): def apt_install(packages, options=None, fatal=False, quiet=False):
"""Install one or more packages. """Install one or more packages.
:param packages: Package(s) to install :param packages: Package(s) to install
@ -267,6 +283,8 @@ def apt_install(packages, options=None, fatal=False):
:param fatal: Whether the command's output should be checked and :param fatal: Whether the command's output should be checked and
retried. retried.
:type fatal: bool :type fatal: bool
:param quiet: if True (default), supress log message to stdout/stderr
:type quiet: bool
:raises: subprocess.CalledProcessError :raises: subprocess.CalledProcessError
""" """
if options is None: if options is None:
@ -279,9 +297,10 @@ def apt_install(packages, options=None, fatal=False):
cmd.append(packages) cmd.append(packages)
else: else:
cmd.extend(packages) cmd.extend(packages)
log("Installing {} with options: {}".format(packages, if not quiet:
options)) log("Installing {} with options: {}"
_run_apt_command(cmd, fatal) .format(packages, options))
_run_apt_command(cmd, fatal, quiet=quiet)
def apt_upgrade(options=None, fatal=False, dist=False): def apt_upgrade(options=None, fatal=False, dist=False):
@ -639,14 +658,17 @@ def _add_apt_repository(spec):
:param spec: the parameter to pass to add_apt_repository :param spec: the parameter to pass to add_apt_repository
:type spec: str :type spec: str
""" """
if '{series}' in spec:
series = get_distrib_codename() series = get_distrib_codename()
if '{series}' in spec:
spec = spec.replace('{series}', series) spec = spec.replace('{series}', series)
# software-properties package for bionic properly reacts to proxy settings # software-properties package for bionic properly reacts to proxy settings
# passed as environment variables (See lp:1433761). This is not the case # set via apt.conf (see lp:1433761), however this is not the case for LTS
# LTS and non-LTS releases below bionic. # and non-LTS releases before bionic.
if series in ('trusty', 'xenial'):
_run_with_retries(['add-apt-repository', '--yes', spec], _run_with_retries(['add-apt-repository', '--yes', spec],
cmd_env=env_proxy_settings(['https', 'http'])) cmd_env=env_proxy_settings(['https', 'http']))
else:
_run_with_retries(['add-apt-repository', '--yes', spec])
def _add_cloud_pocket(pocket): def _add_cloud_pocket(pocket):
@ -723,7 +745,7 @@ def _verify_is_ubuntu_rel(release, os_release):
def _run_with_retries(cmd, max_retries=CMD_RETRY_COUNT, retry_exitcodes=(1,), def _run_with_retries(cmd, max_retries=CMD_RETRY_COUNT, retry_exitcodes=(1,),
retry_message="", cmd_env=None): retry_message="", cmd_env=None, quiet=False):
"""Run a command and retry until success or max_retries is reached. """Run a command and retry until success or max_retries is reached.
:param cmd: The apt command to run. :param cmd: The apt command to run.
@ -738,11 +760,20 @@ def _run_with_retries(cmd, max_retries=CMD_RETRY_COUNT, retry_exitcodes=(1,),
:type retry_message: str :type retry_message: str
:param: cmd_env: Environment variables to add to the command run. :param: cmd_env: Environment variables to add to the command run.
:type cmd_env: Option[None, Dict[str, str]] :type cmd_env: Option[None, Dict[str, str]]
:param quiet: if True, silence the output of the command from stdout and
stderr
:type quiet: bool
""" """
env = get_apt_dpkg_env() env = get_apt_dpkg_env()
if cmd_env: if cmd_env:
env.update(cmd_env) env.update(cmd_env)
kwargs = {}
if quiet:
devnull = os.devnull if six.PY2 else subprocess.DEVNULL
kwargs['stdout'] = devnull
kwargs['stderr'] = devnull
if not retry_message: if not retry_message:
retry_message = "Failed executing '{}'".format(" ".join(cmd)) retry_message = "Failed executing '{}'".format(" ".join(cmd))
retry_message += ". Will retry in {} seconds".format(CMD_RETRY_DELAY) retry_message += ". Will retry in {} seconds".format(CMD_RETRY_DELAY)
@ -753,7 +784,7 @@ def _run_with_retries(cmd, max_retries=CMD_RETRY_COUNT, retry_exitcodes=(1,),
retry_results = (None,) + retry_exitcodes retry_results = (None,) + retry_exitcodes
while result in retry_results: while result in retry_results:
try: try:
result = subprocess.check_call(cmd, env=env) result = subprocess.check_call(cmd, env=env, **kwargs)
except subprocess.CalledProcessError as e: except subprocess.CalledProcessError as e:
retry_count = retry_count + 1 retry_count = retry_count + 1
if retry_count > max_retries: if retry_count > max_retries:
@ -763,7 +794,7 @@ def _run_with_retries(cmd, max_retries=CMD_RETRY_COUNT, retry_exitcodes=(1,),
time.sleep(CMD_RETRY_DELAY) time.sleep(CMD_RETRY_DELAY)
def _run_apt_command(cmd, fatal=False): def _run_apt_command(cmd, fatal=False, quiet=False):
"""Run an apt command with optional retries. """Run an apt command with optional retries.
:param cmd: The apt command to run. :param cmd: The apt command to run.
@ -771,13 +802,22 @@ def _run_apt_command(cmd, fatal=False):
:param fatal: Whether the command's output should be checked and :param fatal: Whether the command's output should be checked and
retried. retried.
:type fatal: bool :type fatal: bool
:param quiet: if True, silence the output of the command from stdout and
stderr
:type quiet: bool
""" """
if fatal: if fatal:
_run_with_retries( _run_with_retries(
cmd, retry_exitcodes=(1, APT_NO_LOCK,), cmd, retry_exitcodes=(1, APT_NO_LOCK,),
retry_message="Couldn't acquire DPKG lock") retry_message="Couldn't acquire DPKG lock",
quiet=quiet)
else: else:
subprocess.call(cmd, env=get_apt_dpkg_env()) kwargs = {}
if quiet:
devnull = os.devnull if six.PY2 else subprocess.DEVNULL
kwargs['stdout'] = devnull
kwargs['stderr'] = devnull
subprocess.call(cmd, env=get_apt_dpkg_env(), **kwargs)
def get_upstream_version(package): def get_upstream_version(package):
@ -799,6 +839,22 @@ def get_upstream_version(package):
return ubuntu_apt_pkg.upstream_version(pkg.current_ver.ver_str) return ubuntu_apt_pkg.upstream_version(pkg.current_ver.ver_str)
def get_installed_version(package):
"""Determine installed version of a package
@returns None (if not installed) or the installed version as
Version object
"""
cache = apt_cache()
dpkg_result = cache._dpkg_list([package]).get(package, {})
current_ver = None
installed_version = dpkg_result.get('version')
if installed_version:
current_ver = ubuntu_apt_pkg.Version({'ver_str': installed_version})
return current_ver
def get_apt_dpkg_env(): def get_apt_dpkg_env():
"""Get environment suitable for execution of APT and DPKG tools. """Get environment suitable for execution of APT and DPKG tools.

View File

@ -42,8 +42,8 @@ oslo.utils<=3.41.0;python_version<'3.6'
coverage>=4.5.2 coverage>=4.5.2
pyudev # for ceph-* charm unit tests (need to fix the ceph-* charm unit tests/mocking) pyudev # for ceph-* charm unit tests (need to fix the ceph-* charm unit tests/mocking)
git+https://github.com/openstack-charmers/zaza.git#egg=zaza;python_version>='3.0' git+https://github.com/openstack-charmers/zaza.git@stable/21.04#egg=zaza;python_version>='3.0'
git+https://github.com/openstack-charmers/zaza-openstack-tests.git#egg=zaza.openstack git+https://github.com/openstack-charmers/zaza-openstack-tests.git@stable/21.04#egg=zaza.openstack
# Needed for charm-glance: # Needed for charm-glance:
git+https://opendev.org/openstack/tempest.git#egg=tempest;python_version>='3.6' git+https://opendev.org/openstack/tempest.git#egg=tempest;python_version>='3.6'