liberty changes
This commit is contained in:
parent
b0c9c6b92f
commit
0bb7aad526
@ -51,7 +51,8 @@ class AmuletDeployment(object):
|
||||
if 'units' not in this_service:
|
||||
this_service['units'] = 1
|
||||
|
||||
self.d.add(this_service['name'], units=this_service['units'])
|
||||
self.d.add(this_service['name'], units=this_service['units'],
|
||||
constraints=this_service.get('constraints'))
|
||||
|
||||
for svc in other_services:
|
||||
if 'location' in svc:
|
||||
@ -64,7 +65,8 @@ class AmuletDeployment(object):
|
||||
if 'units' not in svc:
|
||||
svc['units'] = 1
|
||||
|
||||
self.d.add(svc['name'], charm=branch_location, units=svc['units'])
|
||||
self.d.add(svc['name'], charm=branch_location, units=svc['units'],
|
||||
constraints=svc.get('constraints'))
|
||||
|
||||
def _add_relations(self, relations):
|
||||
"""Add all of the relations for the services."""
|
||||
|
@ -14,17 +14,25 @@
|
||||
# You should have received a copy of the GNU Lesser General Public License
|
||||
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import amulet
|
||||
import ConfigParser
|
||||
import distro_info
|
||||
import io
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import six
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urlparse
|
||||
import uuid
|
||||
|
||||
import amulet
|
||||
import distro_info
|
||||
import six
|
||||
from six.moves import configparser
|
||||
if six.PY3:
|
||||
from urllib import parse as urlparse
|
||||
else:
|
||||
import urlparse
|
||||
|
||||
|
||||
class AmuletUtils(object):
|
||||
@ -108,7 +116,7 @@ class AmuletUtils(object):
|
||||
# /!\ DEPRECATION WARNING (beisner):
|
||||
# New and existing tests should be rewritten to use
|
||||
# validate_services_by_name() as it is aware of init systems.
|
||||
self.log.warn('/!\\ DEPRECATION WARNING: use '
|
||||
self.log.warn('DEPRECATION WARNING: use '
|
||||
'validate_services_by_name instead of validate_services '
|
||||
'due to init system differences.')
|
||||
|
||||
@ -142,19 +150,23 @@ class AmuletUtils(object):
|
||||
|
||||
for service_name in services_list:
|
||||
if (self.ubuntu_releases.index(release) >= systemd_switch or
|
||||
service_name == "rabbitmq-server"):
|
||||
# init is systemd
|
||||
service_name in ['rabbitmq-server', 'apache2']):
|
||||
# init is systemd (or regular sysv)
|
||||
cmd = 'sudo service {} status'.format(service_name)
|
||||
output, code = sentry_unit.run(cmd)
|
||||
service_running = code == 0
|
||||
elif self.ubuntu_releases.index(release) < systemd_switch:
|
||||
# init is upstart
|
||||
cmd = 'sudo status {}'.format(service_name)
|
||||
output, code = sentry_unit.run(cmd)
|
||||
service_running = code == 0 and "start/running" in output
|
||||
|
||||
output, code = sentry_unit.run(cmd)
|
||||
self.log.debug('{} `{}` returned '
|
||||
'{}'.format(sentry_unit.info['unit_name'],
|
||||
cmd, code))
|
||||
if code != 0:
|
||||
return "command `{}` returned {}".format(cmd, str(code))
|
||||
if not service_running:
|
||||
return u"command `{}` returned {} {}".format(
|
||||
cmd, output, str(code))
|
||||
return None
|
||||
|
||||
def _get_config(self, unit, filename):
|
||||
@ -164,7 +176,7 @@ class AmuletUtils(object):
|
||||
# NOTE(beisner): by default, ConfigParser does not handle options
|
||||
# with no value, such as the flags used in the mysql my.cnf file.
|
||||
# https://bugs.python.org/issue7005
|
||||
config = ConfigParser.ConfigParser(allow_no_value=True)
|
||||
config = configparser.ConfigParser(allow_no_value=True)
|
||||
config.readfp(io.StringIO(file_contents))
|
||||
return config
|
||||
|
||||
@ -259,33 +271,52 @@ class AmuletUtils(object):
|
||||
"""Get last modification time of directory."""
|
||||
return sentry_unit.directory_stat(directory)['mtime']
|
||||
|
||||
def _get_proc_start_time(self, sentry_unit, service, pgrep_full=False):
|
||||
"""Get process' start time.
|
||||
def _get_proc_start_time(self, sentry_unit, service, pgrep_full=None):
|
||||
"""Get start time of a process based on the last modification time
|
||||
of the /proc/pid directory.
|
||||
|
||||
Determine start time of the process based on the last modification
|
||||
time of the /proc/pid directory. If pgrep_full is True, the process
|
||||
name is matched against the full command line.
|
||||
"""
|
||||
if pgrep_full:
|
||||
cmd = 'pgrep -o -f {}'.format(service)
|
||||
else:
|
||||
cmd = 'pgrep -o {}'.format(service)
|
||||
cmd = cmd + ' | grep -v pgrep || exit 0'
|
||||
cmd_out = sentry_unit.run(cmd)
|
||||
self.log.debug('CMDout: ' + str(cmd_out))
|
||||
if cmd_out[0]:
|
||||
self.log.debug('Pid for %s %s' % (service, str(cmd_out[0])))
|
||||
proc_dir = '/proc/{}'.format(cmd_out[0].strip())
|
||||
return self._get_dir_mtime(sentry_unit, proc_dir)
|
||||
:sentry_unit: The sentry unit to check for the service on
|
||||
:service: service name to look for in process table
|
||||
:pgrep_full: [Deprecated] Use full command line search mode with pgrep
|
||||
:returns: epoch time of service process start
|
||||
:param commands: list of bash commands
|
||||
:param sentry_units: list of sentry unit pointers
|
||||
:returns: None if successful; Failure message otherwise
|
||||
"""
|
||||
if pgrep_full is not None:
|
||||
# /!\ DEPRECATION WARNING (beisner):
|
||||
# No longer implemented, as pidof is now used instead of pgrep.
|
||||
# https://bugs.launchpad.net/charm-helpers/+bug/1474030
|
||||
self.log.warn('DEPRECATION WARNING: pgrep_full bool is no '
|
||||
'longer implemented re: lp 1474030.')
|
||||
|
||||
pid_list = self.get_process_id_list(sentry_unit, service)
|
||||
pid = pid_list[0]
|
||||
proc_dir = '/proc/{}'.format(pid)
|
||||
self.log.debug('Pid for {} on {}: {}'.format(
|
||||
service, sentry_unit.info['unit_name'], pid))
|
||||
|
||||
return self._get_dir_mtime(sentry_unit, proc_dir)
|
||||
|
||||
def service_restarted(self, sentry_unit, service, filename,
|
||||
pgrep_full=False, sleep_time=20):
|
||||
pgrep_full=None, sleep_time=20):
|
||||
"""Check if service was restarted.
|
||||
|
||||
Compare a service's start time vs a file's last modification time
|
||||
(such as a config file for that service) to determine if the service
|
||||
has been restarted.
|
||||
"""
|
||||
# /!\ DEPRECATION WARNING (beisner):
|
||||
# This method is prone to races in that no before-time is known.
|
||||
# Use validate_service_config_changed instead.
|
||||
|
||||
# NOTE(beisner) pgrep_full is no longer implemented, as pidof is now
|
||||
# used instead of pgrep. pgrep_full is still passed through to ensure
|
||||
# deprecation WARNS. lp1474030
|
||||
self.log.warn('DEPRECATION WARNING: use '
|
||||
'validate_service_config_changed instead of '
|
||||
'service_restarted due to known races.')
|
||||
|
||||
time.sleep(sleep_time)
|
||||
if (self._get_proc_start_time(sentry_unit, service, pgrep_full) >=
|
||||
self._get_file_mtime(sentry_unit, filename)):
|
||||
@ -294,78 +325,122 @@ class AmuletUtils(object):
|
||||
return False
|
||||
|
||||
def service_restarted_since(self, sentry_unit, mtime, service,
|
||||
pgrep_full=False, sleep_time=20,
|
||||
retry_count=2):
|
||||
pgrep_full=None, sleep_time=20,
|
||||
retry_count=30, retry_sleep_time=10):
|
||||
"""Check if service was been started after a given time.
|
||||
|
||||
Args:
|
||||
sentry_unit (sentry): The sentry unit to check for the service on
|
||||
mtime (float): The epoch time to check against
|
||||
service (string): service name to look for in process table
|
||||
pgrep_full (boolean): Use full command line search mode with pgrep
|
||||
sleep_time (int): Seconds to sleep before looking for process
|
||||
retry_count (int): If service is not found, how many times to retry
|
||||
pgrep_full: [Deprecated] Use full command line search mode with pgrep
|
||||
sleep_time (int): Initial sleep time (s) before looking for file
|
||||
retry_sleep_time (int): Time (s) to sleep between retries
|
||||
retry_count (int): If file is not found, how many times to retry
|
||||
|
||||
Returns:
|
||||
bool: True if service found and its start time it newer than mtime,
|
||||
False if service is older than mtime or if service was
|
||||
not found.
|
||||
"""
|
||||
self.log.debug('Checking %s restarted since %s' % (service, mtime))
|
||||
# NOTE(beisner) pgrep_full is no longer implemented, as pidof is now
|
||||
# used instead of pgrep. pgrep_full is still passed through to ensure
|
||||
# deprecation WARNS. lp1474030
|
||||
|
||||
unit_name = sentry_unit.info['unit_name']
|
||||
self.log.debug('Checking that %s service restarted since %s on '
|
||||
'%s' % (service, mtime, unit_name))
|
||||
time.sleep(sleep_time)
|
||||
proc_start_time = self._get_proc_start_time(sentry_unit, service,
|
||||
pgrep_full)
|
||||
while retry_count > 0 and not proc_start_time:
|
||||
self.log.debug('No pid file found for service %s, will retry %i '
|
||||
'more times' % (service, retry_count))
|
||||
time.sleep(30)
|
||||
proc_start_time = self._get_proc_start_time(sentry_unit, service,
|
||||
pgrep_full)
|
||||
retry_count = retry_count - 1
|
||||
proc_start_time = None
|
||||
tries = 0
|
||||
while tries <= retry_count and not proc_start_time:
|
||||
try:
|
||||
proc_start_time = self._get_proc_start_time(sentry_unit,
|
||||
service,
|
||||
pgrep_full)
|
||||
self.log.debug('Attempt {} to get {} proc start time on {} '
|
||||
'OK'.format(tries, service, unit_name))
|
||||
except IOError as e:
|
||||
# NOTE(beisner) - race avoidance, proc may not exist yet.
|
||||
# https://bugs.launchpad.net/charm-helpers/+bug/1474030
|
||||
self.log.debug('Attempt {} to get {} proc start time on {} '
|
||||
'failed\n{}'.format(tries, service,
|
||||
unit_name, e))
|
||||
time.sleep(retry_sleep_time)
|
||||
tries += 1
|
||||
|
||||
if not proc_start_time:
|
||||
self.log.warn('No proc start time found, assuming service did '
|
||||
'not start')
|
||||
return False
|
||||
if proc_start_time >= mtime:
|
||||
self.log.debug('proc start time is newer than provided mtime'
|
||||
'(%s >= %s)' % (proc_start_time, mtime))
|
||||
self.log.debug('Proc start time is newer than provided mtime'
|
||||
'(%s >= %s) on %s (OK)' % (proc_start_time,
|
||||
mtime, unit_name))
|
||||
return True
|
||||
else:
|
||||
self.log.warn('proc start time (%s) is older than provided mtime '
|
||||
'(%s), service did not restart' % (proc_start_time,
|
||||
mtime))
|
||||
self.log.warn('Proc start time (%s) is older than provided mtime '
|
||||
'(%s) on %s, service did not '
|
||||
'restart' % (proc_start_time, mtime, unit_name))
|
||||
return False
|
||||
|
||||
def config_updated_since(self, sentry_unit, filename, mtime,
|
||||
sleep_time=20):
|
||||
sleep_time=20, retry_count=30,
|
||||
retry_sleep_time=10):
|
||||
"""Check if file was modified after a given time.
|
||||
|
||||
Args:
|
||||
sentry_unit (sentry): The sentry unit to check the file mtime on
|
||||
filename (string): The file to check mtime of
|
||||
mtime (float): The epoch time to check against
|
||||
sleep_time (int): Seconds to sleep before looking for process
|
||||
sleep_time (int): Initial sleep time (s) before looking for file
|
||||
retry_sleep_time (int): Time (s) to sleep between retries
|
||||
retry_count (int): If file is not found, how many times to retry
|
||||
|
||||
Returns:
|
||||
bool: True if file was modified more recently than mtime, False if
|
||||
file was modified before mtime,
|
||||
file was modified before mtime, or if file not found.
|
||||
"""
|
||||
self.log.debug('Checking %s updated since %s' % (filename, mtime))
|
||||
unit_name = sentry_unit.info['unit_name']
|
||||
self.log.debug('Checking that %s updated since %s on '
|
||||
'%s' % (filename, mtime, unit_name))
|
||||
time.sleep(sleep_time)
|
||||
file_mtime = self._get_file_mtime(sentry_unit, filename)
|
||||
file_mtime = None
|
||||
tries = 0
|
||||
while tries <= retry_count and not file_mtime:
|
||||
try:
|
||||
file_mtime = self._get_file_mtime(sentry_unit, filename)
|
||||
self.log.debug('Attempt {} to get {} file mtime on {} '
|
||||
'OK'.format(tries, filename, unit_name))
|
||||
except IOError as e:
|
||||
# NOTE(beisner) - race avoidance, file may not exist yet.
|
||||
# https://bugs.launchpad.net/charm-helpers/+bug/1474030
|
||||
self.log.debug('Attempt {} to get {} file mtime on {} '
|
||||
'failed\n{}'.format(tries, filename,
|
||||
unit_name, e))
|
||||
time.sleep(retry_sleep_time)
|
||||
tries += 1
|
||||
|
||||
if not file_mtime:
|
||||
self.log.warn('Could not determine file mtime, assuming '
|
||||
'file does not exist')
|
||||
return False
|
||||
|
||||
if file_mtime >= mtime:
|
||||
self.log.debug('File mtime is newer than provided mtime '
|
||||
'(%s >= %s)' % (file_mtime, mtime))
|
||||
'(%s >= %s) on %s (OK)' % (file_mtime,
|
||||
mtime, unit_name))
|
||||
return True
|
||||
else:
|
||||
self.log.warn('File mtime %s is older than provided mtime %s'
|
||||
% (file_mtime, mtime))
|
||||
self.log.warn('File mtime is older than provided mtime'
|
||||
'(%s < on %s) on %s' % (file_mtime,
|
||||
mtime, unit_name))
|
||||
return False
|
||||
|
||||
def validate_service_config_changed(self, sentry_unit, mtime, service,
|
||||
filename, pgrep_full=False,
|
||||
sleep_time=20, retry_count=2):
|
||||
filename, pgrep_full=None,
|
||||
sleep_time=20, retry_count=30,
|
||||
retry_sleep_time=10):
|
||||
"""Check service and file were updated after mtime
|
||||
|
||||
Args:
|
||||
@ -373,9 +448,10 @@ class AmuletUtils(object):
|
||||
mtime (float): The epoch time to check against
|
||||
service (string): service name to look for in process table
|
||||
filename (string): The file to check mtime of
|
||||
pgrep_full (boolean): Use full command line search mode with pgrep
|
||||
sleep_time (int): Seconds to sleep before looking for process
|
||||
pgrep_full: [Deprecated] Use full command line search mode with pgrep
|
||||
sleep_time (int): Initial sleep in seconds to pass to test helpers
|
||||
retry_count (int): If service is not found, how many times to retry
|
||||
retry_sleep_time (int): Time in seconds to wait between retries
|
||||
|
||||
Typical Usage:
|
||||
u = OpenStackAmuletUtils(ERROR)
|
||||
@ -392,15 +468,27 @@ class AmuletUtils(object):
|
||||
mtime, False if service is older than mtime or if service was
|
||||
not found or if filename was modified before mtime.
|
||||
"""
|
||||
self.log.debug('Checking %s restarted since %s' % (service, mtime))
|
||||
time.sleep(sleep_time)
|
||||
service_restart = self.service_restarted_since(sentry_unit, mtime,
|
||||
service,
|
||||
pgrep_full=pgrep_full,
|
||||
sleep_time=0,
|
||||
retry_count=retry_count)
|
||||
config_update = self.config_updated_since(sentry_unit, filename, mtime,
|
||||
sleep_time=0)
|
||||
|
||||
# NOTE(beisner) pgrep_full is no longer implemented, as pidof is now
|
||||
# used instead of pgrep. pgrep_full is still passed through to ensure
|
||||
# deprecation WARNS. lp1474030
|
||||
|
||||
service_restart = self.service_restarted_since(
|
||||
sentry_unit, mtime,
|
||||
service,
|
||||
pgrep_full=pgrep_full,
|
||||
sleep_time=sleep_time,
|
||||
retry_count=retry_count,
|
||||
retry_sleep_time=retry_sleep_time)
|
||||
|
||||
config_update = self.config_updated_since(
|
||||
sentry_unit,
|
||||
filename,
|
||||
mtime,
|
||||
sleep_time=sleep_time,
|
||||
retry_count=retry_count,
|
||||
retry_sleep_time=retry_sleep_time)
|
||||
|
||||
return service_restart and config_update
|
||||
|
||||
def get_sentry_time(self, sentry_unit):
|
||||
@ -418,7 +506,6 @@ class AmuletUtils(object):
|
||||
"""Return a list of all Ubuntu releases in order of release."""
|
||||
_d = distro_info.UbuntuDistroInfo()
|
||||
_release_list = _d.all
|
||||
self.log.debug('Ubuntu release list: {}'.format(_release_list))
|
||||
return _release_list
|
||||
|
||||
def file_to_url(self, file_rel_path):
|
||||
@ -450,15 +537,20 @@ class AmuletUtils(object):
|
||||
cmd, code, output))
|
||||
return None
|
||||
|
||||
def get_process_id_list(self, sentry_unit, process_name):
|
||||
def get_process_id_list(self, sentry_unit, process_name,
|
||||
expect_success=True):
|
||||
"""Get a list of process ID(s) from a single sentry juju unit
|
||||
for a single process name.
|
||||
|
||||
:param sentry_unit: Pointer to amulet sentry instance (juju unit)
|
||||
:param sentry_unit: Amulet sentry instance (juju unit)
|
||||
:param process_name: Process name
|
||||
:param expect_success: If False, expect the PID to be missing,
|
||||
raise if it is present.
|
||||
:returns: List of process IDs
|
||||
"""
|
||||
cmd = 'pidof {}'.format(process_name)
|
||||
cmd = 'pidof -x {}'.format(process_name)
|
||||
if not expect_success:
|
||||
cmd += " || exit 0 && exit 1"
|
||||
output, code = sentry_unit.run(cmd)
|
||||
if code != 0:
|
||||
msg = ('{} `{}` returned {} '
|
||||
@ -467,14 +559,23 @@ class AmuletUtils(object):
|
||||
amulet.raise_status(amulet.FAIL, msg=msg)
|
||||
return str(output).split()
|
||||
|
||||
def get_unit_process_ids(self, unit_processes):
|
||||
def get_unit_process_ids(self, unit_processes, expect_success=True):
|
||||
"""Construct a dict containing unit sentries, process names, and
|
||||
process IDs."""
|
||||
process IDs.
|
||||
|
||||
:param unit_processes: A dictionary of Amulet sentry instance
|
||||
to list of process names.
|
||||
:param expect_success: if False expect the processes to not be
|
||||
running, raise if they are.
|
||||
:returns: Dictionary of Amulet sentry instance to dictionary
|
||||
of process names to PIDs.
|
||||
"""
|
||||
pid_dict = {}
|
||||
for sentry_unit, process_list in unit_processes.iteritems():
|
||||
for sentry_unit, process_list in six.iteritems(unit_processes):
|
||||
pid_dict[sentry_unit] = {}
|
||||
for process in process_list:
|
||||
pids = self.get_process_id_list(sentry_unit, process)
|
||||
pids = self.get_process_id_list(
|
||||
sentry_unit, process, expect_success=expect_success)
|
||||
pid_dict[sentry_unit].update({process: pids})
|
||||
return pid_dict
|
||||
|
||||
@ -488,7 +589,7 @@ class AmuletUtils(object):
|
||||
return ('Unit count mismatch. expected, actual: {}, '
|
||||
'{} '.format(len(expected), len(actual)))
|
||||
|
||||
for (e_sentry, e_proc_names) in expected.iteritems():
|
||||
for (e_sentry, e_proc_names) in six.iteritems(expected):
|
||||
e_sentry_name = e_sentry.info['unit_name']
|
||||
if e_sentry in actual.keys():
|
||||
a_proc_names = actual[e_sentry]
|
||||
@ -507,11 +608,23 @@ class AmuletUtils(object):
|
||||
'{}'.format(e_proc_name, a_proc_name))
|
||||
|
||||
a_pids_length = len(a_pids)
|
||||
if e_pids_length != a_pids_length:
|
||||
return ('PID count mismatch. {} ({}) expected, actual: '
|
||||
fail_msg = ('PID count mismatch. {} ({}) expected, actual: '
|
||||
'{}, {} ({})'.format(e_sentry_name, e_proc_name,
|
||||
e_pids_length, a_pids_length,
|
||||
a_pids))
|
||||
|
||||
# If expected is not bool, ensure PID quantities match
|
||||
if not isinstance(e_pids_length, bool) and \
|
||||
a_pids_length != e_pids_length:
|
||||
return fail_msg
|
||||
# If expected is bool True, ensure 1 or more PIDs exist
|
||||
elif isinstance(e_pids_length, bool) and \
|
||||
e_pids_length is True and a_pids_length < 1:
|
||||
return fail_msg
|
||||
# If expected is bool False, ensure 0 PIDs exist
|
||||
elif isinstance(e_pids_length, bool) and \
|
||||
e_pids_length is False and a_pids_length != 0:
|
||||
return fail_msg
|
||||
else:
|
||||
self.log.debug('PID check OK: {} {} {}: '
|
||||
'{}'.format(e_sentry_name, e_proc_name,
|
||||
@ -531,3 +644,175 @@ class AmuletUtils(object):
|
||||
return 'Dicts within list are not identical'
|
||||
|
||||
return None
|
||||
|
||||
def validate_sectionless_conf(self, file_contents, expected):
|
||||
"""A crude conf parser. Useful to inspect configuration files which
|
||||
do not have section headers (as would be necessary in order to use
|
||||
the configparser). Such as openstack-dashboard or rabbitmq confs."""
|
||||
for line in file_contents.split('\n'):
|
||||
if '=' in line:
|
||||
args = line.split('=')
|
||||
if len(args) <= 1:
|
||||
continue
|
||||
key = args[0].strip()
|
||||
value = args[1].strip()
|
||||
if key in expected.keys():
|
||||
if expected[key] != value:
|
||||
msg = ('Config mismatch. Expected, actual: {}, '
|
||||
'{}'.format(expected[key], value))
|
||||
amulet.raise_status(amulet.FAIL, msg=msg)
|
||||
|
||||
def get_unit_hostnames(self, units):
|
||||
"""Return a dict of juju unit names to hostnames."""
|
||||
host_names = {}
|
||||
for unit in units:
|
||||
host_names[unit.info['unit_name']] = \
|
||||
str(unit.file_contents('/etc/hostname').strip())
|
||||
self.log.debug('Unit host names: {}'.format(host_names))
|
||||
return host_names
|
||||
|
||||
def run_cmd_unit(self, sentry_unit, cmd):
|
||||
"""Run a command on a unit, return the output and exit code."""
|
||||
output, code = sentry_unit.run(cmd)
|
||||
if code == 0:
|
||||
self.log.debug('{} `{}` command returned {} '
|
||||
'(OK)'.format(sentry_unit.info['unit_name'],
|
||||
cmd, code))
|
||||
else:
|
||||
msg = ('{} `{}` command returned {} '
|
||||
'{}'.format(sentry_unit.info['unit_name'],
|
||||
cmd, code, output))
|
||||
amulet.raise_status(amulet.FAIL, msg=msg)
|
||||
return str(output), code
|
||||
|
||||
def file_exists_on_unit(self, sentry_unit, file_name):
|
||||
"""Check if a file exists on a unit."""
|
||||
try:
|
||||
sentry_unit.file_stat(file_name)
|
||||
return True
|
||||
except IOError:
|
||||
return False
|
||||
except Exception as e:
|
||||
msg = 'Error checking file {}: {}'.format(file_name, e)
|
||||
amulet.raise_status(amulet.FAIL, msg=msg)
|
||||
|
||||
def file_contents_safe(self, sentry_unit, file_name,
|
||||
max_wait=60, fatal=False):
|
||||
"""Get file contents from a sentry unit. Wrap amulet file_contents
|
||||
with retry logic to address races where a file checks as existing,
|
||||
but no longer exists by the time file_contents is called.
|
||||
Return None if file not found. Optionally raise if fatal is True."""
|
||||
unit_name = sentry_unit.info['unit_name']
|
||||
file_contents = False
|
||||
tries = 0
|
||||
while not file_contents and tries < (max_wait / 4):
|
||||
try:
|
||||
file_contents = sentry_unit.file_contents(file_name)
|
||||
except IOError:
|
||||
self.log.debug('Attempt {} to open file {} from {} '
|
||||
'failed'.format(tries, file_name,
|
||||
unit_name))
|
||||
time.sleep(4)
|
||||
tries += 1
|
||||
|
||||
if file_contents:
|
||||
return file_contents
|
||||
elif not fatal:
|
||||
return None
|
||||
elif fatal:
|
||||
msg = 'Failed to get file contents from unit.'
|
||||
amulet.raise_status(amulet.FAIL, msg)
|
||||
|
||||
def port_knock_tcp(self, host="localhost", port=22, timeout=15):
|
||||
"""Open a TCP socket to check for a listening sevice on a host.
|
||||
|
||||
:param host: host name or IP address, default to localhost
|
||||
:param port: TCP port number, default to 22
|
||||
:param timeout: Connect timeout, default to 15 seconds
|
||||
:returns: True if successful, False if connect failed
|
||||
"""
|
||||
|
||||
# Resolve host name if possible
|
||||
try:
|
||||
connect_host = socket.gethostbyname(host)
|
||||
host_human = "{} ({})".format(connect_host, host)
|
||||
except socket.error as e:
|
||||
self.log.warn('Unable to resolve address: '
|
||||
'{} ({}) Trying anyway!'.format(host, e))
|
||||
connect_host = host
|
||||
host_human = connect_host
|
||||
|
||||
# Attempt socket connection
|
||||
try:
|
||||
knock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
knock.settimeout(timeout)
|
||||
knock.connect((connect_host, port))
|
||||
knock.close()
|
||||
self.log.debug('Socket connect OK for host '
|
||||
'{} on port {}.'.format(host_human, port))
|
||||
return True
|
||||
except socket.error as e:
|
||||
self.log.debug('Socket connect FAIL for'
|
||||
' {} port {} ({})'.format(host_human, port, e))
|
||||
return False
|
||||
|
||||
def port_knock_units(self, sentry_units, port=22,
|
||||
timeout=15, expect_success=True):
|
||||
"""Open a TCP socket to check for a listening sevice on each
|
||||
listed juju unit.
|
||||
|
||||
:param sentry_units: list of sentry unit pointers
|
||||
:param port: TCP port number, default to 22
|
||||
:param timeout: Connect timeout, default to 15 seconds
|
||||
:expect_success: True by default, set False to invert logic
|
||||
:returns: None if successful, Failure message otherwise
|
||||
"""
|
||||
for unit in sentry_units:
|
||||
host = unit.info['public-address']
|
||||
connected = self.port_knock_tcp(host, port, timeout)
|
||||
if not connected and expect_success:
|
||||
return 'Socket connect failed.'
|
||||
elif connected and not expect_success:
|
||||
return 'Socket connected unexpectedly.'
|
||||
|
||||
def get_uuid_epoch_stamp(self):
|
||||
"""Returns a stamp string based on uuid4 and epoch time. Useful in
|
||||
generating test messages which need to be unique-ish."""
|
||||
return '[{}-{}]'.format(uuid.uuid4(), time.time())
|
||||
|
||||
# amulet juju action helpers:
|
||||
def run_action(self, unit_sentry, action,
|
||||
_check_output=subprocess.check_output):
|
||||
"""Run the named action on a given unit sentry.
|
||||
|
||||
_check_output parameter is used for dependency injection.
|
||||
|
||||
@return action_id.
|
||||
"""
|
||||
unit_id = unit_sentry.info["unit_name"]
|
||||
command = ["juju", "action", "do", "--format=json", unit_id, action]
|
||||
self.log.info("Running command: %s\n" % " ".join(command))
|
||||
output = _check_output(command, universal_newlines=True)
|
||||
data = json.loads(output)
|
||||
action_id = data[u'Action queued with id']
|
||||
return action_id
|
||||
|
||||
def wait_on_action(self, action_id, _check_output=subprocess.check_output):
|
||||
"""Wait for a given action, returning if it completed or not.
|
||||
|
||||
_check_output parameter is used for dependency injection.
|
||||
"""
|
||||
command = ["juju", "action", "fetch", "--format=json", "--wait=0",
|
||||
action_id]
|
||||
output = _check_output(command, universal_newlines=True)
|
||||
data = json.loads(output)
|
||||
return data.get(u"status") == "completed"
|
||||
|
||||
def status_get(self, unit):
|
||||
"""Return the current service status of this unit."""
|
||||
raw_status, return_code = unit.run(
|
||||
"status-get --format=json --include-data")
|
||||
if return_code != 0:
|
||||
return ("unknown", "")
|
||||
status = json.loads(raw_status)
|
||||
return (status["status"], status["message"])
|
||||
|
@ -148,6 +148,13 @@ define service {{
|
||||
self.description = description
|
||||
self.check_cmd = self._locate_cmd(check_cmd)
|
||||
|
||||
def _get_check_filename(self):
|
||||
return os.path.join(NRPE.nrpe_confdir, '{}.cfg'.format(self.command))
|
||||
|
||||
def _get_service_filename(self, hostname):
|
||||
return os.path.join(NRPE.nagios_exportdir,
|
||||
'service__{}_{}.cfg'.format(hostname, self.command))
|
||||
|
||||
def _locate_cmd(self, check_cmd):
|
||||
search_path = (
|
||||
'/usr/lib/nagios/plugins',
|
||||
@ -163,9 +170,21 @@ define service {{
|
||||
log('Check command not found: {}'.format(parts[0]))
|
||||
return ''
|
||||
|
||||
def _remove_service_files(self):
|
||||
if not os.path.exists(NRPE.nagios_exportdir):
|
||||
return
|
||||
for f in os.listdir(NRPE.nagios_exportdir):
|
||||
if f.endswith('_{}.cfg'.format(self.command)):
|
||||
os.remove(os.path.join(NRPE.nagios_exportdir, f))
|
||||
|
||||
def remove(self, hostname):
|
||||
nrpe_check_file = self._get_check_filename()
|
||||
if os.path.exists(nrpe_check_file):
|
||||
os.remove(nrpe_check_file)
|
||||
self._remove_service_files()
|
||||
|
||||
def write(self, nagios_context, hostname, nagios_servicegroups):
|
||||
nrpe_check_file = '/etc/nagios/nrpe.d/{}.cfg'.format(
|
||||
self.command)
|
||||
nrpe_check_file = self._get_check_filename()
|
||||
with open(nrpe_check_file, 'w') as nrpe_check_config:
|
||||
nrpe_check_config.write("# check {}\n".format(self.shortname))
|
||||
nrpe_check_config.write("command[{}]={}\n".format(
|
||||
@ -180,9 +199,7 @@ define service {{
|
||||
|
||||
def write_service_config(self, nagios_context, hostname,
|
||||
nagios_servicegroups):
|
||||
for f in os.listdir(NRPE.nagios_exportdir):
|
||||
if re.search('.*{}.cfg'.format(self.command), f):
|
||||
os.remove(os.path.join(NRPE.nagios_exportdir, f))
|
||||
self._remove_service_files()
|
||||
|
||||
templ_vars = {
|
||||
'nagios_hostname': hostname,
|
||||
@ -192,8 +209,7 @@ define service {{
|
||||
'command': self.command,
|
||||
}
|
||||
nrpe_service_text = Check.service_template.format(**templ_vars)
|
||||
nrpe_service_file = '{}/service__{}_{}.cfg'.format(
|
||||
NRPE.nagios_exportdir, hostname, self.command)
|
||||
nrpe_service_file = self._get_service_filename(hostname)
|
||||
with open(nrpe_service_file, 'w') as nrpe_service_config:
|
||||
nrpe_service_config.write(str(nrpe_service_text))
|
||||
|
||||
@ -218,12 +234,32 @@ class NRPE(object):
|
||||
if hostname:
|
||||
self.hostname = hostname
|
||||
else:
|
||||
self.hostname = "{}-{}".format(self.nagios_context, self.unit_name)
|
||||
nagios_hostname = get_nagios_hostname()
|
||||
if nagios_hostname:
|
||||
self.hostname = nagios_hostname
|
||||
else:
|
||||
self.hostname = "{}-{}".format(self.nagios_context, self.unit_name)
|
||||
self.checks = []
|
||||
|
||||
def add_check(self, *args, **kwargs):
|
||||
self.checks.append(Check(*args, **kwargs))
|
||||
|
||||
def remove_check(self, *args, **kwargs):
|
||||
if kwargs.get('shortname') is None:
|
||||
raise ValueError('shortname of check must be specified')
|
||||
|
||||
# Use sensible defaults if they're not specified - these are not
|
||||
# actually used during removal, but they're required for constructing
|
||||
# the Check object; check_disk is chosen because it's part of the
|
||||
# nagios-plugins-basic package.
|
||||
if kwargs.get('check_cmd') is None:
|
||||
kwargs['check_cmd'] = 'check_disk'
|
||||
if kwargs.get('description') is None:
|
||||
kwargs['description'] = ''
|
||||
|
||||
check = Check(*args, **kwargs)
|
||||
check.remove(self.hostname)
|
||||
|
||||
def write(self):
|
||||
try:
|
||||
nagios_uid = pwd.getpwnam('nagios').pw_uid
|
||||
@ -260,7 +296,7 @@ def get_nagios_hostcontext(relation_name='nrpe-external-master'):
|
||||
:param str relation_name: Name of relation nrpe sub joined to
|
||||
"""
|
||||
for rel in relations_of_type(relation_name):
|
||||
if 'nagios_hostname' in rel:
|
||||
if 'nagios_host_context' in rel:
|
||||
return rel['nagios_host_context']
|
||||
|
||||
|
||||
@ -301,11 +337,13 @@ def add_init_service_checks(nrpe, services, unit_name):
|
||||
upstart_init = '/etc/init/%s.conf' % svc
|
||||
sysv_init = '/etc/init.d/%s' % svc
|
||||
if os.path.exists(upstart_init):
|
||||
nrpe.add_check(
|
||||
shortname=svc,
|
||||
description='process check {%s}' % unit_name,
|
||||
check_cmd='check_upstart_job %s' % svc
|
||||
)
|
||||
# Don't add a check for these services from neutron-gateway
|
||||
if svc not in ['ext-port', 'os-charm-phy-nic-mtu']:
|
||||
nrpe.add_check(
|
||||
shortname=svc,
|
||||
description='process check {%s}' % unit_name,
|
||||
check_cmd='check_upstart_job %s' % svc
|
||||
)
|
||||
elif os.path.exists(sysv_init):
|
||||
cronpath = '/etc/cron.d/nagios-service-check-%s' % svc
|
||||
cron_file = ('*/5 * * * * root '
|
||||
|
@ -23,7 +23,7 @@ import socket
|
||||
from functools import partial
|
||||
|
||||
from charmhelpers.core.hookenv import unit_get
|
||||
from charmhelpers.fetch import apt_install
|
||||
from charmhelpers.fetch import apt_install, apt_update
|
||||
from charmhelpers.core.hookenv import (
|
||||
log,
|
||||
WARNING,
|
||||
@ -32,13 +32,15 @@ from charmhelpers.core.hookenv import (
|
||||
try:
|
||||
import netifaces
|
||||
except ImportError:
|
||||
apt_install('python-netifaces')
|
||||
apt_update(fatal=True)
|
||||
apt_install('python-netifaces', fatal=True)
|
||||
import netifaces
|
||||
|
||||
try:
|
||||
import netaddr
|
||||
except ImportError:
|
||||
apt_install('python-netaddr')
|
||||
apt_update(fatal=True)
|
||||
apt_install('python-netaddr', fatal=True)
|
||||
import netaddr
|
||||
|
||||
|
||||
@ -51,7 +53,7 @@ def _validate_cidr(network):
|
||||
|
||||
|
||||
def no_ip_found_error_out(network):
|
||||
errmsg = ("No IP address found in network: %s" % network)
|
||||
errmsg = ("No IP address found in network(s): %s" % network)
|
||||
raise ValueError(errmsg)
|
||||
|
||||
|
||||
@ -59,7 +61,7 @@ def get_address_in_network(network, fallback=None, fatal=False):
|
||||
"""Get an IPv4 or IPv6 address within the network from the host.
|
||||
|
||||
:param network (str): CIDR presentation format. For example,
|
||||
'192.168.1.0/24'.
|
||||
'192.168.1.0/24'. Supports multiple networks as a space-delimited list.
|
||||
:param fallback (str): If no address is found, return fallback.
|
||||
:param fatal (boolean): If no address is found, fallback is not
|
||||
set and fatal is True then exit(1).
|
||||
@ -73,24 +75,26 @@ def get_address_in_network(network, fallback=None, fatal=False):
|
||||
else:
|
||||
return None
|
||||
|
||||
_validate_cidr(network)
|
||||
network = netaddr.IPNetwork(network)
|
||||
for iface in netifaces.interfaces():
|
||||
addresses = netifaces.ifaddresses(iface)
|
||||
if network.version == 4 and netifaces.AF_INET in addresses:
|
||||
addr = addresses[netifaces.AF_INET][0]['addr']
|
||||
netmask = addresses[netifaces.AF_INET][0]['netmask']
|
||||
cidr = netaddr.IPNetwork("%s/%s" % (addr, netmask))
|
||||
if cidr in network:
|
||||
return str(cidr.ip)
|
||||
networks = network.split() or [network]
|
||||
for network in networks:
|
||||
_validate_cidr(network)
|
||||
network = netaddr.IPNetwork(network)
|
||||
for iface in netifaces.interfaces():
|
||||
addresses = netifaces.ifaddresses(iface)
|
||||
if network.version == 4 and netifaces.AF_INET in addresses:
|
||||
addr = addresses[netifaces.AF_INET][0]['addr']
|
||||
netmask = addresses[netifaces.AF_INET][0]['netmask']
|
||||
cidr = netaddr.IPNetwork("%s/%s" % (addr, netmask))
|
||||
if cidr in network:
|
||||
return str(cidr.ip)
|
||||
|
||||
if network.version == 6 and netifaces.AF_INET6 in addresses:
|
||||
for addr in addresses[netifaces.AF_INET6]:
|
||||
if not addr['addr'].startswith('fe80'):
|
||||
cidr = netaddr.IPNetwork("%s/%s" % (addr['addr'],
|
||||
addr['netmask']))
|
||||
if cidr in network:
|
||||
return str(cidr.ip)
|
||||
if network.version == 6 and netifaces.AF_INET6 in addresses:
|
||||
for addr in addresses[netifaces.AF_INET6]:
|
||||
if not addr['addr'].startswith('fe80'):
|
||||
cidr = netaddr.IPNetwork("%s/%s" % (addr['addr'],
|
||||
addr['netmask']))
|
||||
if cidr in network:
|
||||
return str(cidr.ip)
|
||||
|
||||
if fallback is not None:
|
||||
return fallback
|
||||
@ -435,8 +439,12 @@ def get_hostname(address, fqdn=True):
|
||||
|
||||
rev = dns.reversename.from_address(address)
|
||||
result = ns_query(rev)
|
||||
|
||||
if not result:
|
||||
return None
|
||||
try:
|
||||
result = socket.gethostbyaddr(address)[0]
|
||||
except:
|
||||
return None
|
||||
else:
|
||||
result = address
|
||||
|
||||
|
@ -40,7 +40,9 @@ Examples:
|
||||
import re
|
||||
import os
|
||||
import subprocess
|
||||
|
||||
from charmhelpers.core import hookenv
|
||||
from charmhelpers.core.kernel import modprobe, is_module_loaded
|
||||
|
||||
__author__ = "Felipe Reyes <felipe.reyes@canonical.com>"
|
||||
|
||||
@ -82,14 +84,11 @@ def is_ipv6_ok(soft_fail=False):
|
||||
# do we have IPv6 in the machine?
|
||||
if os.path.isdir('/proc/sys/net/ipv6'):
|
||||
# is ip6tables kernel module loaded?
|
||||
lsmod = subprocess.check_output(['lsmod'], universal_newlines=True)
|
||||
matches = re.findall('^ip6_tables[ ]+', lsmod, re.M)
|
||||
if len(matches) == 0:
|
||||
if not is_module_loaded('ip6_tables'):
|
||||
# ip6tables support isn't complete, let's try to load it
|
||||
try:
|
||||
subprocess.check_output(['modprobe', 'ip6_tables'],
|
||||
universal_newlines=True)
|
||||
# great, we could load the module
|
||||
modprobe('ip6_tables')
|
||||
# great, we can load the module
|
||||
return True
|
||||
except subprocess.CalledProcessError as ex:
|
||||
hookenv.log("Couldn't load ip6_tables module: %s" % ex.output,
|
||||
|
@ -14,12 +14,18 @@
|
||||
# You should have received a copy of the GNU Lesser General Public License
|
||||
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import logging
|
||||
import re
|
||||
import sys
|
||||
import six
|
||||
from collections import OrderedDict
|
||||
from charmhelpers.contrib.amulet.deployment import (
|
||||
AmuletDeployment
|
||||
)
|
||||
|
||||
DEBUG = logging.DEBUG
|
||||
ERROR = logging.ERROR
|
||||
|
||||
|
||||
class OpenStackAmuletDeployment(AmuletDeployment):
|
||||
"""OpenStack amulet deployment.
|
||||
@ -28,9 +34,12 @@ class OpenStackAmuletDeployment(AmuletDeployment):
|
||||
that is specifically for use by OpenStack charms.
|
||||
"""
|
||||
|
||||
def __init__(self, series=None, openstack=None, source=None, stable=True):
|
||||
def __init__(self, series=None, openstack=None, source=None,
|
||||
stable=True, log_level=DEBUG):
|
||||
"""Initialize the deployment environment."""
|
||||
super(OpenStackAmuletDeployment, self).__init__(series)
|
||||
self.log = self.get_logger(level=log_level)
|
||||
self.log.info('OpenStackAmuletDeployment: init')
|
||||
self.openstack = openstack
|
||||
self.source = source
|
||||
self.stable = stable
|
||||
@ -38,26 +47,55 @@ class OpenStackAmuletDeployment(AmuletDeployment):
|
||||
# out.
|
||||
self.current_next = "trusty"
|
||||
|
||||
def get_logger(self, name="deployment-logger", level=logging.DEBUG):
|
||||
"""Get a logger object that will log to stdout."""
|
||||
log = logging
|
||||
logger = log.getLogger(name)
|
||||
fmt = log.Formatter("%(asctime)s %(funcName)s "
|
||||
"%(levelname)s: %(message)s")
|
||||
|
||||
handler = log.StreamHandler(stream=sys.stdout)
|
||||
handler.setLevel(level)
|
||||
handler.setFormatter(fmt)
|
||||
|
||||
logger.addHandler(handler)
|
||||
logger.setLevel(level)
|
||||
|
||||
return logger
|
||||
|
||||
def _determine_branch_locations(self, other_services):
|
||||
"""Determine the branch locations for the other services.
|
||||
|
||||
Determine if the local branch being tested is derived from its
|
||||
stable or next (dev) branch, and based on this, use the corresonding
|
||||
stable or next branches for the other_services."""
|
||||
base_charms = ['mysql', 'mongodb']
|
||||
|
||||
self.log.info('OpenStackAmuletDeployment: determine branch locations')
|
||||
|
||||
# Charms outside the lp:~openstack-charmers namespace
|
||||
base_charms = ['mysql', 'mongodb', 'nrpe']
|
||||
|
||||
# Force these charms to current series even when using an older series.
|
||||
# ie. Use trusty/nrpe even when series is precise, as the P charm
|
||||
# does not possess the necessary external master config and hooks.
|
||||
force_series_current = ['nrpe']
|
||||
|
||||
if self.series in ['precise', 'trusty']:
|
||||
base_series = self.series
|
||||
else:
|
||||
base_series = self.current_next
|
||||
|
||||
if self.stable:
|
||||
for svc in other_services:
|
||||
for svc in other_services:
|
||||
if svc['name'] in force_series_current:
|
||||
base_series = self.current_next
|
||||
# If a location has been explicitly set, use it
|
||||
if svc.get('location'):
|
||||
continue
|
||||
if self.stable:
|
||||
temp = 'lp:charms/{}/{}'
|
||||
svc['location'] = temp.format(base_series,
|
||||
svc['name'])
|
||||
else:
|
||||
for svc in other_services:
|
||||
else:
|
||||
if svc['name'] in base_charms:
|
||||
temp = 'lp:charms/{}/{}'
|
||||
svc['location'] = temp.format(base_series,
|
||||
@ -66,10 +104,13 @@ class OpenStackAmuletDeployment(AmuletDeployment):
|
||||
temp = 'lp:~openstack-charmers/charms/{}/{}/next'
|
||||
svc['location'] = temp.format(self.current_next,
|
||||
svc['name'])
|
||||
|
||||
return other_services
|
||||
|
||||
def _add_services(self, this_service, other_services):
|
||||
"""Add services to the deployment and set openstack-origin/source."""
|
||||
self.log.info('OpenStackAmuletDeployment: adding services')
|
||||
|
||||
other_services = self._determine_branch_locations(other_services)
|
||||
|
||||
super(OpenStackAmuletDeployment, self)._add_services(this_service,
|
||||
@ -77,29 +118,103 @@ class OpenStackAmuletDeployment(AmuletDeployment):
|
||||
|
||||
services = other_services
|
||||
services.append(this_service)
|
||||
|
||||
# Charms which should use the source config option
|
||||
use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph',
|
||||
'ceph-osd', 'ceph-radosgw']
|
||||
# Most OpenStack subordinate charms do not expose an origin option
|
||||
# as that is controlled by the principle.
|
||||
ignore = ['cinder-ceph', 'hacluster', 'neutron-openvswitch']
|
||||
'ceph-osd', 'ceph-radosgw', 'ceph-mon']
|
||||
|
||||
# Charms which can not use openstack-origin, ie. many subordinates
|
||||
no_origin = ['cinder-ceph', 'hacluster', 'neutron-openvswitch', 'nrpe',
|
||||
'openvswitch-odl', 'neutron-api-odl', 'odl-controller',
|
||||
'cinder-backup']
|
||||
|
||||
if self.openstack:
|
||||
for svc in services:
|
||||
if svc['name'] not in use_source + ignore:
|
||||
if svc['name'] not in use_source + no_origin:
|
||||
config = {'openstack-origin': self.openstack}
|
||||
self.d.configure(svc['name'], config)
|
||||
|
||||
if self.source:
|
||||
for svc in services:
|
||||
if svc['name'] in use_source and svc['name'] not in ignore:
|
||||
if svc['name'] in use_source and svc['name'] not in no_origin:
|
||||
config = {'source': self.source}
|
||||
self.d.configure(svc['name'], config)
|
||||
|
||||
def _configure_services(self, configs):
|
||||
"""Configure all of the services."""
|
||||
self.log.info('OpenStackAmuletDeployment: configure services')
|
||||
for service, config in six.iteritems(configs):
|
||||
self.d.configure(service, config)
|
||||
|
||||
def _auto_wait_for_status(self, message=None, exclude_services=None,
|
||||
include_only=None, timeout=1800):
|
||||
"""Wait for all units to have a specific extended status, except
|
||||
for any defined as excluded. Unless specified via message, any
|
||||
status containing any case of 'ready' will be considered a match.
|
||||
|
||||
Examples of message usage:
|
||||
|
||||
Wait for all unit status to CONTAIN any case of 'ready' or 'ok':
|
||||
message = re.compile('.*ready.*|.*ok.*', re.IGNORECASE)
|
||||
|
||||
Wait for all units to reach this status (exact match):
|
||||
message = re.compile('^Unit is ready and clustered$')
|
||||
|
||||
Wait for all units to reach any one of these (exact match):
|
||||
message = re.compile('Unit is ready|OK|Ready')
|
||||
|
||||
Wait for at least one unit to reach this status (exact match):
|
||||
message = {'ready'}
|
||||
|
||||
See Amulet's sentry.wait_for_messages() for message usage detail.
|
||||
https://github.com/juju/amulet/blob/master/amulet/sentry.py
|
||||
|
||||
:param message: Expected status match
|
||||
:param exclude_services: List of juju service names to ignore,
|
||||
not to be used in conjuction with include_only.
|
||||
:param include_only: List of juju service names to exclusively check,
|
||||
not to be used in conjuction with exclude_services.
|
||||
:param timeout: Maximum time in seconds to wait for status match
|
||||
:returns: None. Raises if timeout is hit.
|
||||
"""
|
||||
self.log.info('Waiting for extended status on units...')
|
||||
|
||||
all_services = self.d.services.keys()
|
||||
|
||||
if exclude_services and include_only:
|
||||
raise ValueError('exclude_services can not be used '
|
||||
'with include_only')
|
||||
|
||||
if message:
|
||||
if isinstance(message, re._pattern_type):
|
||||
match = message.pattern
|
||||
else:
|
||||
match = message
|
||||
|
||||
self.log.debug('Custom extended status wait match: '
|
||||
'{}'.format(match))
|
||||
else:
|
||||
self.log.debug('Default extended status wait match: contains '
|
||||
'READY (case-insensitive)')
|
||||
message = re.compile('.*ready.*', re.IGNORECASE)
|
||||
|
||||
if exclude_services:
|
||||
self.log.debug('Excluding services from extended status match: '
|
||||
'{}'.format(exclude_services))
|
||||
else:
|
||||
exclude_services = []
|
||||
|
||||
if include_only:
|
||||
services = include_only
|
||||
else:
|
||||
services = list(set(all_services) - set(exclude_services))
|
||||
|
||||
self.log.debug('Waiting up to {}s for extended status on services: '
|
||||
'{}'.format(timeout, services))
|
||||
service_messages = {service: message for service in services}
|
||||
self.d.sentry.wait_for_messages(service_messages, timeout=timeout)
|
||||
self.log.info('OK')
|
||||
|
||||
def _get_openstack_release(self):
|
||||
"""Get openstack release.
|
||||
|
||||
@ -111,7 +226,8 @@ class OpenStackAmuletDeployment(AmuletDeployment):
|
||||
self.precise_havana, self.precise_icehouse,
|
||||
self.trusty_icehouse, self.trusty_juno, self.utopic_juno,
|
||||
self.trusty_kilo, self.vivid_kilo, self.trusty_liberty,
|
||||
self.wily_liberty) = range(12)
|
||||
self.wily_liberty, self.trusty_mitaka,
|
||||
self.xenial_mitaka) = range(14)
|
||||
|
||||
releases = {
|
||||
('precise', None): self.precise_essex,
|
||||
@ -123,9 +239,11 @@ class OpenStackAmuletDeployment(AmuletDeployment):
|
||||
('trusty', 'cloud:trusty-juno'): self.trusty_juno,
|
||||
('trusty', 'cloud:trusty-kilo'): self.trusty_kilo,
|
||||
('trusty', 'cloud:trusty-liberty'): self.trusty_liberty,
|
||||
('trusty', 'cloud:trusty-mitaka'): self.trusty_mitaka,
|
||||
('utopic', None): self.utopic_juno,
|
||||
('vivid', None): self.vivid_kilo,
|
||||
('wily', None): self.wily_liberty}
|
||||
('wily', None): self.wily_liberty,
|
||||
('xenial', None): self.xenial_mitaka}
|
||||
return releases[(self.series, self.openstack)]
|
||||
|
||||
def _get_openstack_release_string(self):
|
||||
@ -142,6 +260,7 @@ class OpenStackAmuletDeployment(AmuletDeployment):
|
||||
('utopic', 'juno'),
|
||||
('vivid', 'kilo'),
|
||||
('wily', 'liberty'),
|
||||
('xenial', 'mitaka'),
|
||||
])
|
||||
if self.openstack:
|
||||
os_origin = self.openstack.split(':')[1]
|
||||
|
@ -18,6 +18,7 @@ import amulet
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import six
|
||||
import time
|
||||
import urllib
|
||||
@ -27,6 +28,7 @@ import glanceclient.v1.client as glance_client
|
||||
import heatclient.v1.client as heat_client
|
||||
import keystoneclient.v2_0 as keystone_client
|
||||
import novaclient.v1_1.client as nova_client
|
||||
import pika
|
||||
import swiftclient
|
||||
|
||||
from charmhelpers.contrib.amulet.utils import (
|
||||
@ -602,3 +604,382 @@ class OpenStackAmuletUtils(AmuletUtils):
|
||||
self.log.debug('Ceph {} samples (OK): '
|
||||
'{}'.format(sample_type, samples))
|
||||
return None
|
||||
|
||||
# rabbitmq/amqp specific helpers:
|
||||
|
||||
def rmq_wait_for_cluster(self, deployment, init_sleep=15, timeout=1200):
|
||||
"""Wait for rmq units extended status to show cluster readiness,
|
||||
after an optional initial sleep period. Initial sleep is likely
|
||||
necessary to be effective following a config change, as status
|
||||
message may not instantly update to non-ready."""
|
||||
|
||||
if init_sleep:
|
||||
time.sleep(init_sleep)
|
||||
|
||||
message = re.compile('^Unit is ready and clustered$')
|
||||
deployment._auto_wait_for_status(message=message,
|
||||
timeout=timeout,
|
||||
include_only=['rabbitmq-server'])
|
||||
|
||||
def add_rmq_test_user(self, sentry_units,
|
||||
username="testuser1", password="changeme"):
|
||||
"""Add a test user via the first rmq juju unit, check connection as
|
||||
the new user against all sentry units.
|
||||
|
||||
:param sentry_units: list of sentry unit pointers
|
||||
:param username: amqp user name, default to testuser1
|
||||
:param password: amqp user password
|
||||
:returns: None if successful. Raise on error.
|
||||
"""
|
||||
self.log.debug('Adding rmq user ({})...'.format(username))
|
||||
|
||||
# Check that user does not already exist
|
||||
cmd_user_list = 'rabbitmqctl list_users'
|
||||
output, _ = self.run_cmd_unit(sentry_units[0], cmd_user_list)
|
||||
if username in output:
|
||||
self.log.warning('User ({}) already exists, returning '
|
||||
'gracefully.'.format(username))
|
||||
return
|
||||
|
||||
perms = '".*" ".*" ".*"'
|
||||
cmds = ['rabbitmqctl add_user {} {}'.format(username, password),
|
||||
'rabbitmqctl set_permissions {} {}'.format(username, perms)]
|
||||
|
||||
# Add user via first unit
|
||||
for cmd in cmds:
|
||||
output, _ = self.run_cmd_unit(sentry_units[0], cmd)
|
||||
|
||||
# Check connection against the other sentry_units
|
||||
self.log.debug('Checking user connect against units...')
|
||||
for sentry_unit in sentry_units:
|
||||
connection = self.connect_amqp_by_unit(sentry_unit, ssl=False,
|
||||
username=username,
|
||||
password=password)
|
||||
connection.close()
|
||||
|
||||
def delete_rmq_test_user(self, sentry_units, username="testuser1"):
|
||||
"""Delete a rabbitmq user via the first rmq juju unit.
|
||||
|
||||
:param sentry_units: list of sentry unit pointers
|
||||
:param username: amqp user name, default to testuser1
|
||||
:param password: amqp user password
|
||||
:returns: None if successful or no such user.
|
||||
"""
|
||||
self.log.debug('Deleting rmq user ({})...'.format(username))
|
||||
|
||||
# Check that the user exists
|
||||
cmd_user_list = 'rabbitmqctl list_users'
|
||||
output, _ = self.run_cmd_unit(sentry_units[0], cmd_user_list)
|
||||
|
||||
if username not in output:
|
||||
self.log.warning('User ({}) does not exist, returning '
|
||||
'gracefully.'.format(username))
|
||||
return
|
||||
|
||||
# Delete the user
|
||||
cmd_user_del = 'rabbitmqctl delete_user {}'.format(username)
|
||||
output, _ = self.run_cmd_unit(sentry_units[0], cmd_user_del)
|
||||
|
||||
def get_rmq_cluster_status(self, sentry_unit):
|
||||
"""Execute rabbitmq cluster status command on a unit and return
|
||||
the full output.
|
||||
|
||||
:param unit: sentry unit
|
||||
:returns: String containing console output of cluster status command
|
||||
"""
|
||||
cmd = 'rabbitmqctl cluster_status'
|
||||
output, _ = self.run_cmd_unit(sentry_unit, cmd)
|
||||
self.log.debug('{} cluster_status:\n{}'.format(
|
||||
sentry_unit.info['unit_name'], output))
|
||||
return str(output)
|
||||
|
||||
def get_rmq_cluster_running_nodes(self, sentry_unit):
|
||||
"""Parse rabbitmqctl cluster_status output string, return list of
|
||||
running rabbitmq cluster nodes.
|
||||
|
||||
:param unit: sentry unit
|
||||
:returns: List containing node names of running nodes
|
||||
"""
|
||||
# NOTE(beisner): rabbitmqctl cluster_status output is not
|
||||
# json-parsable, do string chop foo, then json.loads that.
|
||||
str_stat = self.get_rmq_cluster_status(sentry_unit)
|
||||
if 'running_nodes' in str_stat:
|
||||
pos_start = str_stat.find("{running_nodes,") + 15
|
||||
pos_end = str_stat.find("]},", pos_start) + 1
|
||||
str_run_nodes = str_stat[pos_start:pos_end].replace("'", '"')
|
||||
run_nodes = json.loads(str_run_nodes)
|
||||
return run_nodes
|
||||
else:
|
||||
return []
|
||||
|
||||
def validate_rmq_cluster_running_nodes(self, sentry_units):
|
||||
"""Check that all rmq unit hostnames are represented in the
|
||||
cluster_status output of all units.
|
||||
|
||||
:param host_names: dict of juju unit names to host names
|
||||
:param units: list of sentry unit pointers (all rmq units)
|
||||
:returns: None if successful, otherwise return error message
|
||||
"""
|
||||
host_names = self.get_unit_hostnames(sentry_units)
|
||||
errors = []
|
||||
|
||||
# Query every unit for cluster_status running nodes
|
||||
for query_unit in sentry_units:
|
||||
query_unit_name = query_unit.info['unit_name']
|
||||
running_nodes = self.get_rmq_cluster_running_nodes(query_unit)
|
||||
|
||||
# Confirm that every unit is represented in the queried unit's
|
||||
# cluster_status running nodes output.
|
||||
for validate_unit in sentry_units:
|
||||
val_host_name = host_names[validate_unit.info['unit_name']]
|
||||
val_node_name = 'rabbit@{}'.format(val_host_name)
|
||||
|
||||
if val_node_name not in running_nodes:
|
||||
errors.append('Cluster member check failed on {}: {} not '
|
||||
'in {}\n'.format(query_unit_name,
|
||||
val_node_name,
|
||||
running_nodes))
|
||||
if errors:
|
||||
return ''.join(errors)
|
||||
|
||||
def rmq_ssl_is_enabled_on_unit(self, sentry_unit, port=None):
|
||||
"""Check a single juju rmq unit for ssl and port in the config file."""
|
||||
host = sentry_unit.info['public-address']
|
||||
unit_name = sentry_unit.info['unit_name']
|
||||
|
||||
conf_file = '/etc/rabbitmq/rabbitmq.config'
|
||||
conf_contents = str(self.file_contents_safe(sentry_unit,
|
||||
conf_file, max_wait=16))
|
||||
# Checks
|
||||
conf_ssl = 'ssl' in conf_contents
|
||||
conf_port = str(port) in conf_contents
|
||||
|
||||
# Port explicitly checked in config
|
||||
if port and conf_port and conf_ssl:
|
||||
self.log.debug('SSL is enabled @{}:{} '
|
||||
'({})'.format(host, port, unit_name))
|
||||
return True
|
||||
elif port and not conf_port and conf_ssl:
|
||||
self.log.debug('SSL is enabled @{} but not on port {} '
|
||||
'({})'.format(host, port, unit_name))
|
||||
return False
|
||||
# Port not checked (useful when checking that ssl is disabled)
|
||||
elif not port and conf_ssl:
|
||||
self.log.debug('SSL is enabled @{}:{} '
|
||||
'({})'.format(host, port, unit_name))
|
||||
return True
|
||||
elif not conf_ssl:
|
||||
self.log.debug('SSL not enabled @{}:{} '
|
||||
'({})'.format(host, port, unit_name))
|
||||
return False
|
||||
else:
|
||||
msg = ('Unknown condition when checking SSL status @{}:{} '
|
||||
'({})'.format(host, port, unit_name))
|
||||
amulet.raise_status(amulet.FAIL, msg)
|
||||
|
||||
def validate_rmq_ssl_enabled_units(self, sentry_units, port=None):
|
||||
"""Check that ssl is enabled on rmq juju sentry units.
|
||||
|
||||
:param sentry_units: list of all rmq sentry units
|
||||
:param port: optional ssl port override to validate
|
||||
:returns: None if successful, otherwise return error message
|
||||
"""
|
||||
for sentry_unit in sentry_units:
|
||||
if not self.rmq_ssl_is_enabled_on_unit(sentry_unit, port=port):
|
||||
return ('Unexpected condition: ssl is disabled on unit '
|
||||
'({})'.format(sentry_unit.info['unit_name']))
|
||||
return None
|
||||
|
||||
def validate_rmq_ssl_disabled_units(self, sentry_units):
|
||||
"""Check that ssl is enabled on listed rmq juju sentry units.
|
||||
|
||||
:param sentry_units: list of all rmq sentry units
|
||||
:returns: True if successful. Raise on error.
|
||||
"""
|
||||
for sentry_unit in sentry_units:
|
||||
if self.rmq_ssl_is_enabled_on_unit(sentry_unit):
|
||||
return ('Unexpected condition: ssl is enabled on unit '
|
||||
'({})'.format(sentry_unit.info['unit_name']))
|
||||
return None
|
||||
|
||||
def configure_rmq_ssl_on(self, sentry_units, deployment,
|
||||
port=None, max_wait=60):
|
||||
"""Turn ssl charm config option on, with optional non-default
|
||||
ssl port specification. Confirm that it is enabled on every
|
||||
unit.
|
||||
|
||||
:param sentry_units: list of sentry units
|
||||
:param deployment: amulet deployment object pointer
|
||||
:param port: amqp port, use defaults if None
|
||||
:param max_wait: maximum time to wait in seconds to confirm
|
||||
:returns: None if successful. Raise on error.
|
||||
"""
|
||||
self.log.debug('Setting ssl charm config option: on')
|
||||
|
||||
# Enable RMQ SSL
|
||||
config = {'ssl': 'on'}
|
||||
if port:
|
||||
config['ssl_port'] = port
|
||||
|
||||
deployment.d.configure('rabbitmq-server', config)
|
||||
|
||||
# Wait for unit status
|
||||
self.rmq_wait_for_cluster(deployment)
|
||||
|
||||
# Confirm
|
||||
tries = 0
|
||||
ret = self.validate_rmq_ssl_enabled_units(sentry_units, port=port)
|
||||
while ret and tries < (max_wait / 4):
|
||||
time.sleep(4)
|
||||
self.log.debug('Attempt {}: {}'.format(tries, ret))
|
||||
ret = self.validate_rmq_ssl_enabled_units(sentry_units, port=port)
|
||||
tries += 1
|
||||
|
||||
if ret:
|
||||
amulet.raise_status(amulet.FAIL, ret)
|
||||
|
||||
def configure_rmq_ssl_off(self, sentry_units, deployment, max_wait=60):
|
||||
"""Turn ssl charm config option off, confirm that it is disabled
|
||||
on every unit.
|
||||
|
||||
:param sentry_units: list of sentry units
|
||||
:param deployment: amulet deployment object pointer
|
||||
:param max_wait: maximum time to wait in seconds to confirm
|
||||
:returns: None if successful. Raise on error.
|
||||
"""
|
||||
self.log.debug('Setting ssl charm config option: off')
|
||||
|
||||
# Disable RMQ SSL
|
||||
config = {'ssl': 'off'}
|
||||
deployment.d.configure('rabbitmq-server', config)
|
||||
|
||||
# Wait for unit status
|
||||
self.rmq_wait_for_cluster(deployment)
|
||||
|
||||
# Confirm
|
||||
tries = 0
|
||||
ret = self.validate_rmq_ssl_disabled_units(sentry_units)
|
||||
while ret and tries < (max_wait / 4):
|
||||
time.sleep(4)
|
||||
self.log.debug('Attempt {}: {}'.format(tries, ret))
|
||||
ret = self.validate_rmq_ssl_disabled_units(sentry_units)
|
||||
tries += 1
|
||||
|
||||
if ret:
|
||||
amulet.raise_status(amulet.FAIL, ret)
|
||||
|
||||
def connect_amqp_by_unit(self, sentry_unit, ssl=False,
|
||||
port=None, fatal=True,
|
||||
username="testuser1", password="changeme"):
|
||||
"""Establish and return a pika amqp connection to the rabbitmq service
|
||||
running on a rmq juju unit.
|
||||
|
||||
:param sentry_unit: sentry unit pointer
|
||||
:param ssl: boolean, default to False
|
||||
:param port: amqp port, use defaults if None
|
||||
:param fatal: boolean, default to True (raises on connect error)
|
||||
:param username: amqp user name, default to testuser1
|
||||
:param password: amqp user password
|
||||
:returns: pika amqp connection pointer or None if failed and non-fatal
|
||||
"""
|
||||
host = sentry_unit.info['public-address']
|
||||
unit_name = sentry_unit.info['unit_name']
|
||||
|
||||
# Default port logic if port is not specified
|
||||
if ssl and not port:
|
||||
port = 5671
|
||||
elif not ssl and not port:
|
||||
port = 5672
|
||||
|
||||
self.log.debug('Connecting to amqp on {}:{} ({}) as '
|
||||
'{}...'.format(host, port, unit_name, username))
|
||||
|
||||
try:
|
||||
credentials = pika.PlainCredentials(username, password)
|
||||
parameters = pika.ConnectionParameters(host=host, port=port,
|
||||
credentials=credentials,
|
||||
ssl=ssl,
|
||||
connection_attempts=3,
|
||||
retry_delay=5,
|
||||
socket_timeout=1)
|
||||
connection = pika.BlockingConnection(parameters)
|
||||
assert connection.server_properties['product'] == 'RabbitMQ'
|
||||
self.log.debug('Connect OK')
|
||||
return connection
|
||||
except Exception as e:
|
||||
msg = ('amqp connection failed to {}:{} as '
|
||||
'{} ({})'.format(host, port, username, str(e)))
|
||||
if fatal:
|
||||
amulet.raise_status(amulet.FAIL, msg)
|
||||
else:
|
||||
self.log.warn(msg)
|
||||
return None
|
||||
|
||||
def publish_amqp_message_by_unit(self, sentry_unit, message,
|
||||
queue="test", ssl=False,
|
||||
username="testuser1",
|
||||
password="changeme",
|
||||
port=None):
|
||||
"""Publish an amqp message to a rmq juju unit.
|
||||
|
||||
:param sentry_unit: sentry unit pointer
|
||||
:param message: amqp message string
|
||||
:param queue: message queue, default to test
|
||||
:param username: amqp user name, default to testuser1
|
||||
:param password: amqp user password
|
||||
:param ssl: boolean, default to False
|
||||
:param port: amqp port, use defaults if None
|
||||
:returns: None. Raises exception if publish failed.
|
||||
"""
|
||||
self.log.debug('Publishing message to {} queue:\n{}'.format(queue,
|
||||
message))
|
||||
connection = self.connect_amqp_by_unit(sentry_unit, ssl=ssl,
|
||||
port=port,
|
||||
username=username,
|
||||
password=password)
|
||||
|
||||
# NOTE(beisner): extra debug here re: pika hang potential:
|
||||
# https://github.com/pika/pika/issues/297
|
||||
# https://groups.google.com/forum/#!topic/rabbitmq-users/Ja0iyfF0Szw
|
||||
self.log.debug('Defining channel...')
|
||||
channel = connection.channel()
|
||||
self.log.debug('Declaring queue...')
|
||||
channel.queue_declare(queue=queue, auto_delete=False, durable=True)
|
||||
self.log.debug('Publishing message...')
|
||||
channel.basic_publish(exchange='', routing_key=queue, body=message)
|
||||
self.log.debug('Closing channel...')
|
||||
channel.close()
|
||||
self.log.debug('Closing connection...')
|
||||
connection.close()
|
||||
|
||||
def get_amqp_message_by_unit(self, sentry_unit, queue="test",
|
||||
username="testuser1",
|
||||
password="changeme",
|
||||
ssl=False, port=None):
|
||||
"""Get an amqp message from a rmq juju unit.
|
||||
|
||||
:param sentry_unit: sentry unit pointer
|
||||
:param queue: message queue, default to test
|
||||
:param username: amqp user name, default to testuser1
|
||||
:param password: amqp user password
|
||||
:param ssl: boolean, default to False
|
||||
:param port: amqp port, use defaults if None
|
||||
:returns: amqp message body as string. Raise if get fails.
|
||||
"""
|
||||
connection = self.connect_amqp_by_unit(sentry_unit, ssl=ssl,
|
||||
port=port,
|
||||
username=username,
|
||||
password=password)
|
||||
channel = connection.channel()
|
||||
method_frame, _, body = channel.basic_get(queue)
|
||||
|
||||
if method_frame:
|
||||
self.log.debug('Retreived message from {} queue:\n{}'.format(queue,
|
||||
body))
|
||||
channel.basic_ack(method_frame.delivery_tag)
|
||||
channel.close()
|
||||
connection.close()
|
||||
return body
|
||||
else:
|
||||
msg = 'No message retrieved.'
|
||||
amulet.raise_status(amulet.FAIL, msg)
|
||||
|
@ -14,6 +14,7 @@
|
||||
# You should have received a copy of the GNU Lesser General Public License
|
||||
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import glob
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
@ -50,10 +51,13 @@ from charmhelpers.core.sysctl import create as sysctl_create
|
||||
from charmhelpers.core.strutils import bool_from_string
|
||||
|
||||
from charmhelpers.core.host import (
|
||||
get_bond_master,
|
||||
is_phy_iface,
|
||||
list_nics,
|
||||
get_nic_hwaddr,
|
||||
mkdir,
|
||||
write_file,
|
||||
pwgen,
|
||||
)
|
||||
from charmhelpers.contrib.hahelpers.cluster import (
|
||||
determine_apache_port,
|
||||
@ -84,6 +88,14 @@ from charmhelpers.contrib.network.ip import (
|
||||
is_bridge_member,
|
||||
)
|
||||
from charmhelpers.contrib.openstack.utils import get_host_ip
|
||||
from charmhelpers.core.unitdata import kv
|
||||
|
||||
try:
|
||||
import psutil
|
||||
except ImportError:
|
||||
apt_install('python-psutil', fatal=True)
|
||||
import psutil
|
||||
|
||||
CA_CERT_PATH = '/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt'
|
||||
ADDRESS_TYPES = ['admin', 'internal', 'public']
|
||||
|
||||
@ -192,10 +204,50 @@ def config_flags_parser(config_flags):
|
||||
class OSContextGenerator(object):
|
||||
"""Base class for all context generators."""
|
||||
interfaces = []
|
||||
related = False
|
||||
complete = False
|
||||
missing_data = []
|
||||
|
||||
def __call__(self):
|
||||
raise NotImplementedError
|
||||
|
||||
def context_complete(self, ctxt):
|
||||
"""Check for missing data for the required context data.
|
||||
Set self.missing_data if it exists and return False.
|
||||
Set self.complete if no missing data and return True.
|
||||
"""
|
||||
# Fresh start
|
||||
self.complete = False
|
||||
self.missing_data = []
|
||||
for k, v in six.iteritems(ctxt):
|
||||
if v is None or v == '':
|
||||
if k not in self.missing_data:
|
||||
self.missing_data.append(k)
|
||||
|
||||
if self.missing_data:
|
||||
self.complete = False
|
||||
log('Missing required data: %s' % ' '.join(self.missing_data), level=INFO)
|
||||
else:
|
||||
self.complete = True
|
||||
return self.complete
|
||||
|
||||
def get_related(self):
|
||||
"""Check if any of the context interfaces have relation ids.
|
||||
Set self.related and return True if one of the interfaces
|
||||
has relation ids.
|
||||
"""
|
||||
# Fresh start
|
||||
self.related = False
|
||||
try:
|
||||
for interface in self.interfaces:
|
||||
if relation_ids(interface):
|
||||
self.related = True
|
||||
return self.related
|
||||
except AttributeError as e:
|
||||
log("{} {}"
|
||||
"".format(self, e), 'INFO')
|
||||
return self.related
|
||||
|
||||
|
||||
class SharedDBContext(OSContextGenerator):
|
||||
interfaces = ['shared-db']
|
||||
@ -211,6 +263,7 @@ class SharedDBContext(OSContextGenerator):
|
||||
self.database = database
|
||||
self.user = user
|
||||
self.ssl_dir = ssl_dir
|
||||
self.rel_name = self.interfaces[0]
|
||||
|
||||
def __call__(self):
|
||||
self.database = self.database or config('database')
|
||||
@ -244,6 +297,7 @@ class SharedDBContext(OSContextGenerator):
|
||||
password_setting = self.relation_prefix + '_password'
|
||||
|
||||
for rid in relation_ids(self.interfaces[0]):
|
||||
self.related = True
|
||||
for unit in related_units(rid):
|
||||
rdata = relation_get(rid=rid, unit=unit)
|
||||
host = rdata.get('db_host')
|
||||
@ -255,7 +309,7 @@ class SharedDBContext(OSContextGenerator):
|
||||
'database_password': rdata.get(password_setting),
|
||||
'database_type': 'mysql'
|
||||
}
|
||||
if context_complete(ctxt):
|
||||
if self.context_complete(ctxt):
|
||||
db_ssl(rdata, ctxt, self.ssl_dir)
|
||||
return ctxt
|
||||
return {}
|
||||
@ -276,6 +330,7 @@ class PostgresqlDBContext(OSContextGenerator):
|
||||
|
||||
ctxt = {}
|
||||
for rid in relation_ids(self.interfaces[0]):
|
||||
self.related = True
|
||||
for unit in related_units(rid):
|
||||
rel_host = relation_get('host', rid=rid, unit=unit)
|
||||
rel_user = relation_get('user', rid=rid, unit=unit)
|
||||
@ -285,7 +340,7 @@ class PostgresqlDBContext(OSContextGenerator):
|
||||
'database_user': rel_user,
|
||||
'database_password': rel_passwd,
|
||||
'database_type': 'postgresql'}
|
||||
if context_complete(ctxt):
|
||||
if self.context_complete(ctxt):
|
||||
return ctxt
|
||||
|
||||
return {}
|
||||
@ -346,6 +401,7 @@ class IdentityServiceContext(OSContextGenerator):
|
||||
ctxt['signing_dir'] = cachedir
|
||||
|
||||
for rid in relation_ids(self.rel_name):
|
||||
self.related = True
|
||||
for unit in related_units(rid):
|
||||
rdata = relation_get(rid=rid, unit=unit)
|
||||
serv_host = rdata.get('service_host')
|
||||
@ -364,7 +420,7 @@ class IdentityServiceContext(OSContextGenerator):
|
||||
'service_protocol': svc_protocol,
|
||||
'auth_protocol': auth_protocol})
|
||||
|
||||
if context_complete(ctxt):
|
||||
if self.context_complete(ctxt):
|
||||
# NOTE(jamespage) this is required for >= icehouse
|
||||
# so a missing value just indicates keystone needs
|
||||
# upgrading
|
||||
@ -403,6 +459,7 @@ class AMQPContext(OSContextGenerator):
|
||||
ctxt = {}
|
||||
for rid in relation_ids(self.rel_name):
|
||||
ha_vip_only = False
|
||||
self.related = True
|
||||
for unit in related_units(rid):
|
||||
if relation_get('clustered', rid=rid, unit=unit):
|
||||
ctxt['clustered'] = True
|
||||
@ -435,7 +492,7 @@ class AMQPContext(OSContextGenerator):
|
||||
ha_vip_only = relation_get('ha-vip-only',
|
||||
rid=rid, unit=unit) is not None
|
||||
|
||||
if context_complete(ctxt):
|
||||
if self.context_complete(ctxt):
|
||||
if 'rabbit_ssl_ca' in ctxt:
|
||||
if not self.ssl_dir:
|
||||
log("Charm not setup for ssl support but ssl ca "
|
||||
@ -467,7 +524,7 @@ class AMQPContext(OSContextGenerator):
|
||||
ctxt['oslo_messaging_flags'] = config_flags_parser(
|
||||
oslo_messaging_flags)
|
||||
|
||||
if not context_complete(ctxt):
|
||||
if not self.complete:
|
||||
return {}
|
||||
|
||||
return ctxt
|
||||
@ -483,13 +540,15 @@ class CephContext(OSContextGenerator):
|
||||
|
||||
log('Generating template context for ceph', level=DEBUG)
|
||||
mon_hosts = []
|
||||
auth = None
|
||||
key = None
|
||||
use_syslog = str(config('use-syslog')).lower()
|
||||
ctxt = {
|
||||
'use_syslog': str(config('use-syslog')).lower()
|
||||
}
|
||||
for rid in relation_ids('ceph'):
|
||||
for unit in related_units(rid):
|
||||
auth = relation_get('auth', rid=rid, unit=unit)
|
||||
key = relation_get('key', rid=rid, unit=unit)
|
||||
if not ctxt.get('auth'):
|
||||
ctxt['auth'] = relation_get('auth', rid=rid, unit=unit)
|
||||
if not ctxt.get('key'):
|
||||
ctxt['key'] = relation_get('key', rid=rid, unit=unit)
|
||||
ceph_pub_addr = relation_get('ceph-public-address', rid=rid,
|
||||
unit=unit)
|
||||
unit_priv_addr = relation_get('private-address', rid=rid,
|
||||
@ -498,15 +557,12 @@ class CephContext(OSContextGenerator):
|
||||
ceph_addr = format_ipv6_addr(ceph_addr) or ceph_addr
|
||||
mon_hosts.append(ceph_addr)
|
||||
|
||||
ctxt = {'mon_hosts': ' '.join(sorted(mon_hosts)),
|
||||
'auth': auth,
|
||||
'key': key,
|
||||
'use_syslog': use_syslog}
|
||||
ctxt['mon_hosts'] = ' '.join(sorted(mon_hosts))
|
||||
|
||||
if not os.path.isdir('/etc/ceph'):
|
||||
os.mkdir('/etc/ceph')
|
||||
|
||||
if not context_complete(ctxt):
|
||||
if not self.context_complete(ctxt):
|
||||
return {}
|
||||
|
||||
ensure_packages(['ceph-common'])
|
||||
@ -579,15 +635,28 @@ class HAProxyContext(OSContextGenerator):
|
||||
if config('haproxy-client-timeout'):
|
||||
ctxt['haproxy_client_timeout'] = config('haproxy-client-timeout')
|
||||
|
||||
if config('haproxy-queue-timeout'):
|
||||
ctxt['haproxy_queue_timeout'] = config('haproxy-queue-timeout')
|
||||
|
||||
if config('haproxy-connect-timeout'):
|
||||
ctxt['haproxy_connect_timeout'] = config('haproxy-connect-timeout')
|
||||
|
||||
if config('prefer-ipv6'):
|
||||
ctxt['ipv6'] = True
|
||||
ctxt['local_host'] = 'ip6-localhost'
|
||||
ctxt['haproxy_host'] = '::'
|
||||
ctxt['stat_port'] = ':::8888'
|
||||
else:
|
||||
ctxt['local_host'] = '127.0.0.1'
|
||||
ctxt['haproxy_host'] = '0.0.0.0'
|
||||
ctxt['stat_port'] = ':8888'
|
||||
|
||||
ctxt['stat_port'] = '8888'
|
||||
|
||||
db = kv()
|
||||
ctxt['stat_password'] = db.get('stat-password')
|
||||
if not ctxt['stat_password']:
|
||||
ctxt['stat_password'] = db.set('stat-password',
|
||||
pwgen(32))
|
||||
db.flush()
|
||||
|
||||
for frontend in cluster_hosts:
|
||||
if (len(cluster_hosts[frontend]['backends']) > 1 or
|
||||
@ -878,19 +947,6 @@ class NeutronContext(OSContextGenerator):
|
||||
|
||||
return calico_ctxt
|
||||
|
||||
def pg_ctxt(self):
|
||||
driver = neutron_plugin_attribute(self.plugin, 'driver',
|
||||
self.network_manager)
|
||||
config = neutron_plugin_attribute(self.plugin, 'config',
|
||||
self.network_manager)
|
||||
pg_ctxt = {'core_plugin': driver,
|
||||
'neutron_plugin': 'plumgrid',
|
||||
'neutron_security_groups': self.neutron_security_groups,
|
||||
'local_ip': unit_private_ip(),
|
||||
'config': config}
|
||||
|
||||
return pg_ctxt
|
||||
|
||||
def neutron_ctxt(self):
|
||||
if https():
|
||||
proto = 'https'
|
||||
@ -906,6 +962,31 @@ class NeutronContext(OSContextGenerator):
|
||||
'neutron_url': '%s://%s:%s' % (proto, host, '9696')}
|
||||
return ctxt
|
||||
|
||||
def pg_ctxt(self):
|
||||
driver = neutron_plugin_attribute(self.plugin, 'driver',
|
||||
self.network_manager)
|
||||
config = neutron_plugin_attribute(self.plugin, 'config',
|
||||
self.network_manager)
|
||||
ovs_ctxt = {'core_plugin': driver,
|
||||
'neutron_plugin': 'plumgrid',
|
||||
'neutron_security_groups': self.neutron_security_groups,
|
||||
'local_ip': unit_private_ip(),
|
||||
'config': config}
|
||||
return ovs_ctxt
|
||||
|
||||
def midonet_ctxt(self):
|
||||
driver = neutron_plugin_attribute(self.plugin, 'driver',
|
||||
self.network_manager)
|
||||
midonet_config = neutron_plugin_attribute(self.plugin, 'config',
|
||||
self.network_manager)
|
||||
mido_ctxt = {'core_plugin': driver,
|
||||
'neutron_plugin': 'midonet',
|
||||
'neutron_security_groups': self.neutron_security_groups,
|
||||
'local_ip': unit_private_ip(),
|
||||
'config': midonet_config}
|
||||
|
||||
return mido_ctxt
|
||||
|
||||
def __call__(self):
|
||||
if self.network_manager not in ['quantum', 'neutron']:
|
||||
return {}
|
||||
@ -927,6 +1008,8 @@ class NeutronContext(OSContextGenerator):
|
||||
ctxt.update(self.nuage_ctxt())
|
||||
elif self.plugin == 'plumgrid':
|
||||
ctxt.update(self.pg_ctxt())
|
||||
elif self.plugin == 'midonet':
|
||||
ctxt.update(self.midonet_ctxt())
|
||||
|
||||
alchemy_flags = config('neutron-alchemy-flags')
|
||||
if alchemy_flags:
|
||||
@ -938,7 +1021,6 @@ class NeutronContext(OSContextGenerator):
|
||||
|
||||
|
||||
class NeutronPortContext(OSContextGenerator):
|
||||
NIC_PREFIXES = ['eth', 'bond']
|
||||
|
||||
def resolve_ports(self, ports):
|
||||
"""Resolve NICs not yet bound to bridge(s)
|
||||
@ -950,7 +1032,18 @@ class NeutronPortContext(OSContextGenerator):
|
||||
|
||||
hwaddr_to_nic = {}
|
||||
hwaddr_to_ip = {}
|
||||
for nic in list_nics(self.NIC_PREFIXES):
|
||||
for nic in list_nics():
|
||||
# Ignore virtual interfaces (bond masters will be identified from
|
||||
# their slaves)
|
||||
if not is_phy_iface(nic):
|
||||
continue
|
||||
|
||||
_nic = get_bond_master(nic)
|
||||
if _nic:
|
||||
log("Replacing iface '%s' with bond master '%s'" % (nic, _nic),
|
||||
level=DEBUG)
|
||||
nic = _nic
|
||||
|
||||
hwaddr = get_nic_hwaddr(nic)
|
||||
hwaddr_to_nic[hwaddr] = nic
|
||||
addresses = get_ipv4_addr(nic, fatal=False)
|
||||
@ -976,7 +1069,8 @@ class NeutronPortContext(OSContextGenerator):
|
||||
# trust it to be the real external network).
|
||||
resolved.append(entry)
|
||||
|
||||
return resolved
|
||||
# Ensure no duplicates
|
||||
return list(set(resolved))
|
||||
|
||||
|
||||
class OSConfigFlagContext(OSContextGenerator):
|
||||
@ -1016,6 +1110,20 @@ class OSConfigFlagContext(OSContextGenerator):
|
||||
config_flags_parser(config_flags)}
|
||||
|
||||
|
||||
class LibvirtConfigFlagsContext(OSContextGenerator):
|
||||
"""
|
||||
This context provides support for extending
|
||||
the libvirt section through user-defined flags.
|
||||
"""
|
||||
def __call__(self):
|
||||
ctxt = {}
|
||||
libvirt_flags = config('libvirt-flags')
|
||||
if libvirt_flags:
|
||||
ctxt['libvirt_flags'] = config_flags_parser(
|
||||
libvirt_flags)
|
||||
return ctxt
|
||||
|
||||
|
||||
class SubordinateConfigContext(OSContextGenerator):
|
||||
|
||||
"""
|
||||
@ -1048,7 +1156,7 @@ class SubordinateConfigContext(OSContextGenerator):
|
||||
|
||||
ctxt = {
|
||||
... other context ...
|
||||
'subordinate_config': {
|
||||
'subordinate_configuration': {
|
||||
'DEFAULT': {
|
||||
'key1': 'value1',
|
||||
},
|
||||
@ -1066,13 +1174,22 @@ class SubordinateConfigContext(OSContextGenerator):
|
||||
:param config_file : Service's config file to query sections
|
||||
:param interface : Subordinate interface to inspect
|
||||
"""
|
||||
self.service = service
|
||||
self.config_file = config_file
|
||||
self.interface = interface
|
||||
if isinstance(service, list):
|
||||
self.services = service
|
||||
else:
|
||||
self.services = [service]
|
||||
if isinstance(interface, list):
|
||||
self.interfaces = interface
|
||||
else:
|
||||
self.interfaces = [interface]
|
||||
|
||||
def __call__(self):
|
||||
ctxt = {'sections': {}}
|
||||
for rid in relation_ids(self.interface):
|
||||
rids = []
|
||||
for interface in self.interfaces:
|
||||
rids.extend(relation_ids(interface))
|
||||
for rid in rids:
|
||||
for unit in related_units(rid):
|
||||
sub_config = relation_get('subordinate_configuration',
|
||||
rid=rid, unit=unit)
|
||||
@ -1080,33 +1197,37 @@ class SubordinateConfigContext(OSContextGenerator):
|
||||
try:
|
||||
sub_config = json.loads(sub_config)
|
||||
except:
|
||||
log('Could not parse JSON from subordinate_config '
|
||||
'setting from %s' % rid, level=ERROR)
|
||||
log('Could not parse JSON from '
|
||||
'subordinate_configuration setting from %s'
|
||||
% rid, level=ERROR)
|
||||
continue
|
||||
|
||||
if self.service not in sub_config:
|
||||
log('Found subordinate_config on %s but it contained'
|
||||
'nothing for %s service' % (rid, self.service),
|
||||
level=INFO)
|
||||
continue
|
||||
for service in self.services:
|
||||
if service not in sub_config:
|
||||
log('Found subordinate_configuration on %s but it '
|
||||
'contained nothing for %s service'
|
||||
% (rid, service), level=INFO)
|
||||
continue
|
||||
|
||||
sub_config = sub_config[self.service]
|
||||
if self.config_file not in sub_config:
|
||||
log('Found subordinate_config on %s but it contained'
|
||||
'nothing for %s' % (rid, self.config_file),
|
||||
level=INFO)
|
||||
continue
|
||||
|
||||
sub_config = sub_config[self.config_file]
|
||||
for k, v in six.iteritems(sub_config):
|
||||
if k == 'sections':
|
||||
for section, config_dict in six.iteritems(v):
|
||||
log("adding section '%s'" % (section),
|
||||
level=DEBUG)
|
||||
ctxt[k][section] = config_dict
|
||||
else:
|
||||
ctxt[k] = v
|
||||
sub_config = sub_config[service]
|
||||
if self.config_file not in sub_config:
|
||||
log('Found subordinate_configuration on %s but it '
|
||||
'contained nothing for %s'
|
||||
% (rid, self.config_file), level=INFO)
|
||||
continue
|
||||
|
||||
sub_config = sub_config[self.config_file]
|
||||
for k, v in six.iteritems(sub_config):
|
||||
if k == 'sections':
|
||||
for section, config_list in six.iteritems(v):
|
||||
log("adding section '%s'" % (section),
|
||||
level=DEBUG)
|
||||
if ctxt[k].get(section):
|
||||
ctxt[k][section].extend(config_list)
|
||||
else:
|
||||
ctxt[k][section] = config_list
|
||||
else:
|
||||
ctxt[k] = v
|
||||
log("%d section(s) found" % (len(ctxt['sections'])), level=DEBUG)
|
||||
return ctxt
|
||||
|
||||
@ -1143,13 +1264,11 @@ class WorkerConfigContext(OSContextGenerator):
|
||||
|
||||
@property
|
||||
def num_cpus(self):
|
||||
try:
|
||||
from psutil import NUM_CPUS
|
||||
except ImportError:
|
||||
apt_install('python-psutil', fatal=True)
|
||||
from psutil import NUM_CPUS
|
||||
|
||||
return NUM_CPUS
|
||||
# NOTE: use cpu_count if present (16.04 support)
|
||||
if hasattr(psutil, 'cpu_count'):
|
||||
return psutil.cpu_count()
|
||||
else:
|
||||
return psutil.NUM_CPUS
|
||||
|
||||
def __call__(self):
|
||||
multiplier = config('worker-multiplier') or 0
|
||||
@ -1283,15 +1402,19 @@ class DataPortContext(NeutronPortContext):
|
||||
def __call__(self):
|
||||
ports = config('data-port')
|
||||
if ports:
|
||||
# Map of {port/mac:bridge}
|
||||
portmap = parse_data_port_mappings(ports)
|
||||
ports = portmap.values()
|
||||
ports = portmap.keys()
|
||||
# Resolve provided ports or mac addresses and filter out those
|
||||
# already attached to a bridge.
|
||||
resolved = self.resolve_ports(ports)
|
||||
# FIXME: is this necessary?
|
||||
normalized = {get_nic_hwaddr(port): port for port in resolved
|
||||
if port not in ports}
|
||||
normalized.update({port: port for port in resolved
|
||||
if port in ports})
|
||||
if resolved:
|
||||
return {bridge: normalized[port] for bridge, port in
|
||||
return {normalized[port]: bridge for port, bridge in
|
||||
six.iteritems(portmap) if port in normalized.keys()}
|
||||
|
||||
return None
|
||||
@ -1302,12 +1425,22 @@ class PhyNICMTUContext(DataPortContext):
|
||||
def __call__(self):
|
||||
ctxt = {}
|
||||
mappings = super(PhyNICMTUContext, self).__call__()
|
||||
if mappings and mappings.values():
|
||||
ports = mappings.values()
|
||||
if mappings and mappings.keys():
|
||||
ports = sorted(mappings.keys())
|
||||
napi_settings = NeutronAPIContext()()
|
||||
mtu = napi_settings.get('network_device_mtu')
|
||||
all_ports = set()
|
||||
# If any of ports is a vlan device, its underlying device must have
|
||||
# mtu applied first.
|
||||
for port in ports:
|
||||
for lport in glob.glob("/sys/class/net/%s/lower_*" % port):
|
||||
lport = os.path.basename(lport)
|
||||
all_ports.add(lport.split('_')[1])
|
||||
|
||||
all_ports = list(all_ports)
|
||||
all_ports.extend(ports)
|
||||
if mtu:
|
||||
ctxt["devs"] = '\\n'.join(ports)
|
||||
ctxt["devs"] = '\\n'.join(all_ports)
|
||||
ctxt['mtu'] = mtu
|
||||
|
||||
return ctxt
|
||||
@ -1339,6 +1472,6 @@ class NetworkServiceContext(OSContextGenerator):
|
||||
'auth_protocol':
|
||||
rdata.get('auth_protocol') or 'http',
|
||||
}
|
||||
if context_complete(ctxt):
|
||||
if self.context_complete(ctxt):
|
||||
return ctxt
|
||||
return {}
|
||||
|
@ -50,7 +50,7 @@ def determine_dkms_package():
|
||||
if kernel_version() >= (3, 13):
|
||||
return []
|
||||
else:
|
||||
return ['openvswitch-datapath-dkms']
|
||||
return [headers_package(), 'openvswitch-datapath-dkms']
|
||||
|
||||
|
||||
# legacy
|
||||
@ -70,7 +70,7 @@ def quantum_plugins():
|
||||
relation_prefix='neutron',
|
||||
ssl_dir=QUANTUM_CONF_DIR)],
|
||||
'services': ['quantum-plugin-openvswitch-agent'],
|
||||
'packages': [[headers_package()] + determine_dkms_package(),
|
||||
'packages': [determine_dkms_package(),
|
||||
['quantum-plugin-openvswitch-agent']],
|
||||
'server_packages': ['quantum-server',
|
||||
'quantum-plugin-openvswitch'],
|
||||
@ -111,7 +111,7 @@ def neutron_plugins():
|
||||
relation_prefix='neutron',
|
||||
ssl_dir=NEUTRON_CONF_DIR)],
|
||||
'services': ['neutron-plugin-openvswitch-agent'],
|
||||
'packages': [[headers_package()] + determine_dkms_package(),
|
||||
'packages': [determine_dkms_package(),
|
||||
['neutron-plugin-openvswitch-agent']],
|
||||
'server_packages': ['neutron-server',
|
||||
'neutron-plugin-openvswitch'],
|
||||
@ -155,7 +155,7 @@ def neutron_plugins():
|
||||
relation_prefix='neutron',
|
||||
ssl_dir=NEUTRON_CONF_DIR)],
|
||||
'services': [],
|
||||
'packages': [[headers_package()] + determine_dkms_package(),
|
||||
'packages': [determine_dkms_package(),
|
||||
['neutron-plugin-cisco']],
|
||||
'server_packages': ['neutron-server',
|
||||
'neutron-plugin-cisco'],
|
||||
@ -174,7 +174,7 @@ def neutron_plugins():
|
||||
'neutron-dhcp-agent',
|
||||
'nova-api-metadata',
|
||||
'etcd'],
|
||||
'packages': [[headers_package()] + determine_dkms_package(),
|
||||
'packages': [determine_dkms_package(),
|
||||
['calico-compute',
|
||||
'bird',
|
||||
'neutron-dhcp-agent',
|
||||
@ -209,6 +209,20 @@ def neutron_plugins():
|
||||
'server_packages': ['neutron-server',
|
||||
'neutron-plugin-plumgrid'],
|
||||
'server_services': ['neutron-server']
|
||||
},
|
||||
'midonet': {
|
||||
'config': '/etc/neutron/plugins/midonet/midonet.ini',
|
||||
'driver': 'midonet.neutron.plugin.MidonetPluginV2',
|
||||
'contexts': [
|
||||
context.SharedDBContext(user=config('neutron-database-user'),
|
||||
database=config('neutron-database'),
|
||||
relation_prefix='neutron',
|
||||
ssl_dir=NEUTRON_CONF_DIR)],
|
||||
'services': [],
|
||||
'packages': [determine_dkms_package()],
|
||||
'server_packages': ['neutron-server',
|
||||
'python-neutron-plugin-midonet'],
|
||||
'server_services': ['neutron-server']
|
||||
}
|
||||
}
|
||||
if release >= 'icehouse':
|
||||
@ -219,6 +233,14 @@ def neutron_plugins():
|
||||
'neutron-plugin-ml2']
|
||||
# NOTE: patch in vmware renames nvp->nsx for icehouse onwards
|
||||
plugins['nvp'] = plugins['nsx']
|
||||
if release >= 'kilo':
|
||||
plugins['midonet']['driver'] = (
|
||||
'neutron.plugins.midonet.plugin.MidonetPluginV2')
|
||||
if release >= 'liberty':
|
||||
midonet_origin = config('midonet-origin')
|
||||
if midonet_origin is not None and midonet_origin[4:5] == '1':
|
||||
plugins['midonet']['driver'] = (
|
||||
'midonet.neutron.plugin_v1.MidonetPluginV2')
|
||||
return plugins
|
||||
|
||||
|
||||
@ -269,17 +291,30 @@ def network_manager():
|
||||
return 'neutron'
|
||||
|
||||
|
||||
def parse_mappings(mappings):
|
||||
def parse_mappings(mappings, key_rvalue=False):
|
||||
"""By default mappings are lvalue keyed.
|
||||
|
||||
If key_rvalue is True, the mapping will be reversed to allow multiple
|
||||
configs for the same lvalue.
|
||||
"""
|
||||
parsed = {}
|
||||
if mappings:
|
||||
mappings = mappings.split()
|
||||
for m in mappings:
|
||||
p = m.partition(':')
|
||||
key = p[0].strip()
|
||||
if p[1]:
|
||||
parsed[key] = p[2].strip()
|
||||
|
||||
if key_rvalue:
|
||||
key_index = 2
|
||||
val_index = 0
|
||||
# if there is no rvalue skip to next
|
||||
if not p[1]:
|
||||
continue
|
||||
else:
|
||||
parsed[key] = ''
|
||||
key_index = 0
|
||||
val_index = 2
|
||||
|
||||
key = p[key_index].strip()
|
||||
parsed[key] = p[val_index].strip()
|
||||
|
||||
return parsed
|
||||
|
||||
@ -297,25 +332,25 @@ def parse_bridge_mappings(mappings):
|
||||
def parse_data_port_mappings(mappings, default_bridge='br-data'):
|
||||
"""Parse data port mappings.
|
||||
|
||||
Mappings must be a space-delimited list of bridge:port mappings.
|
||||
Mappings must be a space-delimited list of bridge:port.
|
||||
|
||||
Returns dict of the form {bridge:port}.
|
||||
Returns dict of the form {port:bridge} where ports may be mac addresses or
|
||||
interface names.
|
||||
"""
|
||||
_mappings = parse_mappings(mappings)
|
||||
|
||||
# NOTE(dosaboy): we use rvalue for key to allow multiple values to be
|
||||
# proposed for <port> since it may be a mac address which will differ
|
||||
# across units this allowing first-known-good to be chosen.
|
||||
_mappings = parse_mappings(mappings, key_rvalue=True)
|
||||
if not _mappings or list(_mappings.values()) == ['']:
|
||||
if not mappings:
|
||||
return {}
|
||||
|
||||
# For backwards-compatibility we need to support port-only provided in
|
||||
# config.
|
||||
_mappings = {default_bridge: mappings.split()[0]}
|
||||
|
||||
bridges = _mappings.keys()
|
||||
ports = _mappings.values()
|
||||
if len(set(bridges)) != len(bridges):
|
||||
raise Exception("It is not allowed to have more than one port "
|
||||
"configured on the same bridge")
|
||||
_mappings = {mappings.split()[0]: default_bridge}
|
||||
|
||||
ports = _mappings.keys()
|
||||
if len(set(ports)) != len(ports):
|
||||
raise Exception("It is not allowed to have the same port configured "
|
||||
"on more than one bridge")
|
||||
|
@ -18,7 +18,7 @@ import os
|
||||
|
||||
import six
|
||||
|
||||
from charmhelpers.fetch import apt_install
|
||||
from charmhelpers.fetch import apt_install, apt_update
|
||||
from charmhelpers.core.hookenv import (
|
||||
log,
|
||||
ERROR,
|
||||
@ -29,6 +29,7 @@ from charmhelpers.contrib.openstack.utils import OPENSTACK_CODENAMES
|
||||
try:
|
||||
from jinja2 import FileSystemLoader, ChoiceLoader, Environment, exceptions
|
||||
except ImportError:
|
||||
apt_update(fatal=True)
|
||||
apt_install('python-jinja2', fatal=True)
|
||||
from jinja2 import FileSystemLoader, ChoiceLoader, Environment, exceptions
|
||||
|
||||
@ -112,7 +113,7 @@ class OSConfigTemplate(object):
|
||||
|
||||
def complete_contexts(self):
|
||||
'''
|
||||
Return a list of interfaces that have atisfied contexts.
|
||||
Return a list of interfaces that have satisfied contexts.
|
||||
'''
|
||||
if self._complete_contexts:
|
||||
return self._complete_contexts
|
||||
@ -293,3 +294,30 @@ class OSConfigRenderer(object):
|
||||
[interfaces.extend(i.complete_contexts())
|
||||
for i in six.itervalues(self.templates)]
|
||||
return interfaces
|
||||
|
||||
def get_incomplete_context_data(self, interfaces):
|
||||
'''
|
||||
Return dictionary of relation status of interfaces and any missing
|
||||
required context data. Example:
|
||||
{'amqp': {'missing_data': ['rabbitmq_password'], 'related': True},
|
||||
'zeromq-configuration': {'related': False}}
|
||||
'''
|
||||
incomplete_context_data = {}
|
||||
|
||||
for i in six.itervalues(self.templates):
|
||||
for context in i.contexts:
|
||||
for interface in interfaces:
|
||||
related = False
|
||||
if interface in context.interfaces:
|
||||
related = context.get_related()
|
||||
missing_data = context.missing_data
|
||||
if missing_data:
|
||||
incomplete_context_data[interface] = {'missing_data': missing_data}
|
||||
if related:
|
||||
if incomplete_context_data.get(interface):
|
||||
incomplete_context_data[interface].update({'related': True})
|
||||
else:
|
||||
incomplete_context_data[interface] = {'related': True}
|
||||
else:
|
||||
incomplete_context_data[interface] = {'related': False}
|
||||
return incomplete_context_data
|
||||
|
@ -1,5 +1,3 @@
|
||||
#!/usr/bin/python
|
||||
|
||||
# Copyright 2014-2015 Canonical Limited.
|
||||
#
|
||||
# This file is part of charm-helpers.
|
||||
@ -24,8 +22,11 @@ import subprocess
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import re
|
||||
|
||||
import six
|
||||
import traceback
|
||||
import uuid
|
||||
import yaml
|
||||
|
||||
from charmhelpers.contrib.network import ip
|
||||
@ -35,12 +36,17 @@ from charmhelpers.core import (
|
||||
)
|
||||
|
||||
from charmhelpers.core.hookenv import (
|
||||
action_fail,
|
||||
action_set,
|
||||
config,
|
||||
log as juju_log,
|
||||
charm_dir,
|
||||
INFO,
|
||||
related_units,
|
||||
relation_ids,
|
||||
relation_set
|
||||
relation_set,
|
||||
status_set,
|
||||
hook_name
|
||||
)
|
||||
|
||||
from charmhelpers.contrib.storage.linux.lvm import (
|
||||
@ -50,7 +56,8 @@ from charmhelpers.contrib.storage.linux.lvm import (
|
||||
)
|
||||
|
||||
from charmhelpers.contrib.network.ip import (
|
||||
get_ipv6_addr
|
||||
get_ipv6_addr,
|
||||
is_ipv6,
|
||||
)
|
||||
|
||||
from charmhelpers.contrib.python.packages import (
|
||||
@ -69,7 +76,6 @@ CLOUD_ARCHIVE_KEY_ID = '5EDB1B62EC4926EA'
|
||||
DISTRO_PROPOSED = ('deb http://archive.ubuntu.com/ubuntu/ %s-proposed '
|
||||
'restricted main multiverse universe')
|
||||
|
||||
|
||||
UBUNTU_OPENSTACK_RELEASE = OrderedDict([
|
||||
('oneiric', 'diablo'),
|
||||
('precise', 'essex'),
|
||||
@ -80,6 +86,7 @@ UBUNTU_OPENSTACK_RELEASE = OrderedDict([
|
||||
('utopic', 'juno'),
|
||||
('vivid', 'kilo'),
|
||||
('wily', 'liberty'),
|
||||
('xenial', 'mitaka'),
|
||||
])
|
||||
|
||||
|
||||
@ -93,31 +100,73 @@ OPENSTACK_CODENAMES = OrderedDict([
|
||||
('2014.2', 'juno'),
|
||||
('2015.1', 'kilo'),
|
||||
('2015.2', 'liberty'),
|
||||
('2016.1', 'mitaka'),
|
||||
])
|
||||
|
||||
# The ugly duckling
|
||||
# The ugly duckling - must list releases oldest to newest
|
||||
SWIFT_CODENAMES = OrderedDict([
|
||||
('1.4.3', 'diablo'),
|
||||
('1.4.8', 'essex'),
|
||||
('1.7.4', 'folsom'),
|
||||
('1.8.0', 'grizzly'),
|
||||
('1.7.7', 'grizzly'),
|
||||
('1.7.6', 'grizzly'),
|
||||
('1.10.0', 'havana'),
|
||||
('1.9.1', 'havana'),
|
||||
('1.9.0', 'havana'),
|
||||
('1.13.1', 'icehouse'),
|
||||
('1.13.0', 'icehouse'),
|
||||
('1.12.0', 'icehouse'),
|
||||
('1.11.0', 'icehouse'),
|
||||
('2.0.0', 'juno'),
|
||||
('2.1.0', 'juno'),
|
||||
('2.2.0', 'juno'),
|
||||
('2.2.1', 'kilo'),
|
||||
('2.2.2', 'kilo'),
|
||||
('2.3.0', 'liberty'),
|
||||
('diablo',
|
||||
['1.4.3']),
|
||||
('essex',
|
||||
['1.4.8']),
|
||||
('folsom',
|
||||
['1.7.4']),
|
||||
('grizzly',
|
||||
['1.7.6', '1.7.7', '1.8.0']),
|
||||
('havana',
|
||||
['1.9.0', '1.9.1', '1.10.0']),
|
||||
('icehouse',
|
||||
['1.11.0', '1.12.0', '1.13.0', '1.13.1']),
|
||||
('juno',
|
||||
['2.0.0', '2.1.0', '2.2.0']),
|
||||
('kilo',
|
||||
['2.2.1', '2.2.2']),
|
||||
('liberty',
|
||||
['2.3.0', '2.4.0', '2.5.0']),
|
||||
('mitaka',
|
||||
['2.5.0']),
|
||||
])
|
||||
|
||||
# >= Liberty version->codename mapping
|
||||
PACKAGE_CODENAMES = {
|
||||
'nova-common': OrderedDict([
|
||||
('12.0', 'liberty'),
|
||||
('13.0', 'mitaka'),
|
||||
]),
|
||||
'neutron-common': OrderedDict([
|
||||
('7.0', 'liberty'),
|
||||
('8.0', 'mitaka'),
|
||||
]),
|
||||
'cinder-common': OrderedDict([
|
||||
('7.0', 'liberty'),
|
||||
('8.0', 'mitaka'),
|
||||
]),
|
||||
'keystone': OrderedDict([
|
||||
('8.0', 'liberty'),
|
||||
('9.0', 'mitaka'),
|
||||
]),
|
||||
'horizon-common': OrderedDict([
|
||||
('8.0', 'liberty'),
|
||||
('9.0', 'mitaka'),
|
||||
]),
|
||||
'ceilometer-common': OrderedDict([
|
||||
('5.0', 'liberty'),
|
||||
('6.0', 'mitaka'),
|
||||
]),
|
||||
'heat-common': OrderedDict([
|
||||
('5.0', 'liberty'),
|
||||
('6.0', 'mitaka'),
|
||||
]),
|
||||
'glance-common': OrderedDict([
|
||||
('11.0', 'liberty'),
|
||||
('12.0', 'mitaka'),
|
||||
]),
|
||||
'openstack-dashboard': OrderedDict([
|
||||
('8.0', 'liberty'),
|
||||
('9.0', 'mitaka'),
|
||||
]),
|
||||
}
|
||||
|
||||
DEFAULT_LOOPBACK_SIZE = '5G'
|
||||
|
||||
|
||||
@ -167,9 +216,9 @@ def get_os_codename_version(vers):
|
||||
error_out(e)
|
||||
|
||||
|
||||
def get_os_version_codename(codename):
|
||||
def get_os_version_codename(codename, version_map=OPENSTACK_CODENAMES):
|
||||
'''Determine OpenStack version number from codename.'''
|
||||
for k, v in six.iteritems(OPENSTACK_CODENAMES):
|
||||
for k, v in six.iteritems(version_map):
|
||||
if v == codename:
|
||||
return k
|
||||
e = 'Could not derive OpenStack version for '\
|
||||
@ -177,6 +226,33 @@ def get_os_version_codename(codename):
|
||||
error_out(e)
|
||||
|
||||
|
||||
def get_os_version_codename_swift(codename):
|
||||
'''Determine OpenStack version number of swift from codename.'''
|
||||
for k, v in six.iteritems(SWIFT_CODENAMES):
|
||||
if k == codename:
|
||||
return v[-1]
|
||||
e = 'Could not derive swift version for '\
|
||||
'codename: %s' % codename
|
||||
error_out(e)
|
||||
|
||||
|
||||
def get_swift_codename(version):
|
||||
'''Determine OpenStack codename that corresponds to swift version.'''
|
||||
codenames = [k for k, v in six.iteritems(SWIFT_CODENAMES) if version in v]
|
||||
if len(codenames) > 1:
|
||||
# If more than one release codename contains this version we determine
|
||||
# the actual codename based on the highest available install source.
|
||||
for codename in reversed(codenames):
|
||||
releases = UBUNTU_OPENSTACK_RELEASE
|
||||
release = [k for k, v in six.iteritems(releases) if codename in v]
|
||||
ret = subprocess.check_output(['apt-cache', 'policy', 'swift'])
|
||||
if codename in ret or release[0] in ret:
|
||||
return codename
|
||||
elif len(codenames) == 1:
|
||||
return codenames[0]
|
||||
return None
|
||||
|
||||
|
||||
def get_os_codename_package(package, fatal=True):
|
||||
'''Derive OpenStack release codename from an installed package.'''
|
||||
import apt_pkg as apt
|
||||
@ -201,20 +277,33 @@ def get_os_codename_package(package, fatal=True):
|
||||
error_out(e)
|
||||
|
||||
vers = apt.upstream_version(pkg.current_ver.ver_str)
|
||||
if 'swift' in pkg.name:
|
||||
# Fully x.y.z match for swift versions
|
||||
match = re.match('^(\d+)\.(\d+)\.(\d+)', vers)
|
||||
else:
|
||||
# x.y match only for 20XX.X
|
||||
# and ignore patch level for other packages
|
||||
match = re.match('^(\d+)\.(\d+)', vers)
|
||||
|
||||
try:
|
||||
if 'swift' in pkg.name:
|
||||
swift_vers = vers[:5]
|
||||
if swift_vers not in SWIFT_CODENAMES:
|
||||
# Deal with 1.10.0 upward
|
||||
swift_vers = vers[:6]
|
||||
return SWIFT_CODENAMES[swift_vers]
|
||||
else:
|
||||
vers = vers[:6]
|
||||
return OPENSTACK_CODENAMES[vers]
|
||||
except KeyError:
|
||||
e = 'Could not determine OpenStack codename for version %s' % vers
|
||||
error_out(e)
|
||||
if match:
|
||||
vers = match.group(0)
|
||||
|
||||
# >= Liberty independent project versions
|
||||
if (package in PACKAGE_CODENAMES and
|
||||
vers in PACKAGE_CODENAMES[package]):
|
||||
return PACKAGE_CODENAMES[package][vers]
|
||||
else:
|
||||
# < Liberty co-ordinated project versions
|
||||
try:
|
||||
if 'swift' in pkg.name:
|
||||
return get_swift_codename(vers)
|
||||
else:
|
||||
return OPENSTACK_CODENAMES[vers]
|
||||
except KeyError:
|
||||
if not fatal:
|
||||
return None
|
||||
e = 'Could not determine OpenStack codename for version %s' % vers
|
||||
error_out(e)
|
||||
|
||||
|
||||
def get_os_version_package(pkg, fatal=True):
|
||||
@ -226,12 +315,14 @@ def get_os_version_package(pkg, fatal=True):
|
||||
|
||||
if 'swift' in pkg:
|
||||
vers_map = SWIFT_CODENAMES
|
||||
for cname, version in six.iteritems(vers_map):
|
||||
if cname == codename:
|
||||
return version[-1]
|
||||
else:
|
||||
vers_map = OPENSTACK_CODENAMES
|
||||
|
||||
for version, cname in six.iteritems(vers_map):
|
||||
if cname == codename:
|
||||
return version
|
||||
for version, cname in six.iteritems(vers_map):
|
||||
if cname == codename:
|
||||
return version
|
||||
# e = "Could not determine OpenStack version for package: %s" % pkg
|
||||
# error_out(e)
|
||||
|
||||
@ -327,6 +418,9 @@ def configure_installation_source(rel):
|
||||
'liberty': 'trusty-updates/liberty',
|
||||
'liberty/updates': 'trusty-updates/liberty',
|
||||
'liberty/proposed': 'trusty-proposed/liberty',
|
||||
'mitaka': 'trusty-updates/mitaka',
|
||||
'mitaka/updates': 'trusty-updates/mitaka',
|
||||
'mitaka/proposed': 'trusty-proposed/mitaka',
|
||||
}
|
||||
|
||||
try:
|
||||
@ -392,9 +486,18 @@ def openstack_upgrade_available(package):
|
||||
import apt_pkg as apt
|
||||
src = config('openstack-origin')
|
||||
cur_vers = get_os_version_package(package)
|
||||
available_vers = get_os_version_install_source(src)
|
||||
if "swift" in package:
|
||||
codename = get_os_codename_install_source(src)
|
||||
avail_vers = get_os_version_codename_swift(codename)
|
||||
else:
|
||||
avail_vers = get_os_version_install_source(src)
|
||||
apt.init()
|
||||
return apt.version_compare(available_vers, cur_vers) == 1
|
||||
if "swift" in package:
|
||||
major_cur_vers = cur_vers.split('.', 1)[0]
|
||||
major_avail_vers = avail_vers.split('.', 1)[0]
|
||||
major_diff = apt.version_compare(major_avail_vers, major_cur_vers)
|
||||
return avail_vers > cur_vers and (major_diff == 1 or major_diff == 0)
|
||||
return apt.version_compare(avail_vers, cur_vers) == 1
|
||||
|
||||
|
||||
def ensure_block_device(block_device):
|
||||
@ -469,6 +572,12 @@ def sync_db_with_multi_ipv6_addresses(database, database_user,
|
||||
relation_prefix=None):
|
||||
hosts = get_ipv6_addr(dynamic_only=False)
|
||||
|
||||
if config('vip'):
|
||||
vips = config('vip').split()
|
||||
for vip in vips:
|
||||
if vip and is_ipv6(vip):
|
||||
hosts.append(vip)
|
||||
|
||||
kwargs = {'database': database,
|
||||
'username': database_user,
|
||||
'hostname': json.dumps(hosts)}
|
||||
@ -517,7 +626,7 @@ def _git_yaml_load(projects_yaml):
|
||||
return yaml.load(projects_yaml)
|
||||
|
||||
|
||||
def git_clone_and_install(projects_yaml, core_project, depth=1):
|
||||
def git_clone_and_install(projects_yaml, core_project):
|
||||
"""
|
||||
Clone/install all specified OpenStack repositories.
|
||||
|
||||
@ -567,6 +676,9 @@ def git_clone_and_install(projects_yaml, core_project, depth=1):
|
||||
for p in projects['repositories']:
|
||||
repo = p['repository']
|
||||
branch = p['branch']
|
||||
depth = '1'
|
||||
if 'depth' in p.keys():
|
||||
depth = p['depth']
|
||||
if p['name'] == 'requirements':
|
||||
repo_dir = _git_clone_and_install_single(repo, branch, depth,
|
||||
parent_dir, http_proxy,
|
||||
@ -611,19 +723,13 @@ def _git_clone_and_install_single(repo, branch, depth, parent_dir, http_proxy,
|
||||
"""
|
||||
Clone and install a single git repository.
|
||||
"""
|
||||
dest_dir = os.path.join(parent_dir, os.path.basename(repo))
|
||||
|
||||
if not os.path.exists(parent_dir):
|
||||
juju_log('Directory already exists at {}. '
|
||||
'No need to create directory.'.format(parent_dir))
|
||||
os.mkdir(parent_dir)
|
||||
|
||||
if not os.path.exists(dest_dir):
|
||||
juju_log('Cloning git repo: {}, branch: {}'.format(repo, branch))
|
||||
repo_dir = install_remote(repo, dest=parent_dir, branch=branch,
|
||||
depth=depth)
|
||||
else:
|
||||
repo_dir = dest_dir
|
||||
juju_log('Cloning git repo: {}, branch: {}'.format(repo, branch))
|
||||
repo_dir = install_remote(repo, dest=parent_dir, branch=branch, depth=depth)
|
||||
|
||||
venv = os.path.join(parent_dir, 'venv')
|
||||
|
||||
@ -704,3 +810,235 @@ def git_yaml_value(projects_yaml, key):
|
||||
return projects[key]
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def os_workload_status(configs, required_interfaces, charm_func=None):
|
||||
"""
|
||||
Decorator to set workload status based on complete contexts
|
||||
"""
|
||||
def wrap(f):
|
||||
@wraps(f)
|
||||
def wrapped_f(*args, **kwargs):
|
||||
# Run the original function first
|
||||
f(*args, **kwargs)
|
||||
# Set workload status now that contexts have been
|
||||
# acted on
|
||||
set_os_workload_status(configs, required_interfaces, charm_func)
|
||||
return wrapped_f
|
||||
return wrap
|
||||
|
||||
|
||||
def set_os_workload_status(configs, required_interfaces, charm_func=None):
|
||||
"""
|
||||
Set workload status based on complete contexts.
|
||||
status-set missing or incomplete contexts
|
||||
and juju-log details of missing required data.
|
||||
charm_func is a charm specific function to run checking
|
||||
for charm specific requirements such as a VIP setting.
|
||||
"""
|
||||
incomplete_rel_data = incomplete_relation_data(configs, required_interfaces)
|
||||
state = 'active'
|
||||
missing_relations = []
|
||||
incomplete_relations = []
|
||||
message = None
|
||||
charm_state = None
|
||||
charm_message = None
|
||||
|
||||
for generic_interface in incomplete_rel_data.keys():
|
||||
related_interface = None
|
||||
missing_data = {}
|
||||
# Related or not?
|
||||
for interface in incomplete_rel_data[generic_interface]:
|
||||
if incomplete_rel_data[generic_interface][interface].get('related'):
|
||||
related_interface = interface
|
||||
missing_data = incomplete_rel_data[generic_interface][interface].get('missing_data')
|
||||
# No relation ID for the generic_interface
|
||||
if not related_interface:
|
||||
juju_log("{} relation is missing and must be related for "
|
||||
"functionality. ".format(generic_interface), 'WARN')
|
||||
state = 'blocked'
|
||||
if generic_interface not in missing_relations:
|
||||
missing_relations.append(generic_interface)
|
||||
else:
|
||||
# Relation ID exists but no related unit
|
||||
if not missing_data:
|
||||
# Edge case relation ID exists but departing
|
||||
if ('departed' in hook_name() or 'broken' in hook_name()) \
|
||||
and related_interface in hook_name():
|
||||
state = 'blocked'
|
||||
if generic_interface not in missing_relations:
|
||||
missing_relations.append(generic_interface)
|
||||
juju_log("{} relation's interface, {}, "
|
||||
"relationship is departed or broken "
|
||||
"and is required for functionality."
|
||||
"".format(generic_interface, related_interface), "WARN")
|
||||
# Normal case relation ID exists but no related unit
|
||||
# (joining)
|
||||
else:
|
||||
juju_log("{} relations's interface, {}, is related but has "
|
||||
"no units in the relation."
|
||||
"".format(generic_interface, related_interface), "INFO")
|
||||
# Related unit exists and data missing on the relation
|
||||
else:
|
||||
juju_log("{} relation's interface, {}, is related awaiting "
|
||||
"the following data from the relationship: {}. "
|
||||
"".format(generic_interface, related_interface,
|
||||
", ".join(missing_data)), "INFO")
|
||||
if state != 'blocked':
|
||||
state = 'waiting'
|
||||
if generic_interface not in incomplete_relations \
|
||||
and generic_interface not in missing_relations:
|
||||
incomplete_relations.append(generic_interface)
|
||||
|
||||
if missing_relations:
|
||||
message = "Missing relations: {}".format(", ".join(missing_relations))
|
||||
if incomplete_relations:
|
||||
message += "; incomplete relations: {}" \
|
||||
"".format(", ".join(incomplete_relations))
|
||||
state = 'blocked'
|
||||
elif incomplete_relations:
|
||||
message = "Incomplete relations: {}" \
|
||||
"".format(", ".join(incomplete_relations))
|
||||
state = 'waiting'
|
||||
|
||||
# Run charm specific checks
|
||||
if charm_func:
|
||||
charm_state, charm_message = charm_func(configs)
|
||||
if charm_state != 'active' and charm_state != 'unknown':
|
||||
state = workload_state_compare(state, charm_state)
|
||||
if message:
|
||||
charm_message = charm_message.replace("Incomplete relations: ",
|
||||
"")
|
||||
message = "{}, {}".format(message, charm_message)
|
||||
else:
|
||||
message = charm_message
|
||||
|
||||
# Set to active if all requirements have been met
|
||||
if state == 'active':
|
||||
message = "Unit is ready"
|
||||
juju_log(message, "INFO")
|
||||
|
||||
status_set(state, message)
|
||||
|
||||
|
||||
def workload_state_compare(current_workload_state, workload_state):
|
||||
""" Return highest priority of two states"""
|
||||
hierarchy = {'unknown': -1,
|
||||
'active': 0,
|
||||
'maintenance': 1,
|
||||
'waiting': 2,
|
||||
'blocked': 3,
|
||||
}
|
||||
|
||||
if hierarchy.get(workload_state) is None:
|
||||
workload_state = 'unknown'
|
||||
if hierarchy.get(current_workload_state) is None:
|
||||
current_workload_state = 'unknown'
|
||||
|
||||
# Set workload_state based on hierarchy of statuses
|
||||
if hierarchy.get(current_workload_state) > hierarchy.get(workload_state):
|
||||
return current_workload_state
|
||||
else:
|
||||
return workload_state
|
||||
|
||||
|
||||
def incomplete_relation_data(configs, required_interfaces):
|
||||
"""
|
||||
Check complete contexts against required_interfaces
|
||||
Return dictionary of incomplete relation data.
|
||||
|
||||
configs is an OSConfigRenderer object with configs registered
|
||||
|
||||
required_interfaces is a dictionary of required general interfaces
|
||||
with dictionary values of possible specific interfaces.
|
||||
Example:
|
||||
required_interfaces = {'database': ['shared-db', 'pgsql-db']}
|
||||
|
||||
The interface is said to be satisfied if anyone of the interfaces in the
|
||||
list has a complete context.
|
||||
|
||||
Return dictionary of incomplete or missing required contexts with relation
|
||||
status of interfaces and any missing data points. Example:
|
||||
{'message':
|
||||
{'amqp': {'missing_data': ['rabbitmq_password'], 'related': True},
|
||||
'zeromq-configuration': {'related': False}},
|
||||
'identity':
|
||||
{'identity-service': {'related': False}},
|
||||
'database':
|
||||
{'pgsql-db': {'related': False},
|
||||
'shared-db': {'related': True}}}
|
||||
"""
|
||||
complete_ctxts = configs.complete_contexts()
|
||||
incomplete_relations = []
|
||||
for svc_type in required_interfaces.keys():
|
||||
# Avoid duplicates
|
||||
found_ctxt = False
|
||||
for interface in required_interfaces[svc_type]:
|
||||
if interface in complete_ctxts:
|
||||
found_ctxt = True
|
||||
if not found_ctxt:
|
||||
incomplete_relations.append(svc_type)
|
||||
incomplete_context_data = {}
|
||||
for i in incomplete_relations:
|
||||
incomplete_context_data[i] = configs.get_incomplete_context_data(required_interfaces[i])
|
||||
return incomplete_context_data
|
||||
|
||||
|
||||
def do_action_openstack_upgrade(package, upgrade_callback, configs):
|
||||
"""Perform action-managed OpenStack upgrade.
|
||||
|
||||
Upgrades packages to the configured openstack-origin version and sets
|
||||
the corresponding action status as a result.
|
||||
|
||||
If the charm was installed from source we cannot upgrade it.
|
||||
For backwards compatibility a config flag (action-managed-upgrade) must
|
||||
be set for this code to run, otherwise a full service level upgrade will
|
||||
fire on config-changed.
|
||||
|
||||
@param package: package name for determining if upgrade available
|
||||
@param upgrade_callback: function callback to charm's upgrade function
|
||||
@param configs: templating object derived from OSConfigRenderer class
|
||||
|
||||
@return: True if upgrade successful; False if upgrade failed or skipped
|
||||
"""
|
||||
ret = False
|
||||
|
||||
if git_install_requested():
|
||||
action_set({'outcome': 'installed from source, skipped upgrade.'})
|
||||
else:
|
||||
if openstack_upgrade_available(package):
|
||||
if config('action-managed-upgrade'):
|
||||
juju_log('Upgrading OpenStack release')
|
||||
|
||||
try:
|
||||
upgrade_callback(configs=configs)
|
||||
action_set({'outcome': 'success, upgrade completed.'})
|
||||
ret = True
|
||||
except:
|
||||
action_set({'outcome': 'upgrade failed, see traceback.'})
|
||||
action_set({'traceback': traceback.format_exc()})
|
||||
action_fail('do_openstack_upgrade resulted in an '
|
||||
'unexpected error')
|
||||
else:
|
||||
action_set({'outcome': 'action-managed-upgrade config is '
|
||||
'False, skipped upgrade.'})
|
||||
else:
|
||||
action_set({'outcome': 'no upgrade available.'})
|
||||
|
||||
return ret
|
||||
|
||||
|
||||
def remote_restart(rel_name, remote_service=None):
|
||||
trigger = {
|
||||
'restart-trigger': str(uuid.uuid4()),
|
||||
}
|
||||
if remote_service:
|
||||
trigger['remote-service'] = remote_service
|
||||
for rid in relation_ids(rel_name):
|
||||
# This subordinate can be related to two seperate services using
|
||||
# different subordinate relations so only issue the restart if
|
||||
# the principle is conencted down the relation we think it is
|
||||
if related_units(relid=rid):
|
||||
relation_set(relation_id=rid,
|
||||
relation_settings=trigger,
|
||||
)
|
||||
|
@ -59,7 +59,7 @@ def some_hook():
|
||||
"""
|
||||
|
||||
|
||||
def leader_get(attribute=None):
|
||||
def leader_get(attribute=None, rid=None):
|
||||
"""Wrapper to ensure that settings are migrated from the peer relation.
|
||||
|
||||
This is to support upgrading an environment that does not support
|
||||
@ -94,7 +94,8 @@ def leader_get(attribute=None):
|
||||
# If attribute not present in leader db, check if this unit has set
|
||||
# the attribute in the peer relation
|
||||
if not leader_settings:
|
||||
peer_setting = relation_get(attribute=attribute, unit=local_unit())
|
||||
peer_setting = _relation_get(attribute=attribute, unit=local_unit(),
|
||||
rid=rid)
|
||||
if peer_setting:
|
||||
leader_set(settings={attribute: peer_setting})
|
||||
leader_settings = peer_setting
|
||||
@ -103,7 +104,7 @@ def leader_get(attribute=None):
|
||||
settings_migrated = True
|
||||
migrated.add(attribute)
|
||||
else:
|
||||
r_settings = relation_get(unit=local_unit())
|
||||
r_settings = _relation_get(unit=local_unit(), rid=rid)
|
||||
if r_settings:
|
||||
for key in set(r_settings.keys()).difference(migrated):
|
||||
# Leader setting wins
|
||||
@ -151,7 +152,7 @@ def relation_get(attribute=None, unit=None, rid=None):
|
||||
"""
|
||||
try:
|
||||
if rid in relation_ids('cluster'):
|
||||
return leader_get(attribute)
|
||||
return leader_get(attribute, rid)
|
||||
else:
|
||||
raise NotImplementedError
|
||||
except NotImplementedError:
|
||||
|
@ -42,8 +42,12 @@ def parse_options(given, available):
|
||||
yield "--{0}={1}".format(key, value)
|
||||
|
||||
|
||||
def pip_install_requirements(requirements, **options):
|
||||
"""Install a requirements file """
|
||||
def pip_install_requirements(requirements, constraints=None, **options):
|
||||
"""Install a requirements file.
|
||||
|
||||
:param constraints: Path to pip constraints file.
|
||||
http://pip.readthedocs.org/en/stable/user_guide/#constraints-files
|
||||
"""
|
||||
command = ["install"]
|
||||
|
||||
available_options = ('proxy', 'src', 'log', )
|
||||
@ -51,8 +55,13 @@ def pip_install_requirements(requirements, **options):
|
||||
command.append(option)
|
||||
|
||||
command.append("-r {0}".format(requirements))
|
||||
log("Installing from file: {} with options: {}".format(requirements,
|
||||
command))
|
||||
if constraints:
|
||||
command.append("-c {0}".format(constraints))
|
||||
log("Installing from file: {} with constraints {} "
|
||||
"and options: {}".format(requirements, constraints, command))
|
||||
else:
|
||||
log("Installing from file: {} with options: {}".format(requirements,
|
||||
command))
|
||||
pip_execute(command)
|
||||
|
||||
|
||||
|
@ -23,11 +23,14 @@
|
||||
# James Page <james.page@ubuntu.com>
|
||||
# Adam Gandelman <adamg@ubuntu.com>
|
||||
#
|
||||
import bisect
|
||||
import six
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import json
|
||||
import time
|
||||
import uuid
|
||||
|
||||
from subprocess import (
|
||||
check_call,
|
||||
@ -35,8 +38,10 @@ from subprocess import (
|
||||
CalledProcessError,
|
||||
)
|
||||
from charmhelpers.core.hookenv import (
|
||||
local_unit,
|
||||
relation_get,
|
||||
relation_ids,
|
||||
relation_set,
|
||||
related_units,
|
||||
log,
|
||||
DEBUG,
|
||||
@ -56,6 +61,8 @@ from charmhelpers.fetch import (
|
||||
apt_install,
|
||||
)
|
||||
|
||||
from charmhelpers.core.kernel import modprobe
|
||||
|
||||
KEYRING = '/etc/ceph/ceph.client.{}.keyring'
|
||||
KEYFILE = '/etc/ceph/ceph.client.{}.key'
|
||||
|
||||
@ -67,6 +74,394 @@ log to syslog = {use_syslog}
|
||||
err to syslog = {use_syslog}
|
||||
clog to syslog = {use_syslog}
|
||||
"""
|
||||
# For 50 < osds < 240,000 OSDs (Roughly 1 Exabyte at 6T OSDs)
|
||||
powers_of_two = [8192, 16384, 32768, 65536, 131072, 262144, 524288, 1048576, 2097152, 4194304, 8388608]
|
||||
|
||||
|
||||
def validator(value, valid_type, valid_range=None):
|
||||
"""
|
||||
Used to validate these: http://docs.ceph.com/docs/master/rados/operations/pools/#set-pool-values
|
||||
Example input:
|
||||
validator(value=1,
|
||||
valid_type=int,
|
||||
valid_range=[0, 2])
|
||||
This says I'm testing value=1. It must be an int inclusive in [0,2]
|
||||
|
||||
:param value: The value to validate
|
||||
:param valid_type: The type that value should be.
|
||||
:param valid_range: A range of values that value can assume.
|
||||
:return:
|
||||
"""
|
||||
assert isinstance(value, valid_type), "{} is not a {}".format(
|
||||
value,
|
||||
valid_type)
|
||||
if valid_range is not None:
|
||||
assert isinstance(valid_range, list), \
|
||||
"valid_range must be a list, was given {}".format(valid_range)
|
||||
# If we're dealing with strings
|
||||
if valid_type is six.string_types:
|
||||
assert value in valid_range, \
|
||||
"{} is not in the list {}".format(value, valid_range)
|
||||
# Integer, float should have a min and max
|
||||
else:
|
||||
if len(valid_range) != 2:
|
||||
raise ValueError(
|
||||
"Invalid valid_range list of {} for {}. "
|
||||
"List must be [min,max]".format(valid_range, value))
|
||||
assert value >= valid_range[0], \
|
||||
"{} is less than minimum allowed value of {}".format(
|
||||
value, valid_range[0])
|
||||
assert value <= valid_range[1], \
|
||||
"{} is greater than maximum allowed value of {}".format(
|
||||
value, valid_range[1])
|
||||
|
||||
|
||||
class PoolCreationError(Exception):
|
||||
"""
|
||||
A custom error to inform the caller that a pool creation failed. Provides an error message
|
||||
"""
|
||||
def __init__(self, message):
|
||||
super(PoolCreationError, self).__init__(message)
|
||||
|
||||
|
||||
class Pool(object):
|
||||
"""
|
||||
An object oriented approach to Ceph pool creation. This base class is inherited by ReplicatedPool and ErasurePool.
|
||||
Do not call create() on this base class as it will not do anything. Instantiate a child class and call create().
|
||||
"""
|
||||
def __init__(self, service, name):
|
||||
self.service = service
|
||||
self.name = name
|
||||
|
||||
# Create the pool if it doesn't exist already
|
||||
# To be implemented by subclasses
|
||||
def create(self):
|
||||
pass
|
||||
|
||||
def add_cache_tier(self, cache_pool, mode):
|
||||
"""
|
||||
Adds a new cache tier to an existing pool.
|
||||
:param cache_pool: six.string_types. The cache tier pool name to add.
|
||||
:param mode: six.string_types. The caching mode to use for this pool. valid range = ["readonly", "writeback"]
|
||||
:return: None
|
||||
"""
|
||||
# Check the input types and values
|
||||
validator(value=cache_pool, valid_type=six.string_types)
|
||||
validator(value=mode, valid_type=six.string_types, valid_range=["readonly", "writeback"])
|
||||
|
||||
check_call(['ceph', '--id', self.service, 'osd', 'tier', 'add', self.name, cache_pool])
|
||||
check_call(['ceph', '--id', self.service, 'osd', 'tier', 'cache-mode', cache_pool, mode])
|
||||
check_call(['ceph', '--id', self.service, 'osd', 'tier', 'set-overlay', self.name, cache_pool])
|
||||
check_call(['ceph', '--id', self.service, 'osd', 'pool', 'set', cache_pool, 'hit_set_type', 'bloom'])
|
||||
|
||||
def remove_cache_tier(self, cache_pool):
|
||||
"""
|
||||
Removes a cache tier from Ceph. Flushes all dirty objects from writeback pools and waits for that to complete.
|
||||
:param cache_pool: six.string_types. The cache tier pool name to remove.
|
||||
:return: None
|
||||
"""
|
||||
# read-only is easy, writeback is much harder
|
||||
mode = get_cache_mode(cache_pool)
|
||||
if mode == 'readonly':
|
||||
check_call(['ceph', '--id', self.service, 'osd', 'tier', 'cache-mode', cache_pool, 'none'])
|
||||
check_call(['ceph', '--id', self.service, 'osd', 'tier', 'remove', self.name, cache_pool])
|
||||
|
||||
elif mode == 'writeback':
|
||||
check_call(['ceph', '--id', self.service, 'osd', 'tier', 'cache-mode', cache_pool, 'forward'])
|
||||
# Flush the cache and wait for it to return
|
||||
check_call(['ceph', '--id', self.service, '-p', cache_pool, 'cache-flush-evict-all'])
|
||||
check_call(['ceph', '--id', self.service, 'osd', 'tier', 'remove-overlay', self.name])
|
||||
check_call(['ceph', '--id', self.service, 'osd', 'tier', 'remove', self.name, cache_pool])
|
||||
|
||||
def get_pgs(self, pool_size):
|
||||
"""
|
||||
:param pool_size: int. pool_size is either the number of replicas for replicated pools or the K+M sum for
|
||||
erasure coded pools
|
||||
:return: int. The number of pgs to use.
|
||||
"""
|
||||
validator(value=pool_size, valid_type=int)
|
||||
osds = get_osds(self.service)
|
||||
if not osds:
|
||||
# NOTE(james-page): Default to 200 for older ceph versions
|
||||
# which don't support OSD query from cli
|
||||
return 200
|
||||
|
||||
# Calculate based on Ceph best practices
|
||||
if osds < 5:
|
||||
return 128
|
||||
elif 5 < osds < 10:
|
||||
return 512
|
||||
elif 10 < osds < 50:
|
||||
return 4096
|
||||
else:
|
||||
estimate = (osds * 100) / pool_size
|
||||
# Return the next nearest power of 2
|
||||
index = bisect.bisect_right(powers_of_two, estimate)
|
||||
return powers_of_two[index]
|
||||
|
||||
|
||||
class ReplicatedPool(Pool):
|
||||
def __init__(self, service, name, replicas=2):
|
||||
super(ReplicatedPool, self).__init__(service=service, name=name)
|
||||
self.replicas = replicas
|
||||
|
||||
def create(self):
|
||||
if not pool_exists(self.service, self.name):
|
||||
# Create it
|
||||
pgs = self.get_pgs(self.replicas)
|
||||
cmd = ['ceph', '--id', self.service, 'osd', 'pool', 'create', self.name, str(pgs)]
|
||||
try:
|
||||
check_call(cmd)
|
||||
except CalledProcessError:
|
||||
raise
|
||||
|
||||
|
||||
# Default jerasure erasure coded pool
|
||||
class ErasurePool(Pool):
|
||||
def __init__(self, service, name, erasure_code_profile="default"):
|
||||
super(ErasurePool, self).__init__(service=service, name=name)
|
||||
self.erasure_code_profile = erasure_code_profile
|
||||
|
||||
def create(self):
|
||||
if not pool_exists(self.service, self.name):
|
||||
# Try to find the erasure profile information so we can properly size the pgs
|
||||
erasure_profile = get_erasure_profile(service=self.service, name=self.erasure_code_profile)
|
||||
|
||||
# Check for errors
|
||||
if erasure_profile is None:
|
||||
log(message='Failed to discover erasure_profile named={}'.format(self.erasure_code_profile),
|
||||
level=ERROR)
|
||||
raise PoolCreationError(message='unable to find erasure profile {}'.format(self.erasure_code_profile))
|
||||
if 'k' not in erasure_profile or 'm' not in erasure_profile:
|
||||
# Error
|
||||
log(message='Unable to find k (data chunks) or m (coding chunks) in {}'.format(erasure_profile),
|
||||
level=ERROR)
|
||||
raise PoolCreationError(
|
||||
message='unable to find k (data chunks) or m (coding chunks) in {}'.format(erasure_profile))
|
||||
|
||||
pgs = self.get_pgs(int(erasure_profile['k']) + int(erasure_profile['m']))
|
||||
# Create it
|
||||
cmd = ['ceph', '--id', self.service, 'osd', 'pool', 'create', self.name, str(pgs),
|
||||
'erasure', self.erasure_code_profile]
|
||||
try:
|
||||
check_call(cmd)
|
||||
except CalledProcessError:
|
||||
raise
|
||||
|
||||
"""Get an existing erasure code profile if it already exists.
|
||||
Returns json formatted output"""
|
||||
|
||||
|
||||
def get_erasure_profile(service, name):
|
||||
"""
|
||||
:param service: six.string_types. The Ceph user name to run the command under
|
||||
:param name:
|
||||
:return:
|
||||
"""
|
||||
try:
|
||||
out = check_output(['ceph', '--id', service,
|
||||
'osd', 'erasure-code-profile', 'get',
|
||||
name, '--format=json'])
|
||||
return json.loads(out)
|
||||
except (CalledProcessError, OSError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def pool_set(service, pool_name, key, value):
|
||||
"""
|
||||
Sets a value for a RADOS pool in ceph.
|
||||
:param service: six.string_types. The Ceph user name to run the command under
|
||||
:param pool_name: six.string_types
|
||||
:param key: six.string_types
|
||||
:param value:
|
||||
:return: None. Can raise CalledProcessError
|
||||
"""
|
||||
cmd = ['ceph', '--id', service, 'osd', 'pool', 'set', pool_name, key, value]
|
||||
try:
|
||||
check_call(cmd)
|
||||
except CalledProcessError:
|
||||
raise
|
||||
|
||||
|
||||
def snapshot_pool(service, pool_name, snapshot_name):
|
||||
"""
|
||||
Snapshots a RADOS pool in ceph.
|
||||
:param service: six.string_types. The Ceph user name to run the command under
|
||||
:param pool_name: six.string_types
|
||||
:param snapshot_name: six.string_types
|
||||
:return: None. Can raise CalledProcessError
|
||||
"""
|
||||
cmd = ['ceph', '--id', service, 'osd', 'pool', 'mksnap', pool_name, snapshot_name]
|
||||
try:
|
||||
check_call(cmd)
|
||||
except CalledProcessError:
|
||||
raise
|
||||
|
||||
|
||||
def remove_pool_snapshot(service, pool_name, snapshot_name):
|
||||
"""
|
||||
Remove a snapshot from a RADOS pool in ceph.
|
||||
:param service: six.string_types. The Ceph user name to run the command under
|
||||
:param pool_name: six.string_types
|
||||
:param snapshot_name: six.string_types
|
||||
:return: None. Can raise CalledProcessError
|
||||
"""
|
||||
cmd = ['ceph', '--id', service, 'osd', 'pool', 'rmsnap', pool_name, snapshot_name]
|
||||
try:
|
||||
check_call(cmd)
|
||||
except CalledProcessError:
|
||||
raise
|
||||
|
||||
|
||||
# max_bytes should be an int or long
|
||||
def set_pool_quota(service, pool_name, max_bytes):
|
||||
"""
|
||||
:param service: six.string_types. The Ceph user name to run the command under
|
||||
:param pool_name: six.string_types
|
||||
:param max_bytes: int or long
|
||||
:return: None. Can raise CalledProcessError
|
||||
"""
|
||||
# Set a byte quota on a RADOS pool in ceph.
|
||||
cmd = ['ceph', '--id', service, 'osd', 'pool', 'set-quota', pool_name, 'max_bytes', max_bytes]
|
||||
try:
|
||||
check_call(cmd)
|
||||
except CalledProcessError:
|
||||
raise
|
||||
|
||||
|
||||
def remove_pool_quota(service, pool_name):
|
||||
"""
|
||||
Set a byte quota on a RADOS pool in ceph.
|
||||
:param service: six.string_types. The Ceph user name to run the command under
|
||||
:param pool_name: six.string_types
|
||||
:return: None. Can raise CalledProcessError
|
||||
"""
|
||||
cmd = ['ceph', '--id', service, 'osd', 'pool', 'set-quota', pool_name, 'max_bytes', '0']
|
||||
try:
|
||||
check_call(cmd)
|
||||
except CalledProcessError:
|
||||
raise
|
||||
|
||||
|
||||
def create_erasure_profile(service, profile_name, erasure_plugin_name='jerasure', failure_domain='host',
|
||||
data_chunks=2, coding_chunks=1,
|
||||
locality=None, durability_estimator=None):
|
||||
"""
|
||||
Create a new erasure code profile if one does not already exist for it. Updates
|
||||
the profile if it exists. Please see http://docs.ceph.com/docs/master/rados/operations/erasure-code-profile/
|
||||
for more details
|
||||
:param service: six.string_types. The Ceph user name to run the command under
|
||||
:param profile_name: six.string_types
|
||||
:param erasure_plugin_name: six.string_types
|
||||
:param failure_domain: six.string_types. One of ['chassis', 'datacenter', 'host', 'osd', 'pdu', 'pod', 'rack', 'region',
|
||||
'room', 'root', 'row'])
|
||||
:param data_chunks: int
|
||||
:param coding_chunks: int
|
||||
:param locality: int
|
||||
:param durability_estimator: int
|
||||
:return: None. Can raise CalledProcessError
|
||||
"""
|
||||
# Ensure this failure_domain is allowed by Ceph
|
||||
validator(failure_domain, six.string_types,
|
||||
['chassis', 'datacenter', 'host', 'osd', 'pdu', 'pod', 'rack', 'region', 'room', 'root', 'row'])
|
||||
|
||||
cmd = ['ceph', '--id', service, 'osd', 'erasure-code-profile', 'set', profile_name,
|
||||
'plugin=' + erasure_plugin_name, 'k=' + str(data_chunks), 'm=' + str(coding_chunks),
|
||||
'ruleset_failure_domain=' + failure_domain]
|
||||
if locality is not None and durability_estimator is not None:
|
||||
raise ValueError("create_erasure_profile should be called with k, m and one of l or c but not both.")
|
||||
|
||||
# Add plugin specific information
|
||||
if locality is not None:
|
||||
# For local erasure codes
|
||||
cmd.append('l=' + str(locality))
|
||||
if durability_estimator is not None:
|
||||
# For Shec erasure codes
|
||||
cmd.append('c=' + str(durability_estimator))
|
||||
|
||||
if erasure_profile_exists(service, profile_name):
|
||||
cmd.append('--force')
|
||||
|
||||
try:
|
||||
check_call(cmd)
|
||||
except CalledProcessError:
|
||||
raise
|
||||
|
||||
|
||||
def rename_pool(service, old_name, new_name):
|
||||
"""
|
||||
Rename a Ceph pool from old_name to new_name
|
||||
:param service: six.string_types. The Ceph user name to run the command under
|
||||
:param old_name: six.string_types
|
||||
:param new_name: six.string_types
|
||||
:return: None
|
||||
"""
|
||||
validator(value=old_name, valid_type=six.string_types)
|
||||
validator(value=new_name, valid_type=six.string_types)
|
||||
|
||||
cmd = ['ceph', '--id', service, 'osd', 'pool', 'rename', old_name, new_name]
|
||||
check_call(cmd)
|
||||
|
||||
|
||||
def erasure_profile_exists(service, name):
|
||||
"""
|
||||
Check to see if an Erasure code profile already exists.
|
||||
:param service: six.string_types. The Ceph user name to run the command under
|
||||
:param name: six.string_types
|
||||
:return: int or None
|
||||
"""
|
||||
validator(value=name, valid_type=six.string_types)
|
||||
try:
|
||||
check_call(['ceph', '--id', service,
|
||||
'osd', 'erasure-code-profile', 'get',
|
||||
name])
|
||||
return True
|
||||
except CalledProcessError:
|
||||
return False
|
||||
|
||||
|
||||
def get_cache_mode(service, pool_name):
|
||||
"""
|
||||
Find the current caching mode of the pool_name given.
|
||||
:param service: six.string_types. The Ceph user name to run the command under
|
||||
:param pool_name: six.string_types
|
||||
:return: int or None
|
||||
"""
|
||||
validator(value=service, valid_type=six.string_types)
|
||||
validator(value=pool_name, valid_type=six.string_types)
|
||||
out = check_output(['ceph', '--id', service, 'osd', 'dump', '--format=json'])
|
||||
try:
|
||||
osd_json = json.loads(out)
|
||||
for pool in osd_json['pools']:
|
||||
if pool['pool_name'] == pool_name:
|
||||
return pool['cache_mode']
|
||||
return None
|
||||
except ValueError:
|
||||
raise
|
||||
|
||||
|
||||
def pool_exists(service, name):
|
||||
"""Check to see if a RADOS pool already exists."""
|
||||
try:
|
||||
out = check_output(['rados', '--id', service,
|
||||
'lspools']).decode('UTF-8')
|
||||
except CalledProcessError:
|
||||
return False
|
||||
|
||||
return name in out
|
||||
|
||||
|
||||
def get_osds(service):
|
||||
"""Return a list of all Ceph Object Storage Daemons currently in the
|
||||
cluster.
|
||||
"""
|
||||
version = ceph_version()
|
||||
if version and version >= '0.56':
|
||||
return json.loads(check_output(['ceph', '--id', service,
|
||||
'osd', 'ls',
|
||||
'--format=json']).decode('UTF-8'))
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def install():
|
||||
@ -96,53 +491,37 @@ def create_rbd_image(service, pool, image, sizemb):
|
||||
check_call(cmd)
|
||||
|
||||
|
||||
def pool_exists(service, name):
|
||||
"""Check to see if a RADOS pool already exists."""
|
||||
try:
|
||||
out = check_output(['rados', '--id', service,
|
||||
'lspools']).decode('UTF-8')
|
||||
except CalledProcessError:
|
||||
return False
|
||||
def update_pool(client, pool, settings):
|
||||
cmd = ['ceph', '--id', client, 'osd', 'pool', 'set', pool]
|
||||
for k, v in six.iteritems(settings):
|
||||
cmd.append(k)
|
||||
cmd.append(v)
|
||||
|
||||
return name in out
|
||||
check_call(cmd)
|
||||
|
||||
|
||||
def get_osds(service):
|
||||
"""Return a list of all Ceph Object Storage Daemons currently in the
|
||||
cluster.
|
||||
"""
|
||||
version = ceph_version()
|
||||
if version and version >= '0.56':
|
||||
return json.loads(check_output(['ceph', '--id', service,
|
||||
'osd', 'ls',
|
||||
'--format=json']).decode('UTF-8'))
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def create_pool(service, name, replicas=3):
|
||||
def create_pool(service, name, replicas=3, pg_num=None):
|
||||
"""Create a new RADOS pool."""
|
||||
if pool_exists(service, name):
|
||||
log("Ceph pool {} already exists, skipping creation".format(name),
|
||||
level=WARNING)
|
||||
return
|
||||
|
||||
# Calculate the number of placement groups based
|
||||
# on upstream recommended best practices.
|
||||
osds = get_osds(service)
|
||||
if osds:
|
||||
pgnum = (len(osds) * 100 // replicas)
|
||||
else:
|
||||
# NOTE(james-page): Default to 200 for older ceph versions
|
||||
# which don't support OSD query from cli
|
||||
pgnum = 200
|
||||
if not pg_num:
|
||||
# Calculate the number of placement groups based
|
||||
# on upstream recommended best practices.
|
||||
osds = get_osds(service)
|
||||
if osds:
|
||||
pg_num = (len(osds) * 100 // replicas)
|
||||
else:
|
||||
# NOTE(james-page): Default to 200 for older ceph versions
|
||||
# which don't support OSD query from cli
|
||||
pg_num = 200
|
||||
|
||||
cmd = ['ceph', '--id', service, 'osd', 'pool', 'create', name, str(pgnum)]
|
||||
cmd = ['ceph', '--id', service, 'osd', 'pool', 'create', name, str(pg_num)]
|
||||
check_call(cmd)
|
||||
|
||||
cmd = ['ceph', '--id', service, 'osd', 'pool', 'set', name, 'size',
|
||||
str(replicas)]
|
||||
check_call(cmd)
|
||||
update_pool(service, name, settings={'size': str(replicas)})
|
||||
|
||||
|
||||
def delete_pool(service, name):
|
||||
@ -197,10 +576,10 @@ def create_key_file(service, key):
|
||||
log('Created new keyfile at %s.' % keyfile, level=INFO)
|
||||
|
||||
|
||||
def get_ceph_nodes():
|
||||
"""Query named relation 'ceph' to determine current nodes."""
|
||||
def get_ceph_nodes(relation='ceph'):
|
||||
"""Query named relation to determine current nodes."""
|
||||
hosts = []
|
||||
for r_id in relation_ids('ceph'):
|
||||
for r_id in relation_ids(relation):
|
||||
for unit in related_units(r_id):
|
||||
hosts.append(relation_get('private-address', unit=unit, rid=r_id))
|
||||
|
||||
@ -288,17 +667,6 @@ def place_data_on_block_device(blk_device, data_src_dst):
|
||||
os.chown(data_src_dst, uid, gid)
|
||||
|
||||
|
||||
# TODO: re-use
|
||||
def modprobe(module):
|
||||
"""Load a kernel module and configure for auto-load on reboot."""
|
||||
log('Loading kernel module', level=INFO)
|
||||
cmd = ['modprobe', module]
|
||||
check_call(cmd)
|
||||
with open('/etc/modules', 'r+') as modules:
|
||||
if module not in modules.read():
|
||||
modules.write(module)
|
||||
|
||||
|
||||
def copy_files(src, dst, symlinks=False, ignore=None):
|
||||
"""Copy files from src to dst."""
|
||||
for item in os.listdir(src):
|
||||
@ -363,14 +731,14 @@ def ensure_ceph_storage(service, pool, rbd_img, sizemb, mount_point,
|
||||
service_start(svc)
|
||||
|
||||
|
||||
def ensure_ceph_keyring(service, user=None, group=None):
|
||||
def ensure_ceph_keyring(service, user=None, group=None, relation='ceph'):
|
||||
"""Ensures a ceph keyring is created for a named service and optionally
|
||||
ensures user and group ownership.
|
||||
|
||||
Returns False if no ceph key is available in relation state.
|
||||
"""
|
||||
key = None
|
||||
for rid in relation_ids('ceph'):
|
||||
for rid in relation_ids(relation):
|
||||
for unit in related_units(rid):
|
||||
key = relation_get('key', rid=rid, unit=unit)
|
||||
if key:
|
||||
@ -411,17 +779,60 @@ class CephBrokerRq(object):
|
||||
|
||||
The API is versioned and defaults to version 1.
|
||||
"""
|
||||
def __init__(self, api_version=1):
|
||||
|
||||
def __init__(self, api_version=1, request_id=None):
|
||||
self.api_version = api_version
|
||||
if request_id:
|
||||
self.request_id = request_id
|
||||
else:
|
||||
self.request_id = str(uuid.uuid1())
|
||||
self.ops = []
|
||||
|
||||
def add_op_create_pool(self, name, replica_count=3):
|
||||
def add_op_create_pool(self, name, replica_count=3, pg_num=None):
|
||||
"""Adds an operation to create a pool.
|
||||
|
||||
@param pg_num setting: optional setting. If not provided, this value
|
||||
will be calculated by the broker based on how many OSDs are in the
|
||||
cluster at the time of creation. Note that, if provided, this value
|
||||
will be capped at the current available maximum.
|
||||
"""
|
||||
self.ops.append({'op': 'create-pool', 'name': name,
|
||||
'replicas': replica_count})
|
||||
'replicas': replica_count, 'pg_num': pg_num})
|
||||
|
||||
def set_ops(self, ops):
|
||||
"""Set request ops to provided value.
|
||||
|
||||
Useful for injecting ops that come from a previous request
|
||||
to allow comparisons to ensure validity.
|
||||
"""
|
||||
self.ops = ops
|
||||
|
||||
@property
|
||||
def request(self):
|
||||
return json.dumps({'api-version': self.api_version, 'ops': self.ops})
|
||||
return json.dumps({'api-version': self.api_version, 'ops': self.ops,
|
||||
'request-id': self.request_id})
|
||||
|
||||
def _ops_equal(self, other):
|
||||
if len(self.ops) == len(other.ops):
|
||||
for req_no in range(0, len(self.ops)):
|
||||
for key in ['replicas', 'name', 'op', 'pg_num']:
|
||||
if self.ops[req_no].get(key) != other.ops[req_no].get(key):
|
||||
return False
|
||||
else:
|
||||
return False
|
||||
return True
|
||||
|
||||
def __eq__(self, other):
|
||||
if not isinstance(other, self.__class__):
|
||||
return False
|
||||
if self.api_version == other.api_version and \
|
||||
self._ops_equal(other):
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
def __ne__(self, other):
|
||||
return not self.__eq__(other)
|
||||
|
||||
|
||||
class CephBrokerRsp(object):
|
||||
@ -431,10 +842,15 @@ class CephBrokerRsp(object):
|
||||
|
||||
The API is versioned and defaults to version 1.
|
||||
"""
|
||||
|
||||
def __init__(self, encoded_rsp):
|
||||
self.api_version = None
|
||||
self.rsp = json.loads(encoded_rsp)
|
||||
|
||||
@property
|
||||
def request_id(self):
|
||||
return self.rsp.get('request-id')
|
||||
|
||||
@property
|
||||
def exit_code(self):
|
||||
return self.rsp.get('exit-code')
|
||||
@ -442,3 +858,182 @@ class CephBrokerRsp(object):
|
||||
@property
|
||||
def exit_msg(self):
|
||||
return self.rsp.get('stderr')
|
||||
|
||||
|
||||
# Ceph Broker Conversation:
|
||||
# If a charm needs an action to be taken by ceph it can create a CephBrokerRq
|
||||
# and send that request to ceph via the ceph relation. The CephBrokerRq has a
|
||||
# unique id so that the client can identity which CephBrokerRsp is associated
|
||||
# with the request. Ceph will also respond to each client unit individually
|
||||
# creating a response key per client unit eg glance/0 will get a CephBrokerRsp
|
||||
# via key broker-rsp-glance-0
|
||||
#
|
||||
# To use this the charm can just do something like:
|
||||
#
|
||||
# from charmhelpers.contrib.storage.linux.ceph import (
|
||||
# send_request_if_needed,
|
||||
# is_request_complete,
|
||||
# CephBrokerRq,
|
||||
# )
|
||||
#
|
||||
# @hooks.hook('ceph-relation-changed')
|
||||
# def ceph_changed():
|
||||
# rq = CephBrokerRq()
|
||||
# rq.add_op_create_pool(name='poolname', replica_count=3)
|
||||
#
|
||||
# if is_request_complete(rq):
|
||||
# <Request complete actions>
|
||||
# else:
|
||||
# send_request_if_needed(get_ceph_request())
|
||||
#
|
||||
# CephBrokerRq and CephBrokerRsp are serialized into JSON. Below is an example
|
||||
# of glance having sent a request to ceph which ceph has successfully processed
|
||||
# 'ceph:8': {
|
||||
# 'ceph/0': {
|
||||
# 'auth': 'cephx',
|
||||
# 'broker-rsp-glance-0': '{"request-id": "0bc7dc54", "exit-code": 0}',
|
||||
# 'broker_rsp': '{"request-id": "0da543b8", "exit-code": 0}',
|
||||
# 'ceph-public-address': '10.5.44.103',
|
||||
# 'key': 'AQCLDttVuHXINhAAvI144CB09dYchhHyTUY9BQ==',
|
||||
# 'private-address': '10.5.44.103',
|
||||
# },
|
||||
# 'glance/0': {
|
||||
# 'broker_req': ('{"api-version": 1, "request-id": "0bc7dc54", '
|
||||
# '"ops": [{"replicas": 3, "name": "glance", '
|
||||
# '"op": "create-pool"}]}'),
|
||||
# 'private-address': '10.5.44.109',
|
||||
# },
|
||||
# }
|
||||
|
||||
def get_previous_request(rid):
|
||||
"""Return the last ceph broker request sent on a given relation
|
||||
|
||||
@param rid: Relation id to query for request
|
||||
"""
|
||||
request = None
|
||||
broker_req = relation_get(attribute='broker_req', rid=rid,
|
||||
unit=local_unit())
|
||||
if broker_req:
|
||||
request_data = json.loads(broker_req)
|
||||
request = CephBrokerRq(api_version=request_data['api-version'],
|
||||
request_id=request_data['request-id'])
|
||||
request.set_ops(request_data['ops'])
|
||||
|
||||
return request
|
||||
|
||||
|
||||
def get_request_states(request, relation='ceph'):
|
||||
"""Return a dict of requests per relation id with their corresponding
|
||||
completion state.
|
||||
|
||||
This allows a charm, which has a request for ceph, to see whether there is
|
||||
an equivalent request already being processed and if so what state that
|
||||
request is in.
|
||||
|
||||
@param request: A CephBrokerRq object
|
||||
"""
|
||||
complete = []
|
||||
requests = {}
|
||||
for rid in relation_ids(relation):
|
||||
complete = False
|
||||
previous_request = get_previous_request(rid)
|
||||
if request == previous_request:
|
||||
sent = True
|
||||
complete = is_request_complete_for_rid(previous_request, rid)
|
||||
else:
|
||||
sent = False
|
||||
complete = False
|
||||
|
||||
requests[rid] = {
|
||||
'sent': sent,
|
||||
'complete': complete,
|
||||
}
|
||||
|
||||
return requests
|
||||
|
||||
|
||||
def is_request_sent(request, relation='ceph'):
|
||||
"""Check to see if a functionally equivalent request has already been sent
|
||||
|
||||
Returns True if a similair request has been sent
|
||||
|
||||
@param request: A CephBrokerRq object
|
||||
"""
|
||||
states = get_request_states(request, relation=relation)
|
||||
for rid in states.keys():
|
||||
if not states[rid]['sent']:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def is_request_complete(request, relation='ceph'):
|
||||
"""Check to see if a functionally equivalent request has already been
|
||||
completed
|
||||
|
||||
Returns True if a similair request has been completed
|
||||
|
||||
@param request: A CephBrokerRq object
|
||||
"""
|
||||
states = get_request_states(request, relation=relation)
|
||||
for rid in states.keys():
|
||||
if not states[rid]['complete']:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def is_request_complete_for_rid(request, rid):
|
||||
"""Check if a given request has been completed on the given relation
|
||||
|
||||
@param request: A CephBrokerRq object
|
||||
@param rid: Relation ID
|
||||
"""
|
||||
broker_key = get_broker_rsp_key()
|
||||
for unit in related_units(rid):
|
||||
rdata = relation_get(rid=rid, unit=unit)
|
||||
if rdata.get(broker_key):
|
||||
rsp = CephBrokerRsp(rdata.get(broker_key))
|
||||
if rsp.request_id == request.request_id:
|
||||
if not rsp.exit_code:
|
||||
return True
|
||||
else:
|
||||
# The remote unit sent no reply targeted at this unit so either the
|
||||
# remote ceph cluster does not support unit targeted replies or it
|
||||
# has not processed our request yet.
|
||||
if rdata.get('broker_rsp'):
|
||||
request_data = json.loads(rdata['broker_rsp'])
|
||||
if request_data.get('request-id'):
|
||||
log('Ignoring legacy broker_rsp without unit key as remote '
|
||||
'service supports unit specific replies', level=DEBUG)
|
||||
else:
|
||||
log('Using legacy broker_rsp as remote service does not '
|
||||
'supports unit specific replies', level=DEBUG)
|
||||
rsp = CephBrokerRsp(rdata['broker_rsp'])
|
||||
if not rsp.exit_code:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def get_broker_rsp_key():
|
||||
"""Return broker response key for this unit
|
||||
|
||||
This is the key that ceph is going to use to pass request status
|
||||
information back to this unit
|
||||
"""
|
||||
return 'broker-rsp-' + local_unit().replace('/', '-')
|
||||
|
||||
|
||||
def send_request_if_needed(request, relation='ceph'):
|
||||
"""Send broker request if an equivalent request has not already been sent
|
||||
|
||||
@param request: A CephBrokerRq object
|
||||
"""
|
||||
if is_request_sent(request, relation=relation):
|
||||
log('Request already sent but not complete, not sending new request',
|
||||
level=DEBUG)
|
||||
else:
|
||||
for rid in relation_ids(relation):
|
||||
log('Sending request {}'.format(request.request_id), level=DEBUG)
|
||||
relation_set(relation_id=rid, broker_req=request.request)
|
||||
|
@ -76,3 +76,13 @@ def ensure_loopback_device(path, size):
|
||||
check_call(cmd)
|
||||
|
||||
return create_loopback(path)
|
||||
|
||||
|
||||
def is_mapped_loopback_device(device):
|
||||
"""
|
||||
Checks if a given device name is an existing/mapped loopback device.
|
||||
:param device: str: Full path to the device (eg, /dev/loop1).
|
||||
:returns: str: Path to the backing file if is a loopback device
|
||||
empty string otherwise
|
||||
"""
|
||||
return loopback_devices().get(device, "")
|
||||
|
@ -43,9 +43,10 @@ def zap_disk(block_device):
|
||||
|
||||
:param block_device: str: Full path of block device to clean.
|
||||
'''
|
||||
# https://github.com/ceph/ceph/commit/fdd7f8d83afa25c4e09aaedd90ab93f3b64a677b
|
||||
# sometimes sgdisk exits non-zero; this is OK, dd will clean up
|
||||
call(['sgdisk', '--zap-all', '--mbrtogpt',
|
||||
'--clear', block_device])
|
||||
call(['sgdisk', '--zap-all', '--', block_device])
|
||||
call(['sgdisk', '--clear', '--mbrtogpt', '--', block_device])
|
||||
dev_end = check_output(['blockdev', '--getsz',
|
||||
block_device]).decode('UTF-8')
|
||||
gpt_end = int(dev_end.split()[0]) - 100
|
||||
|
@ -18,14 +18,15 @@
|
||||
Templating using the python-jinja2 package.
|
||||
"""
|
||||
import six
|
||||
from charmhelpers.fetch import apt_install
|
||||
from charmhelpers.fetch import apt_install, apt_update
|
||||
try:
|
||||
import jinja2
|
||||
except ImportError:
|
||||
apt_update(fatal=True)
|
||||
if six.PY3:
|
||||
apt_install(["python3-jinja2"])
|
||||
apt_install(["python3-jinja2"], fatal=True)
|
||||
else:
|
||||
apt_install(["python-jinja2"])
|
||||
apt_install(["python-jinja2"], fatal=True)
|
||||
import jinja2
|
||||
|
||||
|
||||
|
@ -74,6 +74,7 @@ def cached(func):
|
||||
res = func(*args, **kwargs)
|
||||
cache[key] = res
|
||||
return res
|
||||
wrapper._wrapped = func
|
||||
return wrapper
|
||||
|
||||
|
||||
@ -173,9 +174,19 @@ def relation_type():
|
||||
return os.environ.get('JUJU_RELATION', None)
|
||||
|
||||
|
||||
def relation_id():
|
||||
"""The relation ID for the current relation hook"""
|
||||
return os.environ.get('JUJU_RELATION_ID', None)
|
||||
@cached
|
||||
def relation_id(relation_name=None, service_or_unit=None):
|
||||
"""The relation ID for the current or a specified relation"""
|
||||
if not relation_name and not service_or_unit:
|
||||
return os.environ.get('JUJU_RELATION_ID', None)
|
||||
elif relation_name and service_or_unit:
|
||||
service_name = service_or_unit.split('/')[0]
|
||||
for relid in relation_ids(relation_name):
|
||||
remote_service = remote_service_name(relid)
|
||||
if remote_service == service_name:
|
||||
return relid
|
||||
else:
|
||||
raise ValueError('Must specify neither or both of relation_name and service_or_unit')
|
||||
|
||||
|
||||
def local_unit():
|
||||
@ -193,9 +204,20 @@ def service_name():
|
||||
return local_unit().split('/')[0]
|
||||
|
||||
|
||||
@cached
|
||||
def remote_service_name(relid=None):
|
||||
"""The remote service name for a given relation-id (or the current relation)"""
|
||||
if relid is None:
|
||||
unit = remote_unit()
|
||||
else:
|
||||
units = related_units(relid)
|
||||
unit = units[0] if units else None
|
||||
return unit.split('/')[0] if unit else None
|
||||
|
||||
|
||||
def hook_name():
|
||||
"""The name of the currently executing hook"""
|
||||
return os.path.basename(sys.argv[0])
|
||||
return os.environ.get('JUJU_HOOK_NAME', os.path.basename(sys.argv[0]))
|
||||
|
||||
|
||||
class Config(dict):
|
||||
@ -468,6 +490,76 @@ def relation_types():
|
||||
return rel_types
|
||||
|
||||
|
||||
@cached
|
||||
def peer_relation_id():
|
||||
'''Get the peers relation id if a peers relation has been joined, else None.'''
|
||||
md = metadata()
|
||||
section = md.get('peers')
|
||||
if section:
|
||||
for key in section:
|
||||
relids = relation_ids(key)
|
||||
if relids:
|
||||
return relids[0]
|
||||
return None
|
||||
|
||||
|
||||
@cached
|
||||
def relation_to_interface(relation_name):
|
||||
"""
|
||||
Given the name of a relation, return the interface that relation uses.
|
||||
|
||||
:returns: The interface name, or ``None``.
|
||||
"""
|
||||
return relation_to_role_and_interface(relation_name)[1]
|
||||
|
||||
|
||||
@cached
|
||||
def relation_to_role_and_interface(relation_name):
|
||||
"""
|
||||
Given the name of a relation, return the role and the name of the interface
|
||||
that relation uses (where role is one of ``provides``, ``requires``, or ``peers``).
|
||||
|
||||
:returns: A tuple containing ``(role, interface)``, or ``(None, None)``.
|
||||
"""
|
||||
_metadata = metadata()
|
||||
for role in ('provides', 'requires', 'peers'):
|
||||
interface = _metadata.get(role, {}).get(relation_name, {}).get('interface')
|
||||
if interface:
|
||||
return role, interface
|
||||
return None, None
|
||||
|
||||
|
||||
@cached
|
||||
def role_and_interface_to_relations(role, interface_name):
|
||||
"""
|
||||
Given a role and interface name, return a list of relation names for the
|
||||
current charm that use that interface under that role (where role is one
|
||||
of ``provides``, ``requires``, or ``peers``).
|
||||
|
||||
:returns: A list of relation names.
|
||||
"""
|
||||
_metadata = metadata()
|
||||
results = []
|
||||
for relation_name, relation in _metadata.get(role, {}).items():
|
||||
if relation['interface'] == interface_name:
|
||||
results.append(relation_name)
|
||||
return results
|
||||
|
||||
|
||||
@cached
|
||||
def interface_to_relations(interface_name):
|
||||
"""
|
||||
Given an interface, return a list of relation names for the current
|
||||
charm that use that interface.
|
||||
|
||||
:returns: A list of relation names.
|
||||
"""
|
||||
results = []
|
||||
for role in ('provides', 'requires', 'peers'):
|
||||
results.extend(role_and_interface_to_relations(role, interface_name))
|
||||
return results
|
||||
|
||||
|
||||
@cached
|
||||
def charm_name():
|
||||
"""Get the name of the current charm as is specified on metadata.yaml"""
|
||||
@ -544,6 +636,38 @@ def unit_private_ip():
|
||||
return unit_get('private-address')
|
||||
|
||||
|
||||
@cached
|
||||
def storage_get(attribute=None, storage_id=None):
|
||||
"""Get storage attributes"""
|
||||
_args = ['storage-get', '--format=json']
|
||||
if storage_id:
|
||||
_args.extend(('-s', storage_id))
|
||||
if attribute:
|
||||
_args.append(attribute)
|
||||
try:
|
||||
return json.loads(subprocess.check_output(_args).decode('UTF-8'))
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
@cached
|
||||
def storage_list(storage_name=None):
|
||||
"""List the storage IDs for the unit"""
|
||||
_args = ['storage-list', '--format=json']
|
||||
if storage_name:
|
||||
_args.append(storage_name)
|
||||
try:
|
||||
return json.loads(subprocess.check_output(_args).decode('UTF-8'))
|
||||
except ValueError:
|
||||
return None
|
||||
except OSError as e:
|
||||
import errno
|
||||
if e.errno == errno.ENOENT:
|
||||
# storage-list does not exist
|
||||
return []
|
||||
raise
|
||||
|
||||
|
||||
class UnregisteredHookError(Exception):
|
||||
"""Raised when an undefined hook is called"""
|
||||
pass
|
||||
@ -644,6 +768,21 @@ def action_fail(message):
|
||||
subprocess.check_call(['action-fail', message])
|
||||
|
||||
|
||||
def action_name():
|
||||
"""Get the name of the currently executing action."""
|
||||
return os.environ.get('JUJU_ACTION_NAME')
|
||||
|
||||
|
||||
def action_uuid():
|
||||
"""Get the UUID of the currently executing action."""
|
||||
return os.environ.get('JUJU_ACTION_UUID')
|
||||
|
||||
|
||||
def action_tag():
|
||||
"""Get the tag for the currently executing action."""
|
||||
return os.environ.get('JUJU_ACTION_TAG')
|
||||
|
||||
|
||||
def status_set(workload_state, message):
|
||||
"""Set the workload state with a message
|
||||
|
||||
@ -673,25 +812,28 @@ def status_set(workload_state, message):
|
||||
|
||||
|
||||
def status_get():
|
||||
"""Retrieve the previously set juju workload state
|
||||
"""Retrieve the previously set juju workload state and message
|
||||
|
||||
If the status-get command is not found then assume this is juju < 1.23 and
|
||||
return 'unknown', ""
|
||||
|
||||
If the status-set command is not found then assume this is juju < 1.23 and
|
||||
return 'unknown'
|
||||
"""
|
||||
cmd = ['status-get']
|
||||
cmd = ['status-get', "--format=json", "--include-data"]
|
||||
try:
|
||||
raw_status = subprocess.check_output(cmd, universal_newlines=True)
|
||||
status = raw_status.rstrip()
|
||||
return status
|
||||
raw_status = subprocess.check_output(cmd)
|
||||
except OSError as e:
|
||||
if e.errno == errno.ENOENT:
|
||||
return 'unknown'
|
||||
return ('unknown', "")
|
||||
else:
|
||||
raise
|
||||
else:
|
||||
status = json.loads(raw_status.decode("UTF-8"))
|
||||
return (status["status"], status["message"])
|
||||
|
||||
|
||||
def translate_exc(from_exc, to_exc):
|
||||
def inner_translate_exc1(f):
|
||||
@wraps(f)
|
||||
def inner_translate_exc2(*args, **kwargs):
|
||||
try:
|
||||
return f(*args, **kwargs)
|
||||
@ -736,6 +878,40 @@ def leader_set(settings=None, **kwargs):
|
||||
subprocess.check_call(cmd)
|
||||
|
||||
|
||||
@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
|
||||
def payload_register(ptype, klass, pid):
|
||||
""" is used while a hook is running to let Juju know that a
|
||||
payload has been started."""
|
||||
cmd = ['payload-register']
|
||||
for x in [ptype, klass, pid]:
|
||||
cmd.append(x)
|
||||
subprocess.check_call(cmd)
|
||||
|
||||
|
||||
@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
|
||||
def payload_unregister(klass, pid):
|
||||
""" is used while a hook is running to let Juju know
|
||||
that a payload has been manually stopped. The <class> and <id> provided
|
||||
must match a payload that has been previously registered with juju using
|
||||
payload-register."""
|
||||
cmd = ['payload-unregister']
|
||||
for x in [klass, pid]:
|
||||
cmd.append(x)
|
||||
subprocess.check_call(cmd)
|
||||
|
||||
|
||||
@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
|
||||
def payload_status_set(klass, pid, status):
|
||||
"""is used to update the current status of a registered payload.
|
||||
The <class> and <id> provided must match a payload that has been previously
|
||||
registered with juju using payload-register. The <status> must be one of the
|
||||
follow: starting, started, stopping, stopped"""
|
||||
cmd = ['payload-status-set']
|
||||
for x in [klass, pid, status]:
|
||||
cmd.append(x)
|
||||
subprocess.check_call(cmd)
|
||||
|
||||
|
||||
@cached
|
||||
def juju_version():
|
||||
"""Full version string (eg. '1.23.3.1-trusty-amd64')"""
|
||||
|
@ -63,55 +63,86 @@ def service_reload(service_name, restart_on_failure=False):
|
||||
return service_result
|
||||
|
||||
|
||||
def service_pause(service_name, init_dir=None):
|
||||
def service_pause(service_name, init_dir="/etc/init", initd_dir="/etc/init.d"):
|
||||
"""Pause a system service.
|
||||
|
||||
Stop it, and prevent it from starting again at boot."""
|
||||
if init_dir is None:
|
||||
init_dir = "/etc/init"
|
||||
stopped = service_stop(service_name)
|
||||
# XXX: Support systemd too
|
||||
override_path = os.path.join(
|
||||
init_dir, '{}.conf.override'.format(service_name))
|
||||
with open(override_path, 'w') as fh:
|
||||
fh.write("manual\n")
|
||||
stopped = True
|
||||
if service_running(service_name):
|
||||
stopped = service_stop(service_name)
|
||||
upstart_file = os.path.join(init_dir, "{}.conf".format(service_name))
|
||||
sysv_file = os.path.join(initd_dir, service_name)
|
||||
if init_is_systemd():
|
||||
service('disable', service_name)
|
||||
elif os.path.exists(upstart_file):
|
||||
override_path = os.path.join(
|
||||
init_dir, '{}.override'.format(service_name))
|
||||
with open(override_path, 'w') as fh:
|
||||
fh.write("manual\n")
|
||||
elif os.path.exists(sysv_file):
|
||||
subprocess.check_call(["update-rc.d", service_name, "disable"])
|
||||
else:
|
||||
raise ValueError(
|
||||
"Unable to detect {0} as SystemD, Upstart {1} or"
|
||||
" SysV {2}".format(
|
||||
service_name, upstart_file, sysv_file))
|
||||
return stopped
|
||||
|
||||
|
||||
def service_resume(service_name, init_dir=None):
|
||||
def service_resume(service_name, init_dir="/etc/init",
|
||||
initd_dir="/etc/init.d"):
|
||||
"""Resume a system service.
|
||||
|
||||
Reenable starting again at boot. Start the service"""
|
||||
# XXX: Support systemd too
|
||||
if init_dir is None:
|
||||
init_dir = "/etc/init"
|
||||
override_path = os.path.join(
|
||||
init_dir, '{}.conf.override'.format(service_name))
|
||||
if os.path.exists(override_path):
|
||||
os.unlink(override_path)
|
||||
started = service_start(service_name)
|
||||
upstart_file = os.path.join(init_dir, "{}.conf".format(service_name))
|
||||
sysv_file = os.path.join(initd_dir, service_name)
|
||||
if init_is_systemd():
|
||||
service('enable', service_name)
|
||||
elif os.path.exists(upstart_file):
|
||||
override_path = os.path.join(
|
||||
init_dir, '{}.override'.format(service_name))
|
||||
if os.path.exists(override_path):
|
||||
os.unlink(override_path)
|
||||
elif os.path.exists(sysv_file):
|
||||
subprocess.check_call(["update-rc.d", service_name, "enable"])
|
||||
else:
|
||||
raise ValueError(
|
||||
"Unable to detect {0} as SystemD, Upstart {1} or"
|
||||
" SysV {2}".format(
|
||||
service_name, upstart_file, sysv_file))
|
||||
|
||||
started = service_running(service_name)
|
||||
if not started:
|
||||
started = service_start(service_name)
|
||||
return started
|
||||
|
||||
|
||||
def service(action, service_name):
|
||||
"""Control a system service"""
|
||||
cmd = ['service', service_name, action]
|
||||
if init_is_systemd():
|
||||
cmd = ['systemctl', action, service_name]
|
||||
else:
|
||||
cmd = ['service', service_name, action]
|
||||
return subprocess.call(cmd) == 0
|
||||
|
||||
|
||||
def service_running(service):
|
||||
def service_running(service_name):
|
||||
"""Determine whether a system service is running"""
|
||||
try:
|
||||
output = subprocess.check_output(
|
||||
['service', service, 'status'],
|
||||
stderr=subprocess.STDOUT).decode('UTF-8')
|
||||
except subprocess.CalledProcessError:
|
||||
return False
|
||||
if init_is_systemd():
|
||||
return service('is-active', service_name)
|
||||
else:
|
||||
if ("start/running" in output or "is running" in output):
|
||||
return True
|
||||
else:
|
||||
try:
|
||||
output = subprocess.check_output(
|
||||
['service', service_name, 'status'],
|
||||
stderr=subprocess.STDOUT).decode('UTF-8')
|
||||
except subprocess.CalledProcessError:
|
||||
return False
|
||||
else:
|
||||
if ("start/running" in output or "is running" in output or
|
||||
"up and running" in output):
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
|
||||
def service_available(service_name):
|
||||
@ -126,8 +157,29 @@ def service_available(service_name):
|
||||
return True
|
||||
|
||||
|
||||
def adduser(username, password=None, shell='/bin/bash', system_user=False):
|
||||
"""Add a user to the system"""
|
||||
SYSTEMD_SYSTEM = '/run/systemd/system'
|
||||
|
||||
|
||||
def init_is_systemd():
|
||||
"""Return True if the host system uses systemd, False otherwise."""
|
||||
return os.path.isdir(SYSTEMD_SYSTEM)
|
||||
|
||||
|
||||
def adduser(username, password=None, shell='/bin/bash', system_user=False,
|
||||
primary_group=None, secondary_groups=None):
|
||||
"""Add a user to the system.
|
||||
|
||||
Will log but otherwise succeed if the user already exists.
|
||||
|
||||
:param str username: Username to create
|
||||
:param str password: Password for user; if ``None``, create a system user
|
||||
:param str shell: The default shell for the user
|
||||
:param bool system_user: Whether to create a login or system user
|
||||
:param str primary_group: Primary group for user; defaults to username
|
||||
:param list secondary_groups: Optional list of additional groups
|
||||
|
||||
:returns: The password database entry struct, as returned by `pwd.getpwnam`
|
||||
"""
|
||||
try:
|
||||
user_info = pwd.getpwnam(username)
|
||||
log('user {0} already exists!'.format(username))
|
||||
@ -142,12 +194,32 @@ def adduser(username, password=None, shell='/bin/bash', system_user=False):
|
||||
'--shell', shell,
|
||||
'--password', password,
|
||||
])
|
||||
if not primary_group:
|
||||
try:
|
||||
grp.getgrnam(username)
|
||||
primary_group = username # avoid "group exists" error
|
||||
except KeyError:
|
||||
pass
|
||||
if primary_group:
|
||||
cmd.extend(['-g', primary_group])
|
||||
if secondary_groups:
|
||||
cmd.extend(['-G', ','.join(secondary_groups)])
|
||||
cmd.append(username)
|
||||
subprocess.check_call(cmd)
|
||||
user_info = pwd.getpwnam(username)
|
||||
return user_info
|
||||
|
||||
|
||||
def user_exists(username):
|
||||
"""Check if a user exists"""
|
||||
try:
|
||||
pwd.getpwnam(username)
|
||||
user_exists = True
|
||||
except KeyError:
|
||||
user_exists = False
|
||||
return user_exists
|
||||
|
||||
|
||||
def add_group(group_name, system_group=False):
|
||||
"""Add a group to the system"""
|
||||
try:
|
||||
@ -229,14 +301,12 @@ def write_file(path, content, owner='root', group='root', perms=0o444):
|
||||
|
||||
|
||||
def fstab_remove(mp):
|
||||
"""Remove the given mountpoint entry from /etc/fstab
|
||||
"""
|
||||
"""Remove the given mountpoint entry from /etc/fstab"""
|
||||
return Fstab.remove_by_mountpoint(mp)
|
||||
|
||||
|
||||
def fstab_add(dev, mp, fs, options=None):
|
||||
"""Adds the given device entry to the /etc/fstab file
|
||||
"""
|
||||
"""Adds the given device entry to the /etc/fstab file"""
|
||||
return Fstab.add(dev, mp, fs, options=options)
|
||||
|
||||
|
||||
@ -280,9 +350,19 @@ def mounts():
|
||||
return system_mounts
|
||||
|
||||
|
||||
def fstab_mount(mountpoint):
|
||||
"""Mount filesystem using fstab"""
|
||||
cmd_args = ['mount', mountpoint]
|
||||
try:
|
||||
subprocess.check_output(cmd_args)
|
||||
except subprocess.CalledProcessError as e:
|
||||
log('Error unmounting {}\n{}'.format(mountpoint, e.output))
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def file_hash(path, hash_type='md5'):
|
||||
"""
|
||||
Generate a hash checksum of the contents of 'path' or None if not found.
|
||||
"""Generate a hash checksum of the contents of 'path' or None if not found.
|
||||
|
||||
:param str hash_type: Any hash alrgorithm supported by :mod:`hashlib`,
|
||||
such as md5, sha1, sha256, sha512, etc.
|
||||
@ -297,10 +377,9 @@ def file_hash(path, hash_type='md5'):
|
||||
|
||||
|
||||
def path_hash(path):
|
||||
"""
|
||||
Generate a hash checksum of all files matching 'path'. Standard wildcards
|
||||
like '*' and '?' are supported, see documentation for the 'glob' module for
|
||||
more information.
|
||||
"""Generate a hash checksum of all files matching 'path'. Standard
|
||||
wildcards like '*' and '?' are supported, see documentation for the 'glob'
|
||||
module for more information.
|
||||
|
||||
:return: dict: A { filename: hash } dictionary for all matched files.
|
||||
Empty if none found.
|
||||
@ -312,8 +391,7 @@ def path_hash(path):
|
||||
|
||||
|
||||
def check_hash(path, checksum, hash_type='md5'):
|
||||
"""
|
||||
Validate a file using a cryptographic checksum.
|
||||
"""Validate a file using a cryptographic checksum.
|
||||
|
||||
:param str checksum: Value of the checksum used to validate the file.
|
||||
:param str hash_type: Hash algorithm used to generate `checksum`.
|
||||
@ -328,6 +406,7 @@ def check_hash(path, checksum, hash_type='md5'):
|
||||
|
||||
|
||||
class ChecksumError(ValueError):
|
||||
"""A class derived from Value error to indicate the checksum failed."""
|
||||
pass
|
||||
|
||||
|
||||
@ -396,36 +475,92 @@ def pwgen(length=None):
|
||||
return(''.join(random_chars))
|
||||
|
||||
|
||||
def list_nics(nic_type):
|
||||
'''Return a list of nics of given type(s)'''
|
||||
def is_phy_iface(interface):
|
||||
"""Returns True if interface is not virtual, otherwise False."""
|
||||
if interface:
|
||||
sys_net = '/sys/class/net'
|
||||
if os.path.isdir(sys_net):
|
||||
for iface in glob.glob(os.path.join(sys_net, '*')):
|
||||
if '/virtual/' in os.path.realpath(iface):
|
||||
continue
|
||||
|
||||
if interface == os.path.basename(iface):
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def get_bond_master(interface):
|
||||
"""Returns bond master if interface is bond slave otherwise None.
|
||||
|
||||
NOTE: the provided interface is expected to be physical
|
||||
"""
|
||||
if interface:
|
||||
iface_path = '/sys/class/net/%s' % (interface)
|
||||
if os.path.exists(iface_path):
|
||||
if '/virtual/' in os.path.realpath(iface_path):
|
||||
return None
|
||||
|
||||
master = os.path.join(iface_path, 'master')
|
||||
if os.path.exists(master):
|
||||
master = os.path.realpath(master)
|
||||
# make sure it is a bond master
|
||||
if os.path.exists(os.path.join(master, 'bonding')):
|
||||
return os.path.basename(master)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def list_nics(nic_type=None):
|
||||
"""Return a list of nics of given type(s)"""
|
||||
if isinstance(nic_type, six.string_types):
|
||||
int_types = [nic_type]
|
||||
else:
|
||||
int_types = nic_type
|
||||
|
||||
interfaces = []
|
||||
for int_type in int_types:
|
||||
cmd = ['ip', 'addr', 'show', 'label', int_type + '*']
|
||||
if nic_type:
|
||||
for int_type in int_types:
|
||||
cmd = ['ip', 'addr', 'show', 'label', int_type + '*']
|
||||
ip_output = subprocess.check_output(cmd).decode('UTF-8')
|
||||
ip_output = ip_output.split('\n')
|
||||
ip_output = (line for line in ip_output if line)
|
||||
for line in ip_output:
|
||||
if line.split()[1].startswith(int_type):
|
||||
matched = re.search('.*: (' + int_type +
|
||||
r'[0-9]+\.[0-9]+)@.*', line)
|
||||
if matched:
|
||||
iface = matched.groups()[0]
|
||||
else:
|
||||
iface = line.split()[1].replace(":", "")
|
||||
|
||||
if iface not in interfaces:
|
||||
interfaces.append(iface)
|
||||
else:
|
||||
cmd = ['ip', 'a']
|
||||
ip_output = subprocess.check_output(cmd).decode('UTF-8').split('\n')
|
||||
ip_output = (line for line in ip_output if line)
|
||||
ip_output = (line.strip() for line in ip_output if line)
|
||||
|
||||
key = re.compile('^[0-9]+:\s+(.+):')
|
||||
for line in ip_output:
|
||||
if line.split()[1].startswith(int_type):
|
||||
matched = re.search('.*: (' + int_type + r'[0-9]+\.[0-9]+)@.*', line)
|
||||
if matched:
|
||||
interface = matched.groups()[0]
|
||||
else:
|
||||
interface = line.split()[1].replace(":", "")
|
||||
interfaces.append(interface)
|
||||
matched = re.search(key, line)
|
||||
if matched:
|
||||
iface = matched.group(1)
|
||||
iface = iface.partition("@")[0]
|
||||
if iface not in interfaces:
|
||||
interfaces.append(iface)
|
||||
|
||||
return interfaces
|
||||
|
||||
|
||||
def set_nic_mtu(nic, mtu):
|
||||
'''Set MTU on a network interface'''
|
||||
"""Set the Maximum Transmission Unit (MTU) on a network interface."""
|
||||
cmd = ['ip', 'link', 'set', nic, 'mtu', mtu]
|
||||
subprocess.check_call(cmd)
|
||||
|
||||
|
||||
def get_nic_mtu(nic):
|
||||
"""Return the Maximum Transmission Unit (MTU) for a network interface."""
|
||||
cmd = ['ip', 'addr', 'show', nic]
|
||||
ip_output = subprocess.check_output(cmd).decode('UTF-8').split('\n')
|
||||
mtu = ""
|
||||
@ -437,6 +572,7 @@ def get_nic_mtu(nic):
|
||||
|
||||
|
||||
def get_nic_hwaddr(nic):
|
||||
"""Return the Media Access Control (MAC) for a network interface."""
|
||||
cmd = ['ip', '-o', '-0', 'addr', 'show', nic]
|
||||
ip_output = subprocess.check_output(cmd).decode('UTF-8')
|
||||
hwaddr = ""
|
||||
@ -447,7 +583,7 @@ def get_nic_hwaddr(nic):
|
||||
|
||||
|
||||
def cmp_pkgrevno(package, revno, pkgcache=None):
|
||||
'''Compare supplied revno with the revno of the installed package
|
||||
"""Compare supplied revno with the revno of the installed package
|
||||
|
||||
* 1 => Installed revno is greater than supplied arg
|
||||
* 0 => Installed revno is the same as supplied arg
|
||||
@ -456,7 +592,7 @@ def cmp_pkgrevno(package, revno, pkgcache=None):
|
||||
This function imports apt_cache function from charmhelpers.fetch if
|
||||
the pkgcache argument is None. Be sure to add charmhelpers.fetch if
|
||||
you call this function, or pass an apt_pkg.Cache() instance.
|
||||
'''
|
||||
"""
|
||||
import apt_pkg
|
||||
if not pkgcache:
|
||||
from charmhelpers.fetch import apt_cache
|
||||
@ -466,15 +602,30 @@ def cmp_pkgrevno(package, revno, pkgcache=None):
|
||||
|
||||
|
||||
@contextmanager
|
||||
def chdir(d):
|
||||
def chdir(directory):
|
||||
"""Change the current working directory to a different directory for a code
|
||||
block and return the previous directory after the block exits. Useful to
|
||||
run commands from a specificed directory.
|
||||
|
||||
:param str directory: The directory path to change to for this context.
|
||||
"""
|
||||
cur = os.getcwd()
|
||||
try:
|
||||
yield os.chdir(d)
|
||||
yield os.chdir(directory)
|
||||
finally:
|
||||
os.chdir(cur)
|
||||
|
||||
|
||||
def chownr(path, owner, group, follow_links=True):
|
||||
def chownr(path, owner, group, follow_links=True, chowntopdir=False):
|
||||
"""Recursively change user and group ownership of files and directories
|
||||
in given path. Doesn't chown path itself by default, only its children.
|
||||
|
||||
:param str path: The string path to start changing ownership.
|
||||
:param str owner: The owner string to use when looking up the uid.
|
||||
:param str group: The group string to use when looking up the gid.
|
||||
:param bool follow_links: Also Chown links if True
|
||||
:param bool chowntopdir: Also chown path itself if True
|
||||
"""
|
||||
uid = pwd.getpwnam(owner).pw_uid
|
||||
gid = grp.getgrnam(group).gr_gid
|
||||
if follow_links:
|
||||
@ -482,6 +633,10 @@ def chownr(path, owner, group, follow_links=True):
|
||||
else:
|
||||
chown = os.lchown
|
||||
|
||||
if chowntopdir:
|
||||
broken_symlink = os.path.lexists(path) and not os.path.exists(path)
|
||||
if not broken_symlink:
|
||||
chown(path, uid, gid)
|
||||
for root, dirs, files in os.walk(path):
|
||||
for name in dirs + files:
|
||||
full = os.path.join(root, name)
|
||||
@ -491,4 +646,28 @@ def chownr(path, owner, group, follow_links=True):
|
||||
|
||||
|
||||
def lchownr(path, owner, group):
|
||||
"""Recursively change user and group ownership of files and directories
|
||||
in a given path, not following symbolic links. See the documentation for
|
||||
'os.lchown' for more information.
|
||||
|
||||
:param str path: The string path to start changing ownership.
|
||||
:param str owner: The owner string to use when looking up the uid.
|
||||
:param str group: The group string to use when looking up the gid.
|
||||
"""
|
||||
chownr(path, owner, group, follow_links=False)
|
||||
|
||||
|
||||
def get_total_ram():
|
||||
"""The total amount of system RAM in bytes.
|
||||
|
||||
This is what is reported by the OS, and may be overcommitted when
|
||||
there are multiple containers hosted on the same machine.
|
||||
"""
|
||||
with open('/proc/meminfo', 'r') as f:
|
||||
for line in f.readlines():
|
||||
if line:
|
||||
key, value, unit = line.split()
|
||||
if key == 'MemTotal:':
|
||||
assert unit == 'kB', 'Unknown unit'
|
||||
return int(value) * 1024 # Classic, not KiB.
|
||||
raise NotImplementedError()
|
||||
|
@ -16,7 +16,9 @@
|
||||
|
||||
import os
|
||||
import yaml
|
||||
|
||||
from charmhelpers.core import hookenv
|
||||
from charmhelpers.core import host
|
||||
from charmhelpers.core import templating
|
||||
|
||||
from charmhelpers.core.services.base import ManagerCallback
|
||||
@ -240,27 +242,50 @@ class TemplateCallback(ManagerCallback):
|
||||
|
||||
:param str source: The template source file, relative to
|
||||
`$CHARM_DIR/templates`
|
||||
:param str target: The target to write the rendered template to
|
||||
|
||||
:param str target: The target to write the rendered template to (or None)
|
||||
:param str owner: The owner of the rendered file
|
||||
:param str group: The group of the rendered file
|
||||
:param int perms: The permissions of the rendered file
|
||||
:param partial on_change_action: functools partial to be executed when
|
||||
rendered file changes
|
||||
:param jinja2 loader template_loader: A jinja2 template loader
|
||||
|
||||
:return str: The rendered template
|
||||
"""
|
||||
def __init__(self, source, target,
|
||||
owner='root', group='root', perms=0o444):
|
||||
owner='root', group='root', perms=0o444,
|
||||
on_change_action=None, template_loader=None):
|
||||
self.source = source
|
||||
self.target = target
|
||||
self.owner = owner
|
||||
self.group = group
|
||||
self.perms = perms
|
||||
self.on_change_action = on_change_action
|
||||
self.template_loader = template_loader
|
||||
|
||||
def __call__(self, manager, service_name, event_name):
|
||||
pre_checksum = ''
|
||||
if self.on_change_action and os.path.isfile(self.target):
|
||||
pre_checksum = host.file_hash(self.target)
|
||||
service = manager.get_service(service_name)
|
||||
context = {}
|
||||
context = {'ctx': {}}
|
||||
for ctx in service.get('required_data', []):
|
||||
context.update(ctx)
|
||||
templating.render(self.source, self.target, context,
|
||||
self.owner, self.group, self.perms)
|
||||
context['ctx'].update(ctx)
|
||||
|
||||
result = templating.render(self.source, self.target, context,
|
||||
self.owner, self.group, self.perms,
|
||||
template_loader=self.template_loader)
|
||||
if self.on_change_action:
|
||||
if pre_checksum == host.file_hash(self.target):
|
||||
hookenv.log(
|
||||
'No change detected: {}'.format(self.target),
|
||||
hookenv.DEBUG)
|
||||
else:
|
||||
self.on_change_action()
|
||||
|
||||
return result
|
||||
|
||||
|
||||
# Convenience aliases for templates
|
||||
|
@ -18,6 +18,7 @@
|
||||
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import six
|
||||
import re
|
||||
|
||||
|
||||
def bool_from_string(value):
|
||||
@ -40,3 +41,32 @@ def bool_from_string(value):
|
||||
|
||||
msg = "Unable to interpret string value '%s' as boolean" % (value)
|
||||
raise ValueError(msg)
|
||||
|
||||
|
||||
def bytes_from_string(value):
|
||||
"""Interpret human readable string value as bytes.
|
||||
|
||||
Returns int
|
||||
"""
|
||||
BYTE_POWER = {
|
||||
'K': 1,
|
||||
'KB': 1,
|
||||
'M': 2,
|
||||
'MB': 2,
|
||||
'G': 3,
|
||||
'GB': 3,
|
||||
'T': 4,
|
||||
'TB': 4,
|
||||
'P': 5,
|
||||
'PB': 5,
|
||||
}
|
||||
if isinstance(value, six.string_types):
|
||||
value = six.text_type(value)
|
||||
else:
|
||||
msg = "Unable to interpret non-string value '%s' as boolean" % (value)
|
||||
raise ValueError(msg)
|
||||
matches = re.match("([0-9]+)([a-zA-Z]+)", value)
|
||||
if not matches:
|
||||
msg = "Unable to interpret string value '%s' as bytes" % (value)
|
||||
raise ValueError(msg)
|
||||
return int(matches.group(1)) * (1024 ** BYTE_POWER[matches.group(2)])
|
||||
|
@ -21,13 +21,14 @@ from charmhelpers.core import hookenv
|
||||
|
||||
|
||||
def render(source, target, context, owner='root', group='root',
|
||||
perms=0o444, templates_dir=None, encoding='UTF-8'):
|
||||
perms=0o444, templates_dir=None, encoding='UTF-8', template_loader=None):
|
||||
"""
|
||||
Render a template.
|
||||
|
||||
The `source` path, if not absolute, is relative to the `templates_dir`.
|
||||
|
||||
The `target` path should be absolute.
|
||||
The `target` path should be absolute. It can also be `None`, in which
|
||||
case no file will be written.
|
||||
|
||||
The context should be a dict containing the values to be replaced in the
|
||||
template.
|
||||
@ -36,6 +37,9 @@ def render(source, target, context, owner='root', group='root',
|
||||
|
||||
If omitted, `templates_dir` defaults to the `templates` folder in the charm.
|
||||
|
||||
The rendered template will be written to the file as well as being returned
|
||||
as a string.
|
||||
|
||||
Note: Using this requires python-jinja2; if it is not installed, calling
|
||||
this will attempt to use charmhelpers.fetch.apt_install to install it.
|
||||
"""
|
||||
@ -52,17 +56,26 @@ def render(source, target, context, owner='root', group='root',
|
||||
apt_install('python-jinja2', fatal=True)
|
||||
from jinja2 import FileSystemLoader, Environment, exceptions
|
||||
|
||||
if templates_dir is None:
|
||||
templates_dir = os.path.join(hookenv.charm_dir(), 'templates')
|
||||
loader = Environment(loader=FileSystemLoader(templates_dir))
|
||||
if template_loader:
|
||||
template_env = Environment(loader=template_loader)
|
||||
else:
|
||||
if templates_dir is None:
|
||||
templates_dir = os.path.join(hookenv.charm_dir(), 'templates')
|
||||
template_env = Environment(loader=FileSystemLoader(templates_dir))
|
||||
try:
|
||||
source = source
|
||||
template = loader.get_template(source)
|
||||
template = template_env.get_template(source)
|
||||
except exceptions.TemplateNotFound as e:
|
||||
hookenv.log('Could not load template %s from %s.' %
|
||||
(source, templates_dir),
|
||||
level=hookenv.ERROR)
|
||||
raise e
|
||||
content = template.render(context)
|
||||
host.mkdir(os.path.dirname(target), owner, group, perms=0o755)
|
||||
host.write_file(target, content.encode(encoding), owner, group, perms)
|
||||
if target is not None:
|
||||
target_dir = os.path.dirname(target)
|
||||
if not os.path.exists(target_dir):
|
||||
# This is a terrible default directory permission, as the file
|
||||
# or its siblings will often contain secrets.
|
||||
host.mkdir(os.path.dirname(target), owner, group, perms=0o755)
|
||||
host.write_file(target, content.encode(encoding), owner, group, perms)
|
||||
return content
|
||||
|
@ -152,6 +152,7 @@ associated to the hookname.
|
||||
import collections
|
||||
import contextlib
|
||||
import datetime
|
||||
import itertools
|
||||
import json
|
||||
import os
|
||||
import pprint
|
||||
@ -164,8 +165,7 @@ __author__ = 'Kapil Thangavelu <kapil.foss@gmail.com>'
|
||||
class Storage(object):
|
||||
"""Simple key value database for local unit state within charms.
|
||||
|
||||
Modifications are automatically committed at hook exit. That's
|
||||
currently regardless of exit code.
|
||||
Modifications are not persisted unless :meth:`flush` is called.
|
||||
|
||||
To support dicts, lists, integer, floats, and booleans values
|
||||
are automatically json encoded/decoded.
|
||||
@ -173,8 +173,11 @@ class Storage(object):
|
||||
def __init__(self, path=None):
|
||||
self.db_path = path
|
||||
if path is None:
|
||||
self.db_path = os.path.join(
|
||||
os.environ.get('CHARM_DIR', ''), '.unit-state.db')
|
||||
if 'UNIT_STATE_DB' in os.environ:
|
||||
self.db_path = os.environ['UNIT_STATE_DB']
|
||||
else:
|
||||
self.db_path = os.path.join(
|
||||
os.environ.get('CHARM_DIR', ''), '.unit-state.db')
|
||||
self.conn = sqlite3.connect('%s' % self.db_path)
|
||||
self.cursor = self.conn.cursor()
|
||||
self.revision = None
|
||||
@ -189,15 +192,8 @@ class Storage(object):
|
||||
self.conn.close()
|
||||
self._closed = True
|
||||
|
||||
def _scoped_query(self, stmt, params=None):
|
||||
if params is None:
|
||||
params = []
|
||||
return stmt, params
|
||||
|
||||
def get(self, key, default=None, record=False):
|
||||
self.cursor.execute(
|
||||
*self._scoped_query(
|
||||
'select data from kv where key=?', [key]))
|
||||
self.cursor.execute('select data from kv where key=?', [key])
|
||||
result = self.cursor.fetchone()
|
||||
if not result:
|
||||
return default
|
||||
@ -206,33 +202,81 @@ class Storage(object):
|
||||
return json.loads(result[0])
|
||||
|
||||
def getrange(self, key_prefix, strip=False):
|
||||
stmt = "select key, data from kv where key like '%s%%'" % key_prefix
|
||||
self.cursor.execute(*self._scoped_query(stmt))
|
||||
"""
|
||||
Get a range of keys starting with a common prefix as a mapping of
|
||||
keys to values.
|
||||
|
||||
:param str key_prefix: Common prefix among all keys
|
||||
:param bool strip: Optionally strip the common prefix from the key
|
||||
names in the returned dict
|
||||
:return dict: A (possibly empty) dict of key-value mappings
|
||||
"""
|
||||
self.cursor.execute("select key, data from kv where key like ?",
|
||||
['%s%%' % key_prefix])
|
||||
result = self.cursor.fetchall()
|
||||
|
||||
if not result:
|
||||
return None
|
||||
return {}
|
||||
if not strip:
|
||||
key_prefix = ''
|
||||
return dict([
|
||||
(k[len(key_prefix):], json.loads(v)) for k, v in result])
|
||||
|
||||
def update(self, mapping, prefix=""):
|
||||
"""
|
||||
Set the values of multiple keys at once.
|
||||
|
||||
:param dict mapping: Mapping of keys to values
|
||||
:param str prefix: Optional prefix to apply to all keys in `mapping`
|
||||
before setting
|
||||
"""
|
||||
for k, v in mapping.items():
|
||||
self.set("%s%s" % (prefix, k), v)
|
||||
|
||||
def unset(self, key):
|
||||
"""
|
||||
Remove a key from the database entirely.
|
||||
"""
|
||||
self.cursor.execute('delete from kv where key=?', [key])
|
||||
if self.revision and self.cursor.rowcount:
|
||||
self.cursor.execute(
|
||||
'insert into kv_revisions values (?, ?, ?)',
|
||||
[key, self.revision, json.dumps('DELETED')])
|
||||
|
||||
def unsetrange(self, keys=None, prefix=""):
|
||||
"""
|
||||
Remove a range of keys starting with a common prefix, from the database
|
||||
entirely.
|
||||
|
||||
:param list keys: List of keys to remove.
|
||||
:param str prefix: Optional prefix to apply to all keys in ``keys``
|
||||
before removing.
|
||||
"""
|
||||
if keys is not None:
|
||||
keys = ['%s%s' % (prefix, key) for key in keys]
|
||||
self.cursor.execute('delete from kv where key in (%s)' % ','.join(['?'] * len(keys)), keys)
|
||||
if self.revision and self.cursor.rowcount:
|
||||
self.cursor.execute(
|
||||
'insert into kv_revisions values %s' % ','.join(['(?, ?, ?)'] * len(keys)),
|
||||
list(itertools.chain.from_iterable((key, self.revision, json.dumps('DELETED')) for key in keys)))
|
||||
else:
|
||||
self.cursor.execute('delete from kv where key like ?',
|
||||
['%s%%' % prefix])
|
||||
if self.revision and self.cursor.rowcount:
|
||||
self.cursor.execute(
|
||||
'insert into kv_revisions values (?, ?, ?)',
|
||||
['%s%%' % prefix, self.revision, json.dumps('DELETED')])
|
||||
|
||||
def set(self, key, value):
|
||||
"""
|
||||
Set a value in the database.
|
||||
|
||||
:param str key: Key to set the value for
|
||||
:param value: Any JSON-serializable value to be set
|
||||
"""
|
||||
serialized = json.dumps(value)
|
||||
|
||||
self.cursor.execute(
|
||||
'select data from kv where key=?', [key])
|
||||
self.cursor.execute('select data from kv where key=?', [key])
|
||||
exists = self.cursor.fetchone()
|
||||
|
||||
# Skip mutations to the same value
|
||||
|
@ -90,6 +90,22 @@ CLOUD_ARCHIVE_POCKETS = {
|
||||
'kilo/proposed': 'trusty-proposed/kilo',
|
||||
'trusty-kilo/proposed': 'trusty-proposed/kilo',
|
||||
'trusty-proposed/kilo': 'trusty-proposed/kilo',
|
||||
# Liberty
|
||||
'liberty': 'trusty-updates/liberty',
|
||||
'trusty-liberty': 'trusty-updates/liberty',
|
||||
'trusty-liberty/updates': 'trusty-updates/liberty',
|
||||
'trusty-updates/liberty': 'trusty-updates/liberty',
|
||||
'liberty/proposed': 'trusty-proposed/liberty',
|
||||
'trusty-liberty/proposed': 'trusty-proposed/liberty',
|
||||
'trusty-proposed/liberty': 'trusty-proposed/liberty',
|
||||
# Mitaka
|
||||
'mitaka': 'trusty-updates/mitaka',
|
||||
'trusty-mitaka': 'trusty-updates/mitaka',
|
||||
'trusty-mitaka/updates': 'trusty-updates/mitaka',
|
||||
'trusty-updates/mitaka': 'trusty-updates/mitaka',
|
||||
'mitaka/proposed': 'trusty-proposed/mitaka',
|
||||
'trusty-mitaka/proposed': 'trusty-proposed/mitaka',
|
||||
'trusty-proposed/mitaka': 'trusty-proposed/mitaka',
|
||||
}
|
||||
|
||||
# The order of this list is very important. Handlers should be listed in from
|
||||
@ -217,12 +233,12 @@ def apt_purge(packages, fatal=False):
|
||||
|
||||
def apt_mark(packages, mark, fatal=False):
|
||||
"""Flag one or more packages using apt-mark"""
|
||||
log("Marking {} as {}".format(packages, mark))
|
||||
cmd = ['apt-mark', mark]
|
||||
if isinstance(packages, six.string_types):
|
||||
cmd.append(packages)
|
||||
else:
|
||||
cmd.extend(packages)
|
||||
log("Holding {}".format(packages))
|
||||
|
||||
if fatal:
|
||||
subprocess.check_call(cmd, universal_newlines=True)
|
||||
@ -403,7 +419,7 @@ def plugins(fetch_handlers=None):
|
||||
importlib.import_module(package),
|
||||
classname)
|
||||
plugin_list.append(handler_class())
|
||||
except (ImportError, AttributeError):
|
||||
except NotImplementedError:
|
||||
# Skip missing plugins so that they can be ommitted from
|
||||
# installation if desired
|
||||
log("FetchHandler {} not found, skipping plugin".format(
|
||||
|
@ -108,7 +108,7 @@ class ArchiveUrlFetchHandler(BaseFetchHandler):
|
||||
install_opener(opener)
|
||||
response = urlopen(source)
|
||||
try:
|
||||
with open(dest, 'w') as dest_file:
|
||||
with open(dest, 'wb') as dest_file:
|
||||
dest_file.write(response.read())
|
||||
except Exception as e:
|
||||
if os.path.isfile(dest):
|
||||
|
@ -15,60 +15,50 @@
|
||||
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import os
|
||||
from subprocess import check_call
|
||||
from charmhelpers.fetch import (
|
||||
BaseFetchHandler,
|
||||
UnhandledSource
|
||||
UnhandledSource,
|
||||
filter_installed_packages,
|
||||
apt_install,
|
||||
)
|
||||
from charmhelpers.core.host import mkdir
|
||||
|
||||
import six
|
||||
if six.PY3:
|
||||
raise ImportError('bzrlib does not support Python3')
|
||||
|
||||
try:
|
||||
from bzrlib.branch import Branch
|
||||
from bzrlib import bzrdir, workingtree, errors
|
||||
except ImportError:
|
||||
from charmhelpers.fetch import apt_install
|
||||
apt_install("python-bzrlib")
|
||||
from bzrlib.branch import Branch
|
||||
from bzrlib import bzrdir, workingtree, errors
|
||||
if filter_installed_packages(['bzr']) != []:
|
||||
apt_install(['bzr'])
|
||||
if filter_installed_packages(['bzr']) != []:
|
||||
raise NotImplementedError('Unable to install bzr')
|
||||
|
||||
|
||||
class BzrUrlFetchHandler(BaseFetchHandler):
|
||||
"""Handler for bazaar branches via generic and lp URLs"""
|
||||
def can_handle(self, source):
|
||||
url_parts = self.parse_url(source)
|
||||
if url_parts.scheme not in ('bzr+ssh', 'lp'):
|
||||
if url_parts.scheme not in ('bzr+ssh', 'lp', ''):
|
||||
return False
|
||||
elif not url_parts.scheme:
|
||||
return os.path.exists(os.path.join(source, '.bzr'))
|
||||
else:
|
||||
return True
|
||||
|
||||
def branch(self, source, dest):
|
||||
url_parts = self.parse_url(source)
|
||||
# If we use lp:branchname scheme we need to load plugins
|
||||
if not self.can_handle(source):
|
||||
raise UnhandledSource("Cannot handle {}".format(source))
|
||||
if url_parts.scheme == "lp":
|
||||
from bzrlib.plugin import load_plugins
|
||||
load_plugins()
|
||||
try:
|
||||
local_branch = bzrdir.BzrDir.create_branch_convenience(dest)
|
||||
except errors.AlreadyControlDirError:
|
||||
local_branch = Branch.open(dest)
|
||||
try:
|
||||
remote_branch = Branch.open(source)
|
||||
remote_branch.push(local_branch)
|
||||
tree = workingtree.WorkingTree.open(dest)
|
||||
tree.update()
|
||||
except Exception as e:
|
||||
raise e
|
||||
if os.path.exists(dest):
|
||||
check_call(['bzr', 'pull', '--overwrite', '-d', dest, source])
|
||||
else:
|
||||
check_call(['bzr', 'branch', source, dest])
|
||||
|
||||
def install(self, source):
|
||||
def install(self, source, dest=None):
|
||||
url_parts = self.parse_url(source)
|
||||
branch_name = url_parts.path.strip("/").split("/")[-1]
|
||||
dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched",
|
||||
branch_name)
|
||||
if dest:
|
||||
dest_dir = os.path.join(dest, branch_name)
|
||||
else:
|
||||
dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched",
|
||||
branch_name)
|
||||
|
||||
if not os.path.exists(dest_dir):
|
||||
mkdir(dest_dir, perms=0o755)
|
||||
try:
|
||||
|
@ -15,24 +15,18 @@
|
||||
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import os
|
||||
from subprocess import check_call, CalledProcessError
|
||||
from charmhelpers.fetch import (
|
||||
BaseFetchHandler,
|
||||
UnhandledSource
|
||||
UnhandledSource,
|
||||
filter_installed_packages,
|
||||
apt_install,
|
||||
)
|
||||
from charmhelpers.core.host import mkdir
|
||||
|
||||
import six
|
||||
if six.PY3:
|
||||
raise ImportError('GitPython does not support Python 3')
|
||||
|
||||
try:
|
||||
from git import Repo
|
||||
except ImportError:
|
||||
from charmhelpers.fetch import apt_install
|
||||
apt_install("python-git")
|
||||
from git import Repo
|
||||
|
||||
from git.exc import GitCommandError # noqa E402
|
||||
if filter_installed_packages(['git']) != []:
|
||||
apt_install(['git'])
|
||||
if filter_installed_packages(['git']) != []:
|
||||
raise NotImplementedError('Unable to install git')
|
||||
|
||||
|
||||
class GitUrlFetchHandler(BaseFetchHandler):
|
||||
@ -40,19 +34,24 @@ class GitUrlFetchHandler(BaseFetchHandler):
|
||||
def can_handle(self, source):
|
||||
url_parts = self.parse_url(source)
|
||||
# TODO (mattyw) no support for ssh git@ yet
|
||||
if url_parts.scheme not in ('http', 'https', 'git'):
|
||||
if url_parts.scheme not in ('http', 'https', 'git', ''):
|
||||
return False
|
||||
elif not url_parts.scheme:
|
||||
return os.path.exists(os.path.join(source, '.git'))
|
||||
else:
|
||||
return True
|
||||
|
||||
def clone(self, source, dest, branch, depth=None):
|
||||
def clone(self, source, dest, branch="master", depth=None):
|
||||
if not self.can_handle(source):
|
||||
raise UnhandledSource("Cannot handle {}".format(source))
|
||||
|
||||
if depth:
|
||||
Repo.clone_from(source, dest, branch=branch, depth=depth)
|
||||
if os.path.exists(dest):
|
||||
cmd = ['git', '-C', dest, 'pull', source, branch]
|
||||
else:
|
||||
Repo.clone_from(source, dest, branch=branch)
|
||||
cmd = ['git', 'clone', source, dest, '--branch', branch]
|
||||
if depth:
|
||||
cmd.extend(['--depth', depth])
|
||||
check_call(cmd)
|
||||
|
||||
def install(self, source, branch="master", dest=None, depth=None):
|
||||
url_parts = self.parse_url(source)
|
||||
@ -62,11 +61,9 @@ class GitUrlFetchHandler(BaseFetchHandler):
|
||||
else:
|
||||
dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched",
|
||||
branch_name)
|
||||
if not os.path.exists(dest_dir):
|
||||
mkdir(dest_dir, perms=0o755)
|
||||
try:
|
||||
self.clone(source, dest_dir, branch, depth)
|
||||
except GitCommandError as e:
|
||||
except CalledProcessError as e:
|
||||
raise UnhandledSource(e)
|
||||
except OSError as e:
|
||||
raise UnhandledSource(e.strerror)
|
||||
|
Loading…
Reference in New Issue
Block a user