819 lines
35 KiB
Python
819 lines
35 KiB
Python
# Copyright 2014-2015 Canonical Limited.
|
|
#
|
|
# This file is part of charm-helpers.
|
|
#
|
|
# charm-helpers is free software: you can redistribute it and/or modify
|
|
# it under the terms of the GNU Lesser General Public License version 3 as
|
|
# published by the Free Software Foundation.
|
|
#
|
|
# charm-helpers is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU Lesser General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU Lesser General Public License
|
|
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
import io
|
|
import json
|
|
import logging
|
|
import os
|
|
import re
|
|
import socket
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import uuid
|
|
|
|
import amulet
|
|
import distro_info
|
|
import six
|
|
from six.moves import configparser
|
|
if six.PY3:
|
|
from urllib import parse as urlparse
|
|
else:
|
|
import urlparse
|
|
|
|
|
|
class AmuletUtils(object):
|
|
"""Amulet utilities.
|
|
|
|
This class provides common utility functions that are used by Amulet
|
|
tests.
|
|
"""
|
|
|
|
def __init__(self, log_level=logging.ERROR):
|
|
self.log = self.get_logger(level=log_level)
|
|
self.ubuntu_releases = self.get_ubuntu_releases()
|
|
|
|
def get_logger(self, name="amulet-logger", level=logging.DEBUG):
|
|
"""Get a logger object that will log to stdout."""
|
|
log = logging
|
|
logger = log.getLogger(name)
|
|
fmt = log.Formatter("%(asctime)s %(funcName)s "
|
|
"%(levelname)s: %(message)s")
|
|
|
|
handler = log.StreamHandler(stream=sys.stdout)
|
|
handler.setLevel(level)
|
|
handler.setFormatter(fmt)
|
|
|
|
logger.addHandler(handler)
|
|
logger.setLevel(level)
|
|
|
|
return logger
|
|
|
|
def valid_ip(self, ip):
|
|
if re.match(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$", ip):
|
|
return True
|
|
else:
|
|
return False
|
|
|
|
def valid_url(self, url):
|
|
p = re.compile(
|
|
r'^(?:http|ftp)s?://'
|
|
r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' # noqa
|
|
r'localhost|'
|
|
r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})'
|
|
r'(?::\d+)?'
|
|
r'(?:/?|[/?]\S+)$',
|
|
re.IGNORECASE)
|
|
if p.match(url):
|
|
return True
|
|
else:
|
|
return False
|
|
|
|
def get_ubuntu_release_from_sentry(self, sentry_unit):
|
|
"""Get Ubuntu release codename from sentry unit.
|
|
|
|
:param sentry_unit: amulet sentry/service unit pointer
|
|
:returns: list of strings - release codename, failure message
|
|
"""
|
|
msg = None
|
|
cmd = 'lsb_release -cs'
|
|
release, code = sentry_unit.run(cmd)
|
|
if code == 0:
|
|
self.log.debug('{} lsb_release: {}'.format(
|
|
sentry_unit.info['unit_name'], release))
|
|
else:
|
|
msg = ('{} `{}` returned {} '
|
|
'{}'.format(sentry_unit.info['unit_name'],
|
|
cmd, release, code))
|
|
if release not in self.ubuntu_releases:
|
|
msg = ("Release ({}) not found in Ubuntu releases "
|
|
"({})".format(release, self.ubuntu_releases))
|
|
return release, msg
|
|
|
|
def validate_services(self, commands):
|
|
"""Validate that lists of commands succeed on service units. Can be
|
|
used to verify system services are running on the corresponding
|
|
service units.
|
|
|
|
:param commands: dict with sentry keys and arbitrary command list vals
|
|
:returns: None if successful, Failure string message otherwise
|
|
"""
|
|
self.log.debug('Checking status of system services...')
|
|
|
|
# /!\ DEPRECATION WARNING (beisner):
|
|
# New and existing tests should be rewritten to use
|
|
# validate_services_by_name() as it is aware of init systems.
|
|
self.log.warn('DEPRECATION WARNING: use '
|
|
'validate_services_by_name instead of validate_services '
|
|
'due to init system differences.')
|
|
|
|
for k, v in six.iteritems(commands):
|
|
for cmd in v:
|
|
output, code = k.run(cmd)
|
|
self.log.debug('{} `{}` returned '
|
|
'{}'.format(k.info['unit_name'],
|
|
cmd, code))
|
|
if code != 0:
|
|
return "command `{}` returned {}".format(cmd, str(code))
|
|
return None
|
|
|
|
def validate_services_by_name(self, sentry_services):
|
|
"""Validate system service status by service name, automatically
|
|
detecting init system based on Ubuntu release codename.
|
|
|
|
:param sentry_services: dict with sentry keys and svc list values
|
|
:returns: None if successful, Failure string message otherwise
|
|
"""
|
|
self.log.debug('Checking status of system services...')
|
|
|
|
# Point at which systemd became a thing
|
|
systemd_switch = self.ubuntu_releases.index('vivid')
|
|
|
|
for sentry_unit, services_list in six.iteritems(sentry_services):
|
|
# Get lsb_release codename from unit
|
|
release, ret = self.get_ubuntu_release_from_sentry(sentry_unit)
|
|
if ret:
|
|
return ret
|
|
|
|
for service_name in services_list:
|
|
if (self.ubuntu_releases.index(release) >= systemd_switch or
|
|
service_name in ['rabbitmq-server', 'apache2']):
|
|
# init is systemd (or regular sysv)
|
|
cmd = 'sudo service {} status'.format(service_name)
|
|
output, code = sentry_unit.run(cmd)
|
|
service_running = code == 0
|
|
elif self.ubuntu_releases.index(release) < systemd_switch:
|
|
# init is upstart
|
|
cmd = 'sudo status {}'.format(service_name)
|
|
output, code = sentry_unit.run(cmd)
|
|
service_running = code == 0 and "start/running" in output
|
|
|
|
self.log.debug('{} `{}` returned '
|
|
'{}'.format(sentry_unit.info['unit_name'],
|
|
cmd, code))
|
|
if not service_running:
|
|
return u"command `{}` returned {} {}".format(
|
|
cmd, output, str(code))
|
|
return None
|
|
|
|
def _get_config(self, unit, filename):
|
|
"""Get a ConfigParser object for parsing a unit's config file."""
|
|
file_contents = unit.file_contents(filename)
|
|
|
|
# NOTE(beisner): by default, ConfigParser does not handle options
|
|
# with no value, such as the flags used in the mysql my.cnf file.
|
|
# https://bugs.python.org/issue7005
|
|
config = configparser.ConfigParser(allow_no_value=True)
|
|
config.readfp(io.StringIO(file_contents))
|
|
return config
|
|
|
|
def validate_config_data(self, sentry_unit, config_file, section,
|
|
expected):
|
|
"""Validate config file data.
|
|
|
|
Verify that the specified section of the config file contains
|
|
the expected option key:value pairs.
|
|
|
|
Compare expected dictionary data vs actual dictionary data.
|
|
The values in the 'expected' dictionary can be strings, bools, ints,
|
|
longs, or can be a function that evaluates a variable and returns a
|
|
bool.
|
|
"""
|
|
self.log.debug('Validating config file data ({} in {} on {})'
|
|
'...'.format(section, config_file,
|
|
sentry_unit.info['unit_name']))
|
|
config = self._get_config(sentry_unit, config_file)
|
|
|
|
if section != 'DEFAULT' and not config.has_section(section):
|
|
return "section [{}] does not exist".format(section)
|
|
|
|
for k in expected.keys():
|
|
if not config.has_option(section, k):
|
|
return "section [{}] is missing option {}".format(section, k)
|
|
|
|
actual = config.get(section, k)
|
|
v = expected[k]
|
|
if (isinstance(v, six.string_types) or
|
|
isinstance(v, bool) or
|
|
isinstance(v, six.integer_types)):
|
|
# handle explicit values
|
|
if actual != v:
|
|
return "section [{}] {}:{} != expected {}:{}".format(
|
|
section, k, actual, k, expected[k])
|
|
# handle function pointers, such as not_null or valid_ip
|
|
elif not v(actual):
|
|
return "section [{}] {}:{} != expected {}:{}".format(
|
|
section, k, actual, k, expected[k])
|
|
return None
|
|
|
|
def _validate_dict_data(self, expected, actual):
|
|
"""Validate dictionary data.
|
|
|
|
Compare expected dictionary data vs actual dictionary data.
|
|
The values in the 'expected' dictionary can be strings, bools, ints,
|
|
longs, or can be a function that evaluates a variable and returns a
|
|
bool.
|
|
"""
|
|
self.log.debug('actual: {}'.format(repr(actual)))
|
|
self.log.debug('expected: {}'.format(repr(expected)))
|
|
|
|
for k, v in six.iteritems(expected):
|
|
if k in actual:
|
|
if (isinstance(v, six.string_types) or
|
|
isinstance(v, bool) or
|
|
isinstance(v, six.integer_types)):
|
|
# handle explicit values
|
|
if v != actual[k]:
|
|
return "{}:{}".format(k, actual[k])
|
|
# handle function pointers, such as not_null or valid_ip
|
|
elif not v(actual[k]):
|
|
return "{}:{}".format(k, actual[k])
|
|
else:
|
|
return "key '{}' does not exist".format(k)
|
|
return None
|
|
|
|
def validate_relation_data(self, sentry_unit, relation, expected):
|
|
"""Validate actual relation data based on expected relation data."""
|
|
actual = sentry_unit.relation(relation[0], relation[1])
|
|
return self._validate_dict_data(expected, actual)
|
|
|
|
def _validate_list_data(self, expected, actual):
|
|
"""Compare expected list vs actual list data."""
|
|
for e in expected:
|
|
if e not in actual:
|
|
return "expected item {} not found in actual list".format(e)
|
|
return None
|
|
|
|
def not_null(self, string):
|
|
if string is not None:
|
|
return True
|
|
else:
|
|
return False
|
|
|
|
def _get_file_mtime(self, sentry_unit, filename):
|
|
"""Get last modification time of file."""
|
|
return sentry_unit.file_stat(filename)['mtime']
|
|
|
|
def _get_dir_mtime(self, sentry_unit, directory):
|
|
"""Get last modification time of directory."""
|
|
return sentry_unit.directory_stat(directory)['mtime']
|
|
|
|
def _get_proc_start_time(self, sentry_unit, service, pgrep_full=None):
|
|
"""Get start time of a process based on the last modification time
|
|
of the /proc/pid directory.
|
|
|
|
:sentry_unit: The sentry unit to check for the service on
|
|
:service: service name to look for in process table
|
|
:pgrep_full: [Deprecated] Use full command line search mode with pgrep
|
|
:returns: epoch time of service process start
|
|
:param commands: list of bash commands
|
|
:param sentry_units: list of sentry unit pointers
|
|
:returns: None if successful; Failure message otherwise
|
|
"""
|
|
if pgrep_full is not None:
|
|
# /!\ DEPRECATION WARNING (beisner):
|
|
# No longer implemented, as pidof is now used instead of pgrep.
|
|
# https://bugs.launchpad.net/charm-helpers/+bug/1474030
|
|
self.log.warn('DEPRECATION WARNING: pgrep_full bool is no '
|
|
'longer implemented re: lp 1474030.')
|
|
|
|
pid_list = self.get_process_id_list(sentry_unit, service)
|
|
pid = pid_list[0]
|
|
proc_dir = '/proc/{}'.format(pid)
|
|
self.log.debug('Pid for {} on {}: {}'.format(
|
|
service, sentry_unit.info['unit_name'], pid))
|
|
|
|
return self._get_dir_mtime(sentry_unit, proc_dir)
|
|
|
|
def service_restarted(self, sentry_unit, service, filename,
|
|
pgrep_full=None, sleep_time=20):
|
|
"""Check if service was restarted.
|
|
|
|
Compare a service's start time vs a file's last modification time
|
|
(such as a config file for that service) to determine if the service
|
|
has been restarted.
|
|
"""
|
|
# /!\ DEPRECATION WARNING (beisner):
|
|
# This method is prone to races in that no before-time is known.
|
|
# Use validate_service_config_changed instead.
|
|
|
|
# NOTE(beisner) pgrep_full is no longer implemented, as pidof is now
|
|
# used instead of pgrep. pgrep_full is still passed through to ensure
|
|
# deprecation WARNS. lp1474030
|
|
self.log.warn('DEPRECATION WARNING: use '
|
|
'validate_service_config_changed instead of '
|
|
'service_restarted due to known races.')
|
|
|
|
time.sleep(sleep_time)
|
|
if (self._get_proc_start_time(sentry_unit, service, pgrep_full) >=
|
|
self._get_file_mtime(sentry_unit, filename)):
|
|
return True
|
|
else:
|
|
return False
|
|
|
|
def service_restarted_since(self, sentry_unit, mtime, service,
|
|
pgrep_full=None, sleep_time=20,
|
|
retry_count=30, retry_sleep_time=10):
|
|
"""Check if service was been started after a given time.
|
|
|
|
Args:
|
|
sentry_unit (sentry): The sentry unit to check for the service on
|
|
mtime (float): The epoch time to check against
|
|
service (string): service name to look for in process table
|
|
pgrep_full: [Deprecated] Use full command line search mode with pgrep
|
|
sleep_time (int): Initial sleep time (s) before looking for file
|
|
retry_sleep_time (int): Time (s) to sleep between retries
|
|
retry_count (int): If file is not found, how many times to retry
|
|
|
|
Returns:
|
|
bool: True if service found and its start time it newer than mtime,
|
|
False if service is older than mtime or if service was
|
|
not found.
|
|
"""
|
|
# NOTE(beisner) pgrep_full is no longer implemented, as pidof is now
|
|
# used instead of pgrep. pgrep_full is still passed through to ensure
|
|
# deprecation WARNS. lp1474030
|
|
|
|
unit_name = sentry_unit.info['unit_name']
|
|
self.log.debug('Checking that %s service restarted since %s on '
|
|
'%s' % (service, mtime, unit_name))
|
|
time.sleep(sleep_time)
|
|
proc_start_time = None
|
|
tries = 0
|
|
while tries <= retry_count and not proc_start_time:
|
|
try:
|
|
proc_start_time = self._get_proc_start_time(sentry_unit,
|
|
service,
|
|
pgrep_full)
|
|
self.log.debug('Attempt {} to get {} proc start time on {} '
|
|
'OK'.format(tries, service, unit_name))
|
|
except IOError as e:
|
|
# NOTE(beisner) - race avoidance, proc may not exist yet.
|
|
# https://bugs.launchpad.net/charm-helpers/+bug/1474030
|
|
self.log.debug('Attempt {} to get {} proc start time on {} '
|
|
'failed\n{}'.format(tries, service,
|
|
unit_name, e))
|
|
time.sleep(retry_sleep_time)
|
|
tries += 1
|
|
|
|
if not proc_start_time:
|
|
self.log.warn('No proc start time found, assuming service did '
|
|
'not start')
|
|
return False
|
|
if proc_start_time >= mtime:
|
|
self.log.debug('Proc start time is newer than provided mtime'
|
|
'(%s >= %s) on %s (OK)' % (proc_start_time,
|
|
mtime, unit_name))
|
|
return True
|
|
else:
|
|
self.log.warn('Proc start time (%s) is older than provided mtime '
|
|
'(%s) on %s, service did not '
|
|
'restart' % (proc_start_time, mtime, unit_name))
|
|
return False
|
|
|
|
def config_updated_since(self, sentry_unit, filename, mtime,
|
|
sleep_time=20, retry_count=30,
|
|
retry_sleep_time=10):
|
|
"""Check if file was modified after a given time.
|
|
|
|
Args:
|
|
sentry_unit (sentry): The sentry unit to check the file mtime on
|
|
filename (string): The file to check mtime of
|
|
mtime (float): The epoch time to check against
|
|
sleep_time (int): Initial sleep time (s) before looking for file
|
|
retry_sleep_time (int): Time (s) to sleep between retries
|
|
retry_count (int): If file is not found, how many times to retry
|
|
|
|
Returns:
|
|
bool: True if file was modified more recently than mtime, False if
|
|
file was modified before mtime, or if file not found.
|
|
"""
|
|
unit_name = sentry_unit.info['unit_name']
|
|
self.log.debug('Checking that %s updated since %s on '
|
|
'%s' % (filename, mtime, unit_name))
|
|
time.sleep(sleep_time)
|
|
file_mtime = None
|
|
tries = 0
|
|
while tries <= retry_count and not file_mtime:
|
|
try:
|
|
file_mtime = self._get_file_mtime(sentry_unit, filename)
|
|
self.log.debug('Attempt {} to get {} file mtime on {} '
|
|
'OK'.format(tries, filename, unit_name))
|
|
except IOError as e:
|
|
# NOTE(beisner) - race avoidance, file may not exist yet.
|
|
# https://bugs.launchpad.net/charm-helpers/+bug/1474030
|
|
self.log.debug('Attempt {} to get {} file mtime on {} '
|
|
'failed\n{}'.format(tries, filename,
|
|
unit_name, e))
|
|
time.sleep(retry_sleep_time)
|
|
tries += 1
|
|
|
|
if not file_mtime:
|
|
self.log.warn('Could not determine file mtime, assuming '
|
|
'file does not exist')
|
|
return False
|
|
|
|
if file_mtime >= mtime:
|
|
self.log.debug('File mtime is newer than provided mtime '
|
|
'(%s >= %s) on %s (OK)' % (file_mtime,
|
|
mtime, unit_name))
|
|
return True
|
|
else:
|
|
self.log.warn('File mtime is older than provided mtime'
|
|
'(%s < on %s) on %s' % (file_mtime,
|
|
mtime, unit_name))
|
|
return False
|
|
|
|
def validate_service_config_changed(self, sentry_unit, mtime, service,
|
|
filename, pgrep_full=None,
|
|
sleep_time=20, retry_count=30,
|
|
retry_sleep_time=10):
|
|
"""Check service and file were updated after mtime
|
|
|
|
Args:
|
|
sentry_unit (sentry): The sentry unit to check for the service on
|
|
mtime (float): The epoch time to check against
|
|
service (string): service name to look for in process table
|
|
filename (string): The file to check mtime of
|
|
pgrep_full: [Deprecated] Use full command line search mode with pgrep
|
|
sleep_time (int): Initial sleep in seconds to pass to test helpers
|
|
retry_count (int): If service is not found, how many times to retry
|
|
retry_sleep_time (int): Time in seconds to wait between retries
|
|
|
|
Typical Usage:
|
|
u = OpenStackAmuletUtils(ERROR)
|
|
...
|
|
mtime = u.get_sentry_time(self.cinder_sentry)
|
|
self.d.configure('cinder', {'verbose': 'True', 'debug': 'True'})
|
|
if not u.validate_service_config_changed(self.cinder_sentry,
|
|
mtime,
|
|
'cinder-api',
|
|
'/etc/cinder/cinder.conf')
|
|
amulet.raise_status(amulet.FAIL, msg='update failed')
|
|
Returns:
|
|
bool: True if both service and file where updated/restarted after
|
|
mtime, False if service is older than mtime or if service was
|
|
not found or if filename was modified before mtime.
|
|
"""
|
|
|
|
# NOTE(beisner) pgrep_full is no longer implemented, as pidof is now
|
|
# used instead of pgrep. pgrep_full is still passed through to ensure
|
|
# deprecation WARNS. lp1474030
|
|
|
|
service_restart = self.service_restarted_since(
|
|
sentry_unit, mtime,
|
|
service,
|
|
pgrep_full=pgrep_full,
|
|
sleep_time=sleep_time,
|
|
retry_count=retry_count,
|
|
retry_sleep_time=retry_sleep_time)
|
|
|
|
config_update = self.config_updated_since(
|
|
sentry_unit,
|
|
filename,
|
|
mtime,
|
|
sleep_time=sleep_time,
|
|
retry_count=retry_count,
|
|
retry_sleep_time=retry_sleep_time)
|
|
|
|
return service_restart and config_update
|
|
|
|
def get_sentry_time(self, sentry_unit):
|
|
"""Return current epoch time on a sentry"""
|
|
cmd = "date +'%s'"
|
|
return float(sentry_unit.run(cmd)[0])
|
|
|
|
def relation_error(self, name, data):
|
|
return 'unexpected relation data in {} - {}'.format(name, data)
|
|
|
|
def endpoint_error(self, name, data):
|
|
return 'unexpected endpoint data in {} - {}'.format(name, data)
|
|
|
|
def get_ubuntu_releases(self):
|
|
"""Return a list of all Ubuntu releases in order of release."""
|
|
_d = distro_info.UbuntuDistroInfo()
|
|
_release_list = _d.all
|
|
return _release_list
|
|
|
|
def file_to_url(self, file_rel_path):
|
|
"""Convert a relative file path to a file URL."""
|
|
_abs_path = os.path.abspath(file_rel_path)
|
|
return urlparse.urlparse(_abs_path, scheme='file').geturl()
|
|
|
|
def check_commands_on_units(self, commands, sentry_units):
|
|
"""Check that all commands in a list exit zero on all
|
|
sentry units in a list.
|
|
|
|
:param commands: list of bash commands
|
|
:param sentry_units: list of sentry unit pointers
|
|
:returns: None if successful; Failure message otherwise
|
|
"""
|
|
self.log.debug('Checking exit codes for {} commands on {} '
|
|
'sentry units...'.format(len(commands),
|
|
len(sentry_units)))
|
|
for sentry_unit in sentry_units:
|
|
for cmd in commands:
|
|
output, code = sentry_unit.run(cmd)
|
|
if code == 0:
|
|
self.log.debug('{} `{}` returned {} '
|
|
'(OK)'.format(sentry_unit.info['unit_name'],
|
|
cmd, code))
|
|
else:
|
|
return ('{} `{}` returned {} '
|
|
'{}'.format(sentry_unit.info['unit_name'],
|
|
cmd, code, output))
|
|
return None
|
|
|
|
def get_process_id_list(self, sentry_unit, process_name,
|
|
expect_success=True):
|
|
"""Get a list of process ID(s) from a single sentry juju unit
|
|
for a single process name.
|
|
|
|
:param sentry_unit: Amulet sentry instance (juju unit)
|
|
:param process_name: Process name
|
|
:param expect_success: If False, expect the PID to be missing,
|
|
raise if it is present.
|
|
:returns: List of process IDs
|
|
"""
|
|
cmd = 'pidof -x {}'.format(process_name)
|
|
if not expect_success:
|
|
cmd += " || exit 0 && exit 1"
|
|
output, code = sentry_unit.run(cmd)
|
|
if code != 0:
|
|
msg = ('{} `{}` returned {} '
|
|
'{}'.format(sentry_unit.info['unit_name'],
|
|
cmd, code, output))
|
|
amulet.raise_status(amulet.FAIL, msg=msg)
|
|
return str(output).split()
|
|
|
|
def get_unit_process_ids(self, unit_processes, expect_success=True):
|
|
"""Construct a dict containing unit sentries, process names, and
|
|
process IDs.
|
|
|
|
:param unit_processes: A dictionary of Amulet sentry instance
|
|
to list of process names.
|
|
:param expect_success: if False expect the processes to not be
|
|
running, raise if they are.
|
|
:returns: Dictionary of Amulet sentry instance to dictionary
|
|
of process names to PIDs.
|
|
"""
|
|
pid_dict = {}
|
|
for sentry_unit, process_list in six.iteritems(unit_processes):
|
|
pid_dict[sentry_unit] = {}
|
|
for process in process_list:
|
|
pids = self.get_process_id_list(
|
|
sentry_unit, process, expect_success=expect_success)
|
|
pid_dict[sentry_unit].update({process: pids})
|
|
return pid_dict
|
|
|
|
def validate_unit_process_ids(self, expected, actual):
|
|
"""Validate process id quantities for services on units."""
|
|
self.log.debug('Checking units for running processes...')
|
|
self.log.debug('Expected PIDs: {}'.format(expected))
|
|
self.log.debug('Actual PIDs: {}'.format(actual))
|
|
|
|
if len(actual) != len(expected):
|
|
return ('Unit count mismatch. expected, actual: {}, '
|
|
'{} '.format(len(expected), len(actual)))
|
|
|
|
for (e_sentry, e_proc_names) in six.iteritems(expected):
|
|
e_sentry_name = e_sentry.info['unit_name']
|
|
if e_sentry in actual.keys():
|
|
a_proc_names = actual[e_sentry]
|
|
else:
|
|
return ('Expected sentry ({}) not found in actual dict data.'
|
|
'{}'.format(e_sentry_name, e_sentry))
|
|
|
|
if len(e_proc_names.keys()) != len(a_proc_names.keys()):
|
|
return ('Process name count mismatch. expected, actual: {}, '
|
|
'{}'.format(len(expected), len(actual)))
|
|
|
|
for (e_proc_name, e_pids_length), (a_proc_name, a_pids) in \
|
|
zip(e_proc_names.items(), a_proc_names.items()):
|
|
if e_proc_name != a_proc_name:
|
|
return ('Process name mismatch. expected, actual: {}, '
|
|
'{}'.format(e_proc_name, a_proc_name))
|
|
|
|
a_pids_length = len(a_pids)
|
|
fail_msg = ('PID count mismatch. {} ({}) expected, actual: '
|
|
'{}, {} ({})'.format(e_sentry_name, e_proc_name,
|
|
e_pids_length, a_pids_length,
|
|
a_pids))
|
|
|
|
# If expected is not bool, ensure PID quantities match
|
|
if not isinstance(e_pids_length, bool) and \
|
|
a_pids_length != e_pids_length:
|
|
return fail_msg
|
|
# If expected is bool True, ensure 1 or more PIDs exist
|
|
elif isinstance(e_pids_length, bool) and \
|
|
e_pids_length is True and a_pids_length < 1:
|
|
return fail_msg
|
|
# If expected is bool False, ensure 0 PIDs exist
|
|
elif isinstance(e_pids_length, bool) and \
|
|
e_pids_length is False and a_pids_length != 0:
|
|
return fail_msg
|
|
else:
|
|
self.log.debug('PID check OK: {} {} {}: '
|
|
'{}'.format(e_sentry_name, e_proc_name,
|
|
e_pids_length, a_pids))
|
|
return None
|
|
|
|
def validate_list_of_identical_dicts(self, list_of_dicts):
|
|
"""Check that all dicts within a list are identical."""
|
|
hashes = []
|
|
for _dict in list_of_dicts:
|
|
hashes.append(hash(frozenset(_dict.items())))
|
|
|
|
self.log.debug('Hashes: {}'.format(hashes))
|
|
if len(set(hashes)) == 1:
|
|
self.log.debug('Dicts within list are identical')
|
|
else:
|
|
return 'Dicts within list are not identical'
|
|
|
|
return None
|
|
|
|
def validate_sectionless_conf(self, file_contents, expected):
|
|
"""A crude conf parser. Useful to inspect configuration files which
|
|
do not have section headers (as would be necessary in order to use
|
|
the configparser). Such as openstack-dashboard or rabbitmq confs."""
|
|
for line in file_contents.split('\n'):
|
|
if '=' in line:
|
|
args = line.split('=')
|
|
if len(args) <= 1:
|
|
continue
|
|
key = args[0].strip()
|
|
value = args[1].strip()
|
|
if key in expected.keys():
|
|
if expected[key] != value:
|
|
msg = ('Config mismatch. Expected, actual: {}, '
|
|
'{}'.format(expected[key], value))
|
|
amulet.raise_status(amulet.FAIL, msg=msg)
|
|
|
|
def get_unit_hostnames(self, units):
|
|
"""Return a dict of juju unit names to hostnames."""
|
|
host_names = {}
|
|
for unit in units:
|
|
host_names[unit.info['unit_name']] = \
|
|
str(unit.file_contents('/etc/hostname').strip())
|
|
self.log.debug('Unit host names: {}'.format(host_names))
|
|
return host_names
|
|
|
|
def run_cmd_unit(self, sentry_unit, cmd):
|
|
"""Run a command on a unit, return the output and exit code."""
|
|
output, code = sentry_unit.run(cmd)
|
|
if code == 0:
|
|
self.log.debug('{} `{}` command returned {} '
|
|
'(OK)'.format(sentry_unit.info['unit_name'],
|
|
cmd, code))
|
|
else:
|
|
msg = ('{} `{}` command returned {} '
|
|
'{}'.format(sentry_unit.info['unit_name'],
|
|
cmd, code, output))
|
|
amulet.raise_status(amulet.FAIL, msg=msg)
|
|
return str(output), code
|
|
|
|
def file_exists_on_unit(self, sentry_unit, file_name):
|
|
"""Check if a file exists on a unit."""
|
|
try:
|
|
sentry_unit.file_stat(file_name)
|
|
return True
|
|
except IOError:
|
|
return False
|
|
except Exception as e:
|
|
msg = 'Error checking file {}: {}'.format(file_name, e)
|
|
amulet.raise_status(amulet.FAIL, msg=msg)
|
|
|
|
def file_contents_safe(self, sentry_unit, file_name,
|
|
max_wait=60, fatal=False):
|
|
"""Get file contents from a sentry unit. Wrap amulet file_contents
|
|
with retry logic to address races where a file checks as existing,
|
|
but no longer exists by the time file_contents is called.
|
|
Return None if file not found. Optionally raise if fatal is True."""
|
|
unit_name = sentry_unit.info['unit_name']
|
|
file_contents = False
|
|
tries = 0
|
|
while not file_contents and tries < (max_wait / 4):
|
|
try:
|
|
file_contents = sentry_unit.file_contents(file_name)
|
|
except IOError:
|
|
self.log.debug('Attempt {} to open file {} from {} '
|
|
'failed'.format(tries, file_name,
|
|
unit_name))
|
|
time.sleep(4)
|
|
tries += 1
|
|
|
|
if file_contents:
|
|
return file_contents
|
|
elif not fatal:
|
|
return None
|
|
elif fatal:
|
|
msg = 'Failed to get file contents from unit.'
|
|
amulet.raise_status(amulet.FAIL, msg)
|
|
|
|
def port_knock_tcp(self, host="localhost", port=22, timeout=15):
|
|
"""Open a TCP socket to check for a listening sevice on a host.
|
|
|
|
:param host: host name or IP address, default to localhost
|
|
:param port: TCP port number, default to 22
|
|
:param timeout: Connect timeout, default to 15 seconds
|
|
:returns: True if successful, False if connect failed
|
|
"""
|
|
|
|
# Resolve host name if possible
|
|
try:
|
|
connect_host = socket.gethostbyname(host)
|
|
host_human = "{} ({})".format(connect_host, host)
|
|
except socket.error as e:
|
|
self.log.warn('Unable to resolve address: '
|
|
'{} ({}) Trying anyway!'.format(host, e))
|
|
connect_host = host
|
|
host_human = connect_host
|
|
|
|
# Attempt socket connection
|
|
try:
|
|
knock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
|
knock.settimeout(timeout)
|
|
knock.connect((connect_host, port))
|
|
knock.close()
|
|
self.log.debug('Socket connect OK for host '
|
|
'{} on port {}.'.format(host_human, port))
|
|
return True
|
|
except socket.error as e:
|
|
self.log.debug('Socket connect FAIL for'
|
|
' {} port {} ({})'.format(host_human, port, e))
|
|
return False
|
|
|
|
def port_knock_units(self, sentry_units, port=22,
|
|
timeout=15, expect_success=True):
|
|
"""Open a TCP socket to check for a listening sevice on each
|
|
listed juju unit.
|
|
|
|
:param sentry_units: list of sentry unit pointers
|
|
:param port: TCP port number, default to 22
|
|
:param timeout: Connect timeout, default to 15 seconds
|
|
:expect_success: True by default, set False to invert logic
|
|
:returns: None if successful, Failure message otherwise
|
|
"""
|
|
for unit in sentry_units:
|
|
host = unit.info['public-address']
|
|
connected = self.port_knock_tcp(host, port, timeout)
|
|
if not connected and expect_success:
|
|
return 'Socket connect failed.'
|
|
elif connected and not expect_success:
|
|
return 'Socket connected unexpectedly.'
|
|
|
|
def get_uuid_epoch_stamp(self):
|
|
"""Returns a stamp string based on uuid4 and epoch time. Useful in
|
|
generating test messages which need to be unique-ish."""
|
|
return '[{}-{}]'.format(uuid.uuid4(), time.time())
|
|
|
|
# amulet juju action helpers:
|
|
def run_action(self, unit_sentry, action,
|
|
_check_output=subprocess.check_output):
|
|
"""Run the named action on a given unit sentry.
|
|
|
|
_check_output parameter is used for dependency injection.
|
|
|
|
@return action_id.
|
|
"""
|
|
unit_id = unit_sentry.info["unit_name"]
|
|
command = ["juju", "action", "do", "--format=json", unit_id, action]
|
|
self.log.info("Running command: %s\n" % " ".join(command))
|
|
output = _check_output(command, universal_newlines=True)
|
|
data = json.loads(output)
|
|
action_id = data[u'Action queued with id']
|
|
return action_id
|
|
|
|
def wait_on_action(self, action_id, _check_output=subprocess.check_output):
|
|
"""Wait for a given action, returning if it completed or not.
|
|
|
|
_check_output parameter is used for dependency injection.
|
|
"""
|
|
command = ["juju", "action", "fetch", "--format=json", "--wait=0",
|
|
action_id]
|
|
output = _check_output(command, universal_newlines=True)
|
|
data = json.loads(output)
|
|
return data.get(u"status") == "completed"
|
|
|
|
def status_get(self, unit):
|
|
"""Return the current service status of this unit."""
|
|
raw_status, return_code = unit.run(
|
|
"status-get --format=json --include-data")
|
|
if return_code != 0:
|
|
return ("unknown", "")
|
|
status = json.loads(raw_status)
|
|
return (status["status"], status["message"])
|