diff --git a/hooks/charmhelpers/contrib/charmsupport/nrpe.py b/hooks/charmhelpers/contrib/charmsupport/nrpe.py
index 0fd0a9d..9d961cf 100644
--- a/hooks/charmhelpers/contrib/charmsupport/nrpe.py
+++ b/hooks/charmhelpers/contrib/charmsupport/nrpe.py
@@ -24,6 +24,8 @@ import subprocess
import pwd
import grp
import os
+import glob
+import shutil
import re
import shlex
import yaml
@@ -161,7 +163,7 @@ define service {{
log('Check command not found: {}'.format(parts[0]))
return ''
- def write(self, nagios_context, hostname, nagios_servicegroups=None):
+ def write(self, nagios_context, hostname, nagios_servicegroups):
nrpe_check_file = '/etc/nagios/nrpe.d/{}.cfg'.format(
self.command)
with open(nrpe_check_file, 'w') as nrpe_check_config:
@@ -177,14 +179,11 @@ define service {{
nagios_servicegroups)
def write_service_config(self, nagios_context, hostname,
- nagios_servicegroups=None):
+ nagios_servicegroups):
for f in os.listdir(NRPE.nagios_exportdir):
if re.search('.*{}.cfg'.format(self.command), f):
os.remove(os.path.join(NRPE.nagios_exportdir, f))
- if not nagios_servicegroups:
- nagios_servicegroups = nagios_context
-
templ_vars = {
'nagios_hostname': hostname,
'nagios_servicegroup': nagios_servicegroups,
@@ -211,10 +210,10 @@ class NRPE(object):
super(NRPE, self).__init__()
self.config = config()
self.nagios_context = self.config['nagios_context']
- if 'nagios_servicegroups' in self.config:
+ if 'nagios_servicegroups' in self.config and self.config['nagios_servicegroups']:
self.nagios_servicegroups = self.config['nagios_servicegroups']
else:
- self.nagios_servicegroups = 'juju'
+ self.nagios_servicegroups = self.nagios_context
self.unit_name = local_unit().replace('/', '-')
if hostname:
self.hostname = hostname
@@ -322,3 +321,38 @@ def add_init_service_checks(nrpe, services, unit_name):
check_cmd='check_status_file.py -f '
'/var/lib/nagios/service-check-%s.txt' % svc,
)
+
+
+def copy_nrpe_checks():
+ """
+ Copy the nrpe checks into place
+
+ """
+ NAGIOS_PLUGINS = '/usr/local/lib/nagios/plugins'
+ nrpe_files_dir = os.path.join(os.getenv('CHARM_DIR'), 'hooks',
+ 'charmhelpers', 'contrib', 'openstack',
+ 'files')
+
+ if not os.path.exists(NAGIOS_PLUGINS):
+ os.makedirs(NAGIOS_PLUGINS)
+ for fname in glob.glob(os.path.join(nrpe_files_dir, "check_*")):
+ if os.path.isfile(fname):
+ shutil.copy2(fname,
+ os.path.join(NAGIOS_PLUGINS, os.path.basename(fname)))
+
+
+def add_haproxy_checks(nrpe, unit_name):
+ """
+ Add checks for each service in list
+
+ :param NRPE nrpe: NRPE object to add check to
+ :param str unit_name: Unit name to use in check description
+ """
+ nrpe.add_check(
+ shortname='haproxy_servers',
+ description='Check HAProxy {%s}' % unit_name,
+ check_cmd='check_haproxy.sh')
+ nrpe.add_check(
+ shortname='haproxy_queue',
+ description='Check HAProxy queue depth {%s}' % unit_name,
+ check_cmd='check_haproxy_queue_depth.sh')
diff --git a/hooks/charmhelpers/core/fstab.py b/hooks/charmhelpers/core/fstab.py
index be7de24..3056fba 100644
--- a/hooks/charmhelpers/core/fstab.py
+++ b/hooks/charmhelpers/core/fstab.py
@@ -17,11 +17,11 @@
# You should have received a copy of the GNU Lesser General Public License
# along with charm-helpers. If not, see .
-__author__ = 'Jorge Niedbalski R. '
-
import io
import os
+__author__ = 'Jorge Niedbalski R. '
+
class Fstab(io.FileIO):
"""This class extends file in order to implement a file reader/writer
@@ -77,7 +77,7 @@ class Fstab(io.FileIO):
for line in self.readlines():
line = line.decode('us-ascii')
try:
- if line.strip() and not line.startswith("#"):
+ if line.strip() and not line.strip().startswith("#"):
yield self._hydrate_entry(line)
except ValueError:
pass
@@ -104,7 +104,7 @@ class Fstab(io.FileIO):
found = False
for index, line in enumerate(lines):
- if not line.startswith("#"):
+ if line.strip() and not line.strip().startswith("#"):
if self._hydrate_entry(line) == entry:
found = True
break
diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py
index cf2cbe1..b771c61 100644
--- a/hooks/charmhelpers/core/host.py
+++ b/hooks/charmhelpers/core/host.py
@@ -191,11 +191,11 @@ def mkdir(path, owner='root', group='root', perms=0o555, force=False):
def write_file(path, content, owner='root', group='root', perms=0o444):
- """Create or overwrite a file with the contents of a string"""
+ """Create or overwrite a file with the contents of a byte string."""
log("Writing file {} {}:{} {:o}".format(path, owner, group, perms))
uid = pwd.getpwnam(owner).pw_uid
gid = grp.getgrnam(group).gr_gid
- with open(path, 'w') as target:
+ with open(path, 'wb') as target:
os.fchown(target.fileno(), uid, gid)
os.fchmod(target.fileno(), perms)
target.write(content)
@@ -305,11 +305,11 @@ def restart_on_change(restart_map, stopstart=False):
ceph_client_changed function.
"""
def wrap(f):
- def wrapped_f(*args):
+ def wrapped_f(*args, **kwargs):
checksums = {}
for path in restart_map:
checksums[path] = file_hash(path)
- f(*args)
+ f(*args, **kwargs)
restarts = []
for path in restart_map:
if checksums[path] != file_hash(path):
@@ -361,7 +361,7 @@ def list_nics(nic_type):
ip_output = (line for line in ip_output if line)
for line in ip_output:
if line.split()[1].startswith(int_type):
- matched = re.search('.*: (bond[0-9]+\.[0-9]+)@.*', line)
+ matched = re.search('.*: (' + int_type + r'[0-9]+\.[0-9]+)@.*', line)
if matched:
interface = matched.groups()[0]
else:
diff --git a/hooks/charmhelpers/core/strutils.py b/hooks/charmhelpers/core/strutils.py
new file mode 100644
index 0000000..efc4402
--- /dev/null
+++ b/hooks/charmhelpers/core/strutils.py
@@ -0,0 +1,42 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright 2014-2015 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers. If not, see .
+
+import six
+
+
+def bool_from_string(value):
+ """Interpret string value as boolean.
+
+ Returns True if value translates to True otherwise False.
+ """
+ if isinstance(value, six.string_types):
+ value = six.text_type(value)
+ else:
+ msg = "Unable to interpret non-string value '%s' as boolean" % (value)
+ raise ValueError(msg)
+
+ value = value.strip().lower()
+
+ if value in ['y', 'yes', 'true', 't']:
+ return True
+ elif value in ['n', 'no', 'false', 'f']:
+ return False
+
+ msg = "Unable to interpret string value '%s' as boolean" % (value)
+ raise ValueError(msg)
diff --git a/hooks/charmhelpers/core/sysctl.py b/hooks/charmhelpers/core/sysctl.py
index d642a37..21cc8ab 100644
--- a/hooks/charmhelpers/core/sysctl.py
+++ b/hooks/charmhelpers/core/sysctl.py
@@ -17,8 +17,6 @@
# You should have received a copy of the GNU Lesser General Public License
# along with charm-helpers. If not, see .
-__author__ = 'Jorge Niedbalski R. '
-
import yaml
from subprocess import check_call
@@ -26,25 +24,33 @@ from subprocess import check_call
from charmhelpers.core.hookenv import (
log,
DEBUG,
+ ERROR,
)
+__author__ = 'Jorge Niedbalski R. '
+
def create(sysctl_dict, sysctl_file):
"""Creates a sysctl.conf file from a YAML associative array
- :param sysctl_dict: a dict of sysctl options eg { 'kernel.max_pid': 1337 }
- :type sysctl_dict: dict
+ :param sysctl_dict: a YAML-formatted string of sysctl options eg "{ 'kernel.max_pid': 1337 }"
+ :type sysctl_dict: str
:param sysctl_file: path to the sysctl file to be saved
:type sysctl_file: str or unicode
:returns: None
"""
- sysctl_dict = yaml.load(sysctl_dict)
+ try:
+ sysctl_dict_parsed = yaml.safe_load(sysctl_dict)
+ except yaml.YAMLError:
+ log("Error parsing YAML sysctl_dict: {}".format(sysctl_dict),
+ level=ERROR)
+ return
with open(sysctl_file, "w") as fd:
- for key, value in sysctl_dict.items():
+ for key, value in sysctl_dict_parsed.items():
fd.write("{}={}\n".format(key, value))
- log("Updating sysctl_file: %s values: %s" % (sysctl_file, sysctl_dict),
+ log("Updating sysctl_file: %s values: %s" % (sysctl_file, sysctl_dict_parsed),
level=DEBUG)
check_call(["sysctl", "-p", sysctl_file])
diff --git a/hooks/charmhelpers/core/templating.py b/hooks/charmhelpers/core/templating.py
index 9766909..4531999 100644
--- a/hooks/charmhelpers/core/templating.py
+++ b/hooks/charmhelpers/core/templating.py
@@ -21,7 +21,7 @@ from charmhelpers.core import hookenv
def render(source, target, context, owner='root', group='root',
- perms=0o444, templates_dir=None):
+ perms=0o444, templates_dir=None, encoding='UTF-8'):
"""
Render a template.
@@ -64,5 +64,5 @@ def render(source, target, context, owner='root', group='root',
level=hookenv.ERROR)
raise e
content = template.render(context)
- host.mkdir(os.path.dirname(target), owner, group)
- host.write_file(target, content, owner, group, perms)
+ host.mkdir(os.path.dirname(target), owner, group, perms=0o755)
+ host.write_file(target, content.encode(encoding), owner, group, perms)
diff --git a/hooks/charmhelpers/core/unitdata.py b/hooks/charmhelpers/core/unitdata.py
new file mode 100644
index 0000000..3000134
--- /dev/null
+++ b/hooks/charmhelpers/core/unitdata.py
@@ -0,0 +1,477 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright 2014-2015 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers. If not, see .
+#
+#
+# Authors:
+# Kapil Thangavelu
+#
+"""
+Intro
+-----
+
+A simple way to store state in units. This provides a key value
+storage with support for versioned, transactional operation,
+and can calculate deltas from previous values to simplify unit logic
+when processing changes.
+
+
+Hook Integration
+----------------
+
+There are several extant frameworks for hook execution, including
+
+ - charmhelpers.core.hookenv.Hooks
+ - charmhelpers.core.services.ServiceManager
+
+The storage classes are framework agnostic, one simple integration is
+via the HookData contextmanager. It will record the current hook
+execution environment (including relation data, config data, etc.),
+setup a transaction and allow easy access to the changes from
+previously seen values. One consequence of the integration is the
+reservation of particular keys ('rels', 'unit', 'env', 'config',
+'charm_revisions') for their respective values.
+
+Here's a fully worked integration example using hookenv.Hooks::
+
+ from charmhelper.core import hookenv, unitdata
+
+ hook_data = unitdata.HookData()
+ db = unitdata.kv()
+ hooks = hookenv.Hooks()
+
+ @hooks.hook
+ def config_changed():
+ # Print all changes to configuration from previously seen
+ # values.
+ for changed, (prev, cur) in hook_data.conf.items():
+ print('config changed', changed,
+ 'previous value', prev,
+ 'current value', cur)
+
+ # Get some unit specific bookeeping
+ if not db.get('pkg_key'):
+ key = urllib.urlopen('https://example.com/pkg_key').read()
+ db.set('pkg_key', key)
+
+ # Directly access all charm config as a mapping.
+ conf = db.getrange('config', True)
+
+ # Directly access all relation data as a mapping
+ rels = db.getrange('rels', True)
+
+ if __name__ == '__main__':
+ with hook_data():
+ hook.execute()
+
+
+A more basic integration is via the hook_scope context manager which simply
+manages transaction scope (and records hook name, and timestamp)::
+
+ >>> from unitdata import kv
+ >>> db = kv()
+ >>> with db.hook_scope('install'):
+ ... # do work, in transactional scope.
+ ... db.set('x', 1)
+ >>> db.get('x')
+ 1
+
+
+Usage
+-----
+
+Values are automatically json de/serialized to preserve basic typing
+and complex data struct capabilities (dicts, lists, ints, booleans, etc).
+
+Individual values can be manipulated via get/set::
+
+ >>> kv.set('y', True)
+ >>> kv.get('y')
+ True
+
+ # We can set complex values (dicts, lists) as a single key.
+ >>> kv.set('config', {'a': 1, 'b': True'})
+
+ # Also supports returning dictionaries as a record which
+ # provides attribute access.
+ >>> config = kv.get('config', record=True)
+ >>> config.b
+ True
+
+
+Groups of keys can be manipulated with update/getrange::
+
+ >>> kv.update({'z': 1, 'y': 2}, prefix="gui.")
+ >>> kv.getrange('gui.', strip=True)
+ {'z': 1, 'y': 2}
+
+When updating values, its very helpful to understand which values
+have actually changed and how have they changed. The storage
+provides a delta method to provide for this::
+
+ >>> data = {'debug': True, 'option': 2}
+ >>> delta = kv.delta(data, 'config.')
+ >>> delta.debug.previous
+ None
+ >>> delta.debug.current
+ True
+ >>> delta
+ {'debug': (None, True), 'option': (None, 2)}
+
+Note the delta method does not persist the actual change, it needs to
+be explicitly saved via 'update' method::
+
+ >>> kv.update(data, 'config.')
+
+Values modified in the context of a hook scope retain historical values
+associated to the hookname.
+
+ >>> with db.hook_scope('config-changed'):
+ ... db.set('x', 42)
+ >>> db.gethistory('x')
+ [(1, u'x', 1, u'install', u'2015-01-21T16:49:30.038372'),
+ (2, u'x', 42, u'config-changed', u'2015-01-21T16:49:30.038786')]
+
+"""
+
+import collections
+import contextlib
+import datetime
+import json
+import os
+import pprint
+import sqlite3
+import sys
+
+__author__ = 'Kapil Thangavelu '
+
+
+class Storage(object):
+ """Simple key value database for local unit state within charms.
+
+ Modifications are automatically committed at hook exit. That's
+ currently regardless of exit code.
+
+ To support dicts, lists, integer, floats, and booleans values
+ are automatically json encoded/decoded.
+ """
+ def __init__(self, path=None):
+ self.db_path = path
+ if path is None:
+ self.db_path = os.path.join(
+ os.environ.get('CHARM_DIR', ''), '.unit-state.db')
+ self.conn = sqlite3.connect('%s' % self.db_path)
+ self.cursor = self.conn.cursor()
+ self.revision = None
+ self._closed = False
+ self._init()
+
+ def close(self):
+ if self._closed:
+ return
+ self.flush(False)
+ self.cursor.close()
+ self.conn.close()
+ self._closed = True
+
+ def _scoped_query(self, stmt, params=None):
+ if params is None:
+ params = []
+ return stmt, params
+
+ def get(self, key, default=None, record=False):
+ self.cursor.execute(
+ *self._scoped_query(
+ 'select data from kv where key=?', [key]))
+ result = self.cursor.fetchone()
+ if not result:
+ return default
+ if record:
+ return Record(json.loads(result[0]))
+ return json.loads(result[0])
+
+ def getrange(self, key_prefix, strip=False):
+ stmt = "select key, data from kv where key like '%s%%'" % key_prefix
+ self.cursor.execute(*self._scoped_query(stmt))
+ result = self.cursor.fetchall()
+
+ if not result:
+ return None
+ if not strip:
+ key_prefix = ''
+ return dict([
+ (k[len(key_prefix):], json.loads(v)) for k, v in result])
+
+ def update(self, mapping, prefix=""):
+ for k, v in mapping.items():
+ self.set("%s%s" % (prefix, k), v)
+
+ def unset(self, key):
+ self.cursor.execute('delete from kv where key=?', [key])
+ if self.revision and self.cursor.rowcount:
+ self.cursor.execute(
+ 'insert into kv_revisions values (?, ?, ?)',
+ [key, self.revision, json.dumps('DELETED')])
+
+ def set(self, key, value):
+ serialized = json.dumps(value)
+
+ self.cursor.execute(
+ 'select data from kv where key=?', [key])
+ exists = self.cursor.fetchone()
+
+ # Skip mutations to the same value
+ if exists:
+ if exists[0] == serialized:
+ return value
+
+ if not exists:
+ self.cursor.execute(
+ 'insert into kv (key, data) values (?, ?)',
+ (key, serialized))
+ else:
+ self.cursor.execute('''
+ update kv
+ set data = ?
+ where key = ?''', [serialized, key])
+
+ # Save
+ if not self.revision:
+ return value
+
+ self.cursor.execute(
+ 'select 1 from kv_revisions where key=? and revision=?',
+ [key, self.revision])
+ exists = self.cursor.fetchone()
+
+ if not exists:
+ self.cursor.execute(
+ '''insert into kv_revisions (
+ revision, key, data) values (?, ?, ?)''',
+ (self.revision, key, serialized))
+ else:
+ self.cursor.execute(
+ '''
+ update kv_revisions
+ set data = ?
+ where key = ?
+ and revision = ?''',
+ [serialized, key, self.revision])
+
+ return value
+
+ def delta(self, mapping, prefix):
+ """
+ return a delta containing values that have changed.
+ """
+ previous = self.getrange(prefix, strip=True)
+ if not previous:
+ pk = set()
+ else:
+ pk = set(previous.keys())
+ ck = set(mapping.keys())
+ delta = DeltaSet()
+
+ # added
+ for k in ck.difference(pk):
+ delta[k] = Delta(None, mapping[k])
+
+ # removed
+ for k in pk.difference(ck):
+ delta[k] = Delta(previous[k], None)
+
+ # changed
+ for k in pk.intersection(ck):
+ c = mapping[k]
+ p = previous[k]
+ if c != p:
+ delta[k] = Delta(p, c)
+
+ return delta
+
+ @contextlib.contextmanager
+ def hook_scope(self, name=""):
+ """Scope all future interactions to the current hook execution
+ revision."""
+ assert not self.revision
+ self.cursor.execute(
+ 'insert into hooks (hook, date) values (?, ?)',
+ (name or sys.argv[0],
+ datetime.datetime.utcnow().isoformat()))
+ self.revision = self.cursor.lastrowid
+ try:
+ yield self.revision
+ self.revision = None
+ except:
+ self.flush(False)
+ self.revision = None
+ raise
+ else:
+ self.flush()
+
+ def flush(self, save=True):
+ if save:
+ self.conn.commit()
+ elif self._closed:
+ return
+ else:
+ self.conn.rollback()
+
+ def _init(self):
+ self.cursor.execute('''
+ create table if not exists kv (
+ key text,
+ data text,
+ primary key (key)
+ )''')
+ self.cursor.execute('''
+ create table if not exists kv_revisions (
+ key text,
+ revision integer,
+ data text,
+ primary key (key, revision)
+ )''')
+ self.cursor.execute('''
+ create table if not exists hooks (
+ version integer primary key autoincrement,
+ hook text,
+ date text
+ )''')
+ self.conn.commit()
+
+ def gethistory(self, key, deserialize=False):
+ self.cursor.execute(
+ '''
+ select kv.revision, kv.key, kv.data, h.hook, h.date
+ from kv_revisions kv,
+ hooks h
+ where kv.key=?
+ and kv.revision = h.version
+ ''', [key])
+ if deserialize is False:
+ return self.cursor.fetchall()
+ return map(_parse_history, self.cursor.fetchall())
+
+ def debug(self, fh=sys.stderr):
+ self.cursor.execute('select * from kv')
+ pprint.pprint(self.cursor.fetchall(), stream=fh)
+ self.cursor.execute('select * from kv_revisions')
+ pprint.pprint(self.cursor.fetchall(), stream=fh)
+
+
+def _parse_history(d):
+ return (d[0], d[1], json.loads(d[2]), d[3],
+ datetime.datetime.strptime(d[-1], "%Y-%m-%dT%H:%M:%S.%f"))
+
+
+class HookData(object):
+ """Simple integration for existing hook exec frameworks.
+
+ Records all unit information, and stores deltas for processing
+ by the hook.
+
+ Sample::
+
+ from charmhelper.core import hookenv, unitdata
+
+ changes = unitdata.HookData()
+ db = unitdata.kv()
+ hooks = hookenv.Hooks()
+
+ @hooks.hook
+ def config_changed():
+ # View all changes to configuration
+ for changed, (prev, cur) in changes.conf.items():
+ print('config changed', changed,
+ 'previous value', prev,
+ 'current value', cur)
+
+ # Get some unit specific bookeeping
+ if not db.get('pkg_key'):
+ key = urllib.urlopen('https://example.com/pkg_key').read()
+ db.set('pkg_key', key)
+
+ if __name__ == '__main__':
+ with changes():
+ hook.execute()
+
+ """
+ def __init__(self):
+ self.kv = kv()
+ self.conf = None
+ self.rels = None
+
+ @contextlib.contextmanager
+ def __call__(self):
+ from charmhelpers.core import hookenv
+ hook_name = hookenv.hook_name()
+
+ with self.kv.hook_scope(hook_name):
+ self._record_charm_version(hookenv.charm_dir())
+ delta_config, delta_relation = self._record_hook(hookenv)
+ yield self.kv, delta_config, delta_relation
+
+ def _record_charm_version(self, charm_dir):
+ # Record revisions.. charm revisions are meaningless
+ # to charm authors as they don't control the revision.
+ # so logic dependnent on revision is not particularly
+ # useful, however it is useful for debugging analysis.
+ charm_rev = open(
+ os.path.join(charm_dir, 'revision')).read().strip()
+ charm_rev = charm_rev or '0'
+ revs = self.kv.get('charm_revisions', [])
+ if charm_rev not in revs:
+ revs.append(charm_rev.strip() or '0')
+ self.kv.set('charm_revisions', revs)
+
+ def _record_hook(self, hookenv):
+ data = hookenv.execution_environment()
+ self.conf = conf_delta = self.kv.delta(data['conf'], 'config')
+ self.rels = rels_delta = self.kv.delta(data['rels'], 'rels')
+ self.kv.set('env', data['env'])
+ self.kv.set('unit', data['unit'])
+ self.kv.set('relid', data.get('relid'))
+ return conf_delta, rels_delta
+
+
+class Record(dict):
+
+ __slots__ = ()
+
+ def __getattr__(self, k):
+ if k in self:
+ return self[k]
+ raise AttributeError(k)
+
+
+class DeltaSet(Record):
+
+ __slots__ = ()
+
+
+Delta = collections.namedtuple('Delta', ['previous', 'current'])
+
+
+_KV = None
+
+
+def kv():
+ global _KV
+ if _KV is None:
+ _KV = Storage()
+ return _KV
diff --git a/hooks/charmhelpers/fetch/archiveurl.py b/hooks/charmhelpers/fetch/archiveurl.py
index d25a0dd..8dfce50 100644
--- a/hooks/charmhelpers/fetch/archiveurl.py
+++ b/hooks/charmhelpers/fetch/archiveurl.py
@@ -18,6 +18,16 @@ import os
import hashlib
import re
+from charmhelpers.fetch import (
+ BaseFetchHandler,
+ UnhandledSource
+)
+from charmhelpers.payload.archive import (
+ get_archive_handler,
+ extract,
+)
+from charmhelpers.core.host import mkdir, check_hash
+
import six
if six.PY3:
from urllib.request import (
@@ -35,16 +45,6 @@ else:
)
from urlparse import urlparse, urlunparse, parse_qs
-from charmhelpers.fetch import (
- BaseFetchHandler,
- UnhandledSource
-)
-from charmhelpers.payload.archive import (
- get_archive_handler,
- extract,
-)
-from charmhelpers.core.host import mkdir, check_hash
-
def splituser(host):
'''urllib.splituser(), but six's support of this seems broken'''
diff --git a/hooks/charmhelpers/fetch/giturl.py b/hooks/charmhelpers/fetch/giturl.py
index 5376786..93aae87 100644
--- a/hooks/charmhelpers/fetch/giturl.py
+++ b/hooks/charmhelpers/fetch/giturl.py
@@ -32,7 +32,7 @@ except ImportError:
apt_install("python-git")
from git import Repo
-from git.exc import GitCommandError
+from git.exc import GitCommandError # noqa E402
class GitUrlFetchHandler(BaseFetchHandler):
diff --git a/tests/charmhelpers/contrib/amulet/utils.py b/tests/charmhelpers/contrib/amulet/utils.py
index 3464b87..65219d3 100644
--- a/tests/charmhelpers/contrib/amulet/utils.py
+++ b/tests/charmhelpers/contrib/amulet/utils.py
@@ -169,8 +169,13 @@ class AmuletUtils(object):
cmd = 'pgrep -o -f {}'.format(service)
else:
cmd = 'pgrep -o {}'.format(service)
- proc_dir = '/proc/{}'.format(sentry_unit.run(cmd)[0].strip())
- return self._get_dir_mtime(sentry_unit, proc_dir)
+ cmd = cmd + ' | grep -v pgrep || exit 0'
+ cmd_out = sentry_unit.run(cmd)
+ self.log.debug('CMDout: ' + str(cmd_out))
+ if cmd_out[0]:
+ self.log.debug('Pid for %s %s' % (service, str(cmd_out[0])))
+ proc_dir = '/proc/{}'.format(cmd_out[0].strip())
+ return self._get_dir_mtime(sentry_unit, proc_dir)
def service_restarted(self, sentry_unit, service, filename,
pgrep_full=False, sleep_time=20):
@@ -187,6 +192,121 @@ class AmuletUtils(object):
else:
return False
+ def service_restarted_since(self, sentry_unit, mtime, service,
+ pgrep_full=False, sleep_time=20,
+ retry_count=2):
+ """Check if service was been started after a given time.
+
+ Args:
+ sentry_unit (sentry): The sentry unit to check for the service on
+ mtime (float): The epoch time to check against
+ service (string): service name to look for in process table
+ pgrep_full (boolean): Use full command line search mode with pgrep
+ sleep_time (int): Seconds to sleep before looking for process
+ retry_count (int): If service is not found, how many times to retry
+
+ Returns:
+ bool: True if service found and its start time it newer than mtime,
+ False if service is older than mtime or if service was
+ not found.
+ """
+ self.log.debug('Checking %s restarted since %s' % (service, mtime))
+ time.sleep(sleep_time)
+ proc_start_time = self._get_proc_start_time(sentry_unit, service,
+ pgrep_full)
+ while retry_count > 0 and not proc_start_time:
+ self.log.debug('No pid file found for service %s, will retry %i '
+ 'more times' % (service, retry_count))
+ time.sleep(30)
+ proc_start_time = self._get_proc_start_time(sentry_unit, service,
+ pgrep_full)
+ retry_count = retry_count - 1
+
+ if not proc_start_time:
+ self.log.warn('No proc start time found, assuming service did '
+ 'not start')
+ return False
+ if proc_start_time >= mtime:
+ self.log.debug('proc start time is newer than provided mtime'
+ '(%s >= %s)' % (proc_start_time, mtime))
+ return True
+ else:
+ self.log.warn('proc start time (%s) is older than provided mtime '
+ '(%s), service did not restart' % (proc_start_time,
+ mtime))
+ return False
+
+ def config_updated_since(self, sentry_unit, filename, mtime,
+ sleep_time=20):
+ """Check if file was modified after a given time.
+
+ Args:
+ sentry_unit (sentry): The sentry unit to check the file mtime on
+ filename (string): The file to check mtime of
+ mtime (float): The epoch time to check against
+ sleep_time (int): Seconds to sleep before looking for process
+
+ Returns:
+ bool: True if file was modified more recently than mtime, False if
+ file was modified before mtime,
+ """
+ self.log.debug('Checking %s updated since %s' % (filename, mtime))
+ time.sleep(sleep_time)
+ file_mtime = self._get_file_mtime(sentry_unit, filename)
+ if file_mtime >= mtime:
+ self.log.debug('File mtime is newer than provided mtime '
+ '(%s >= %s)' % (file_mtime, mtime))
+ return True
+ else:
+ self.log.warn('File mtime %s is older than provided mtime %s'
+ % (file_mtime, mtime))
+ return False
+
+ def validate_service_config_changed(self, sentry_unit, mtime, service,
+ filename, pgrep_full=False,
+ sleep_time=20, retry_count=2):
+ """Check service and file were updated after mtime
+
+ Args:
+ sentry_unit (sentry): The sentry unit to check for the service on
+ mtime (float): The epoch time to check against
+ service (string): service name to look for in process table
+ filename (string): The file to check mtime of
+ pgrep_full (boolean): Use full command line search mode with pgrep
+ sleep_time (int): Seconds to sleep before looking for process
+ retry_count (int): If service is not found, how many times to retry
+
+ Typical Usage:
+ u = OpenStackAmuletUtils(ERROR)
+ ...
+ mtime = u.get_sentry_time(self.cinder_sentry)
+ self.d.configure('cinder', {'verbose': 'True', 'debug': 'True'})
+ if not u.validate_service_config_changed(self.cinder_sentry,
+ mtime,
+ 'cinder-api',
+ '/etc/cinder/cinder.conf')
+ amulet.raise_status(amulet.FAIL, msg='update failed')
+ Returns:
+ bool: True if both service and file where updated/restarted after
+ mtime, False if service is older than mtime or if service was
+ not found or if filename was modified before mtime.
+ """
+ self.log.debug('Checking %s restarted since %s' % (service, mtime))
+ time.sleep(sleep_time)
+ service_restart = self.service_restarted_since(sentry_unit, mtime,
+ service,
+ pgrep_full=pgrep_full,
+ sleep_time=0,
+ retry_count=retry_count)
+ config_update = self.config_updated_since(sentry_unit, filename, mtime,
+ sleep_time=0)
+ return service_restart and config_update
+
+ def get_sentry_time(self, sentry_unit):
+ """Return current epoch time on a sentry"""
+ cmd = "date +'%s'"
+ return float(sentry_unit.run(cmd)[0])
+
def relation_error(self, name, data):
return 'unexpected relation data in {} - {}'.format(name, data)
diff --git a/tests/charmhelpers/contrib/openstack/amulet/deployment.py b/tests/charmhelpers/contrib/openstack/amulet/deployment.py
index c50d3ec..0cfeaa4 100644
--- a/tests/charmhelpers/contrib/openstack/amulet/deployment.py
+++ b/tests/charmhelpers/contrib/openstack/amulet/deployment.py
@@ -71,16 +71,19 @@ class OpenStackAmuletDeployment(AmuletDeployment):
services.append(this_service)
use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph',
'ceph-osd', 'ceph-radosgw']
+ # Openstack subordinate charms do not expose an origin option as that
+ # is controlled by the principle
+ ignore = ['neutron-openvswitch']
if self.openstack:
for svc in services:
- if svc['name'] not in use_source:
+ if svc['name'] not in use_source + ignore:
config = {'openstack-origin': self.openstack}
self.d.configure(svc['name'], config)
if self.source:
for svc in services:
- if svc['name'] in use_source:
+ if svc['name'] in use_source and svc['name'] not in ignore:
config = {'source': self.source}
self.d.configure(svc['name'], config)