diff --git a/hooks/charmhelpers/contrib/charmsupport/nrpe.py b/hooks/charmhelpers/contrib/charmsupport/nrpe.py
index 0fd0a9d8..8229f6b5 100644
--- a/hooks/charmhelpers/contrib/charmsupport/nrpe.py
+++ b/hooks/charmhelpers/contrib/charmsupport/nrpe.py
@@ -24,6 +24,8 @@ import subprocess
import pwd
import grp
import os
+import glob
+import shutil
import re
import shlex
import yaml
@@ -161,7 +163,7 @@ define service {{
log('Check command not found: {}'.format(parts[0]))
return ''
- def write(self, nagios_context, hostname, nagios_servicegroups=None):
+ def write(self, nagios_context, hostname, nagios_servicegroups):
nrpe_check_file = '/etc/nagios/nrpe.d/{}.cfg'.format(
self.command)
with open(nrpe_check_file, 'w') as nrpe_check_config:
@@ -177,14 +179,11 @@ define service {{
nagios_servicegroups)
def write_service_config(self, nagios_context, hostname,
- nagios_servicegroups=None):
+ nagios_servicegroups):
for f in os.listdir(NRPE.nagios_exportdir):
if re.search('.*{}.cfg'.format(self.command), f):
os.remove(os.path.join(NRPE.nagios_exportdir, f))
- if not nagios_servicegroups:
- nagios_servicegroups = nagios_context
-
templ_vars = {
'nagios_hostname': hostname,
'nagios_servicegroup': nagios_servicegroups,
@@ -214,7 +213,7 @@ class NRPE(object):
if 'nagios_servicegroups' in self.config:
self.nagios_servicegroups = self.config['nagios_servicegroups']
else:
- self.nagios_servicegroups = 'juju'
+ self.nagios_servicegroups = self.nagios_context
self.unit_name = local_unit().replace('/', '-')
if hostname:
self.hostname = hostname
@@ -322,3 +321,38 @@ def add_init_service_checks(nrpe, services, unit_name):
check_cmd='check_status_file.py -f '
'/var/lib/nagios/service-check-%s.txt' % svc,
)
+
+
+def copy_nrpe_checks():
+ """
+ Copy the nrpe checks into place
+
+ """
+ NAGIOS_PLUGINS = '/usr/local/lib/nagios/plugins'
+ nrpe_files_dir = os.path.join(os.getenv('CHARM_DIR'), 'hooks',
+ 'charmhelpers', 'contrib', 'openstack',
+ 'files')
+
+ if not os.path.exists(NAGIOS_PLUGINS):
+ os.makedirs(NAGIOS_PLUGINS)
+ for fname in glob.glob(os.path.join(nrpe_files_dir, "check_*")):
+ if os.path.isfile(fname):
+ shutil.copy2(fname,
+ os.path.join(NAGIOS_PLUGINS, os.path.basename(fname)))
+
+
+def add_haproxy_checks(nrpe, unit_name):
+ """
+ Add checks for each service in list
+
+ :param NRPE nrpe: NRPE object to add check to
+ :param str unit_name: Unit name to use in check description
+ """
+ nrpe.add_check(
+ shortname='haproxy_servers',
+ description='Check HAProxy {%s}' % unit_name,
+ check_cmd='check_haproxy.sh')
+ nrpe.add_check(
+ shortname='haproxy_queue',
+ description='Check HAProxy queue depth {%s}' % unit_name,
+ check_cmd='check_haproxy_queue_depth.sh')
diff --git a/hooks/charmhelpers/contrib/hahelpers/cluster.py b/hooks/charmhelpers/contrib/hahelpers/cluster.py
index 9a2588b6..9333efc3 100644
--- a/hooks/charmhelpers/contrib/hahelpers/cluster.py
+++ b/hooks/charmhelpers/contrib/hahelpers/cluster.py
@@ -48,6 +48,9 @@ from charmhelpers.core.hookenv import (
from charmhelpers.core.decorators import (
retry_on_exception,
)
+from charmhelpers.core.strutils import (
+ bool_from_string,
+)
class HAIncompleteConfig(Exception):
@@ -164,7 +167,8 @@ def https():
.
returns: boolean
'''
- if config_get('use-https') == "yes":
+ use_https = config_get('use-https')
+ if use_https and bool_from_string(use_https):
return True
if config_get('ssl_cert') and config_get('ssl_key'):
return True
diff --git a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py
index c50d3ec6..0cfeaa4c 100644
--- a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py
+++ b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py
@@ -71,16 +71,19 @@ class OpenStackAmuletDeployment(AmuletDeployment):
services.append(this_service)
use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph',
'ceph-osd', 'ceph-radosgw']
+ # Openstack subordinate charms do not expose an origin option as that
+ # is controlled by the principle
+ ignore = ['neutron-openvswitch']
if self.openstack:
for svc in services:
- if svc['name'] not in use_source:
+ if svc['name'] not in use_source + ignore:
config = {'openstack-origin': self.openstack}
self.d.configure(svc['name'], config)
if self.source:
for svc in services:
- if svc['name'] in use_source:
+ if svc['name'] in use_source and svc['name'] not in ignore:
config = {'source': self.source}
self.d.configure(svc['name'], config)
diff --git a/hooks/charmhelpers/contrib/openstack/files/__init__.py b/hooks/charmhelpers/contrib/openstack/files/__init__.py
new file mode 100644
index 00000000..75876796
--- /dev/null
+++ b/hooks/charmhelpers/contrib/openstack/files/__init__.py
@@ -0,0 +1,18 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers. If not, see .
+
+# dummy __init__.py to fool syncer into thinking this is a syncable python
+# module
diff --git a/hooks/charmhelpers/contrib/openstack/files/check_haproxy.sh b/hooks/charmhelpers/contrib/openstack/files/check_haproxy.sh
new file mode 100755
index 00000000..eb8527f5
--- /dev/null
+++ b/hooks/charmhelpers/contrib/openstack/files/check_haproxy.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+#--------------------------------------------
+# This file is managed by Juju
+#--------------------------------------------
+#
+# Copyright 2009,2012 Canonical Ltd.
+# Author: Tom Haddon
+
+CRITICAL=0
+NOTACTIVE=''
+LOGFILE=/var/log/nagios/check_haproxy.log
+AUTH=$(grep -r "stats auth" /etc/haproxy | head -1 | awk '{print $4}')
+
+for appserver in $(grep ' server' /etc/haproxy/haproxy.cfg | awk '{print $2'});
+do
+ output=$(/usr/lib/nagios/plugins/check_http -a ${AUTH} -I 127.0.0.1 -p 8888 --regex="class=\"(active|backup)(2|3).*${appserver}" -e ' 200 OK')
+ if [ $? != 0 ]; then
+ date >> $LOGFILE
+ echo $output >> $LOGFILE
+ /usr/lib/nagios/plugins/check_http -a ${AUTH} -I 127.0.0.1 -p 8888 -v | grep $appserver >> $LOGFILE 2>&1
+ CRITICAL=1
+ NOTACTIVE="${NOTACTIVE} $appserver"
+ fi
+done
+
+if [ $CRITICAL = 1 ]; then
+ echo "CRITICAL:${NOTACTIVE}"
+ exit 2
+fi
+
+echo "OK: All haproxy instances looking good"
+exit 0
diff --git a/hooks/charmhelpers/contrib/openstack/files/check_haproxy_queue_depth.sh b/hooks/charmhelpers/contrib/openstack/files/check_haproxy_queue_depth.sh
new file mode 100755
index 00000000..3ebb5329
--- /dev/null
+++ b/hooks/charmhelpers/contrib/openstack/files/check_haproxy_queue_depth.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+#--------------------------------------------
+# This file is managed by Juju
+#--------------------------------------------
+#
+# Copyright 2009,2012 Canonical Ltd.
+# Author: Tom Haddon
+
+# These should be config options at some stage
+CURRQthrsh=0
+MAXQthrsh=100
+
+AUTH=$(grep -r "stats auth" /etc/haproxy | head -1 | awk '{print $4}')
+
+HAPROXYSTATS=$(/usr/lib/nagios/plugins/check_http -a ${AUTH} -I 127.0.0.1 -p 8888 -u '/;csv' -v)
+
+for BACKEND in $(echo $HAPROXYSTATS| xargs -n1 | grep BACKEND | awk -F , '{print $1}')
+do
+ CURRQ=$(echo "$HAPROXYSTATS" | grep $BACKEND | grep BACKEND | cut -d , -f 3)
+ MAXQ=$(echo "$HAPROXYSTATS" | grep $BACKEND | grep BACKEND | cut -d , -f 4)
+
+ if [[ $CURRQ -gt $CURRQthrsh || $MAXQ -gt $MAXQthrsh ]] ; then
+ echo "CRITICAL: queue depth for $BACKEND - CURRENT:$CURRQ MAX:$MAXQ"
+ exit 2
+ fi
+done
+
+echo "OK: All haproxy queue depths looking good"
+exit 0
+
diff --git a/hooks/charmhelpers/contrib/openstack/ip.py b/hooks/charmhelpers/contrib/openstack/ip.py
index 9eabed73..29bbddcb 100644
--- a/hooks/charmhelpers/contrib/openstack/ip.py
+++ b/hooks/charmhelpers/contrib/openstack/ip.py
@@ -26,6 +26,8 @@ from charmhelpers.contrib.network.ip import (
)
from charmhelpers.contrib.hahelpers.cluster import is_clustered
+from functools import partial
+
PUBLIC = 'public'
INTERNAL = 'int'
ADMIN = 'admin'
@@ -107,3 +109,38 @@ def resolve_address(endpoint_type=PUBLIC):
"clustered=%s)" % (net_type, clustered))
return resolved_address
+
+
+def endpoint_url(configs, url_template, port, endpoint_type=PUBLIC,
+ override=None):
+ """Returns the correct endpoint URL to advertise to Keystone.
+
+ This method provides the correct endpoint URL which should be advertised to
+ the keystone charm for endpoint creation. This method allows for the url to
+ be overridden to force a keystone endpoint to have specific URL for any of
+ the defined scopes (admin, internal, public).
+
+ :param configs: OSTemplateRenderer config templating object to inspect
+ for a complete https context.
+ :param url_template: str format string for creating the url template. Only
+ two values will be passed - the scheme+hostname
+ returned by the canonical_url and the port.
+ :param endpoint_type: str endpoint type to resolve.
+ :param override: str the name of the config option which overrides the
+ endpoint URL defined by the charm itself. None will
+ disable any overrides (default).
+ """
+ if override:
+ # Return any user-defined overrides for the keystone endpoint URL.
+ user_value = config(override)
+ if user_value:
+ return user_value.strip()
+
+ return url_template % (canonical_url(configs, endpoint_type), port)
+
+
+public_endpoint = partial(endpoint_url, endpoint_type=PUBLIC)
+
+internal_endpoint = partial(endpoint_url, endpoint_type=INTERNAL)
+
+admin_endpoint = partial(endpoint_url, endpoint_type=ADMIN)
diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py
index 26259a03..af2b3596 100644
--- a/hooks/charmhelpers/contrib/openstack/utils.py
+++ b/hooks/charmhelpers/contrib/openstack/utils.py
@@ -103,6 +103,7 @@ SWIFT_CODENAMES = OrderedDict([
('2.1.0', 'juno'),
('2.2.0', 'juno'),
('2.2.1', 'kilo'),
+ ('2.2.2', 'kilo'),
])
DEFAULT_LOOPBACK_SIZE = '5G'
diff --git a/hooks/charmhelpers/contrib/python/packages.py b/hooks/charmhelpers/contrib/python/packages.py
index d848a120..8659516b 100644
--- a/hooks/charmhelpers/contrib/python/packages.py
+++ b/hooks/charmhelpers/contrib/python/packages.py
@@ -17,8 +17,6 @@
# You should have received a copy of the GNU Lesser General Public License
# along with charm-helpers. If not, see .
-__author__ = "Jorge Niedbalski "
-
from charmhelpers.fetch import apt_install, apt_update
from charmhelpers.core.hookenv import log
@@ -29,6 +27,8 @@ except ImportError:
apt_install('python-pip')
from pip import main as pip_execute
+__author__ = "Jorge Niedbalski "
+
def parse_options(given, available):
"""Given a set of options, check if available"""
diff --git a/hooks/charmhelpers/core/fstab.py b/hooks/charmhelpers/core/fstab.py
index be7de248..3056fbac 100644
--- a/hooks/charmhelpers/core/fstab.py
+++ b/hooks/charmhelpers/core/fstab.py
@@ -17,11 +17,11 @@
# You should have received a copy of the GNU Lesser General Public License
# along with charm-helpers. If not, see .
-__author__ = 'Jorge Niedbalski R. '
-
import io
import os
+__author__ = 'Jorge Niedbalski R. '
+
class Fstab(io.FileIO):
"""This class extends file in order to implement a file reader/writer
@@ -77,7 +77,7 @@ class Fstab(io.FileIO):
for line in self.readlines():
line = line.decode('us-ascii')
try:
- if line.strip() and not line.startswith("#"):
+ if line.strip() and not line.strip().startswith("#"):
yield self._hydrate_entry(line)
except ValueError:
pass
@@ -104,7 +104,7 @@ class Fstab(io.FileIO):
found = False
for index, line in enumerate(lines):
- if not line.startswith("#"):
+ if line.strip() and not line.strip().startswith("#"):
if self._hydrate_entry(line) == entry:
found = True
break
diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py
index cf2cbe14..b771c611 100644
--- a/hooks/charmhelpers/core/host.py
+++ b/hooks/charmhelpers/core/host.py
@@ -191,11 +191,11 @@ def mkdir(path, owner='root', group='root', perms=0o555, force=False):
def write_file(path, content, owner='root', group='root', perms=0o444):
- """Create or overwrite a file with the contents of a string"""
+ """Create or overwrite a file with the contents of a byte string."""
log("Writing file {} {}:{} {:o}".format(path, owner, group, perms))
uid = pwd.getpwnam(owner).pw_uid
gid = grp.getgrnam(group).gr_gid
- with open(path, 'w') as target:
+ with open(path, 'wb') as target:
os.fchown(target.fileno(), uid, gid)
os.fchmod(target.fileno(), perms)
target.write(content)
@@ -305,11 +305,11 @@ def restart_on_change(restart_map, stopstart=False):
ceph_client_changed function.
"""
def wrap(f):
- def wrapped_f(*args):
+ def wrapped_f(*args, **kwargs):
checksums = {}
for path in restart_map:
checksums[path] = file_hash(path)
- f(*args)
+ f(*args, **kwargs)
restarts = []
for path in restart_map:
if checksums[path] != file_hash(path):
@@ -361,7 +361,7 @@ def list_nics(nic_type):
ip_output = (line for line in ip_output if line)
for line in ip_output:
if line.split()[1].startswith(int_type):
- matched = re.search('.*: (bond[0-9]+\.[0-9]+)@.*', line)
+ matched = re.search('.*: (' + int_type + r'[0-9]+\.[0-9]+)@.*', line)
if matched:
interface = matched.groups()[0]
else:
diff --git a/hooks/charmhelpers/core/strutils.py b/hooks/charmhelpers/core/strutils.py
new file mode 100644
index 00000000..efc4402e
--- /dev/null
+++ b/hooks/charmhelpers/core/strutils.py
@@ -0,0 +1,42 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright 2014-2015 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers. If not, see .
+
+import six
+
+
+def bool_from_string(value):
+ """Interpret string value as boolean.
+
+ Returns True if value translates to True otherwise False.
+ """
+ if isinstance(value, six.string_types):
+ value = six.text_type(value)
+ else:
+ msg = "Unable to interpret non-string value '%s' as boolean" % (value)
+ raise ValueError(msg)
+
+ value = value.strip().lower()
+
+ if value in ['y', 'yes', 'true', 't']:
+ return True
+ elif value in ['n', 'no', 'false', 'f']:
+ return False
+
+ msg = "Unable to interpret string value '%s' as boolean" % (value)
+ raise ValueError(msg)
diff --git a/hooks/charmhelpers/core/sysctl.py b/hooks/charmhelpers/core/sysctl.py
index d642a371..21cc8ab2 100644
--- a/hooks/charmhelpers/core/sysctl.py
+++ b/hooks/charmhelpers/core/sysctl.py
@@ -17,8 +17,6 @@
# You should have received a copy of the GNU Lesser General Public License
# along with charm-helpers. If not, see .
-__author__ = 'Jorge Niedbalski R. '
-
import yaml
from subprocess import check_call
@@ -26,25 +24,33 @@ from subprocess import check_call
from charmhelpers.core.hookenv import (
log,
DEBUG,
+ ERROR,
)
+__author__ = 'Jorge Niedbalski R. '
+
def create(sysctl_dict, sysctl_file):
"""Creates a sysctl.conf file from a YAML associative array
- :param sysctl_dict: a dict of sysctl options eg { 'kernel.max_pid': 1337 }
- :type sysctl_dict: dict
+ :param sysctl_dict: a YAML-formatted string of sysctl options eg "{ 'kernel.max_pid': 1337 }"
+ :type sysctl_dict: str
:param sysctl_file: path to the sysctl file to be saved
:type sysctl_file: str or unicode
:returns: None
"""
- sysctl_dict = yaml.load(sysctl_dict)
+ try:
+ sysctl_dict_parsed = yaml.safe_load(sysctl_dict)
+ except yaml.YAMLError:
+ log("Error parsing YAML sysctl_dict: {}".format(sysctl_dict),
+ level=ERROR)
+ return
with open(sysctl_file, "w") as fd:
- for key, value in sysctl_dict.items():
+ for key, value in sysctl_dict_parsed.items():
fd.write("{}={}\n".format(key, value))
- log("Updating sysctl_file: %s values: %s" % (sysctl_file, sysctl_dict),
+ log("Updating sysctl_file: %s values: %s" % (sysctl_file, sysctl_dict_parsed),
level=DEBUG)
check_call(["sysctl", "-p", sysctl_file])
diff --git a/hooks/charmhelpers/core/templating.py b/hooks/charmhelpers/core/templating.py
index 97669092..45319998 100644
--- a/hooks/charmhelpers/core/templating.py
+++ b/hooks/charmhelpers/core/templating.py
@@ -21,7 +21,7 @@ from charmhelpers.core import hookenv
def render(source, target, context, owner='root', group='root',
- perms=0o444, templates_dir=None):
+ perms=0o444, templates_dir=None, encoding='UTF-8'):
"""
Render a template.
@@ -64,5 +64,5 @@ def render(source, target, context, owner='root', group='root',
level=hookenv.ERROR)
raise e
content = template.render(context)
- host.mkdir(os.path.dirname(target), owner, group)
- host.write_file(target, content, owner, group, perms)
+ host.mkdir(os.path.dirname(target), owner, group, perms=0o755)
+ host.write_file(target, content.encode(encoding), owner, group, perms)
diff --git a/hooks/charmhelpers/core/unitdata.py b/hooks/charmhelpers/core/unitdata.py
new file mode 100644
index 00000000..3000134a
--- /dev/null
+++ b/hooks/charmhelpers/core/unitdata.py
@@ -0,0 +1,477 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright 2014-2015 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers. If not, see .
+#
+#
+# Authors:
+# Kapil Thangavelu
+#
+"""
+Intro
+-----
+
+A simple way to store state in units. This provides a key value
+storage with support for versioned, transactional operation,
+and can calculate deltas from previous values to simplify unit logic
+when processing changes.
+
+
+Hook Integration
+----------------
+
+There are several extant frameworks for hook execution, including
+
+ - charmhelpers.core.hookenv.Hooks
+ - charmhelpers.core.services.ServiceManager
+
+The storage classes are framework agnostic, one simple integration is
+via the HookData contextmanager. It will record the current hook
+execution environment (including relation data, config data, etc.),
+setup a transaction and allow easy access to the changes from
+previously seen values. One consequence of the integration is the
+reservation of particular keys ('rels', 'unit', 'env', 'config',
+'charm_revisions') for their respective values.
+
+Here's a fully worked integration example using hookenv.Hooks::
+
+ from charmhelper.core import hookenv, unitdata
+
+ hook_data = unitdata.HookData()
+ db = unitdata.kv()
+ hooks = hookenv.Hooks()
+
+ @hooks.hook
+ def config_changed():
+ # Print all changes to configuration from previously seen
+ # values.
+ for changed, (prev, cur) in hook_data.conf.items():
+ print('config changed', changed,
+ 'previous value', prev,
+ 'current value', cur)
+
+ # Get some unit specific bookeeping
+ if not db.get('pkg_key'):
+ key = urllib.urlopen('https://example.com/pkg_key').read()
+ db.set('pkg_key', key)
+
+ # Directly access all charm config as a mapping.
+ conf = db.getrange('config', True)
+
+ # Directly access all relation data as a mapping
+ rels = db.getrange('rels', True)
+
+ if __name__ == '__main__':
+ with hook_data():
+ hook.execute()
+
+
+A more basic integration is via the hook_scope context manager which simply
+manages transaction scope (and records hook name, and timestamp)::
+
+ >>> from unitdata import kv
+ >>> db = kv()
+ >>> with db.hook_scope('install'):
+ ... # do work, in transactional scope.
+ ... db.set('x', 1)
+ >>> db.get('x')
+ 1
+
+
+Usage
+-----
+
+Values are automatically json de/serialized to preserve basic typing
+and complex data struct capabilities (dicts, lists, ints, booleans, etc).
+
+Individual values can be manipulated via get/set::
+
+ >>> kv.set('y', True)
+ >>> kv.get('y')
+ True
+
+ # We can set complex values (dicts, lists) as a single key.
+ >>> kv.set('config', {'a': 1, 'b': True'})
+
+ # Also supports returning dictionaries as a record which
+ # provides attribute access.
+ >>> config = kv.get('config', record=True)
+ >>> config.b
+ True
+
+
+Groups of keys can be manipulated with update/getrange::
+
+ >>> kv.update({'z': 1, 'y': 2}, prefix="gui.")
+ >>> kv.getrange('gui.', strip=True)
+ {'z': 1, 'y': 2}
+
+When updating values, its very helpful to understand which values
+have actually changed and how have they changed. The storage
+provides a delta method to provide for this::
+
+ >>> data = {'debug': True, 'option': 2}
+ >>> delta = kv.delta(data, 'config.')
+ >>> delta.debug.previous
+ None
+ >>> delta.debug.current
+ True
+ >>> delta
+ {'debug': (None, True), 'option': (None, 2)}
+
+Note the delta method does not persist the actual change, it needs to
+be explicitly saved via 'update' method::
+
+ >>> kv.update(data, 'config.')
+
+Values modified in the context of a hook scope retain historical values
+associated to the hookname.
+
+ >>> with db.hook_scope('config-changed'):
+ ... db.set('x', 42)
+ >>> db.gethistory('x')
+ [(1, u'x', 1, u'install', u'2015-01-21T16:49:30.038372'),
+ (2, u'x', 42, u'config-changed', u'2015-01-21T16:49:30.038786')]
+
+"""
+
+import collections
+import contextlib
+import datetime
+import json
+import os
+import pprint
+import sqlite3
+import sys
+
+__author__ = 'Kapil Thangavelu '
+
+
+class Storage(object):
+ """Simple key value database for local unit state within charms.
+
+ Modifications are automatically committed at hook exit. That's
+ currently regardless of exit code.
+
+ To support dicts, lists, integer, floats, and booleans values
+ are automatically json encoded/decoded.
+ """
+ def __init__(self, path=None):
+ self.db_path = path
+ if path is None:
+ self.db_path = os.path.join(
+ os.environ.get('CHARM_DIR', ''), '.unit-state.db')
+ self.conn = sqlite3.connect('%s' % self.db_path)
+ self.cursor = self.conn.cursor()
+ self.revision = None
+ self._closed = False
+ self._init()
+
+ def close(self):
+ if self._closed:
+ return
+ self.flush(False)
+ self.cursor.close()
+ self.conn.close()
+ self._closed = True
+
+ def _scoped_query(self, stmt, params=None):
+ if params is None:
+ params = []
+ return stmt, params
+
+ def get(self, key, default=None, record=False):
+ self.cursor.execute(
+ *self._scoped_query(
+ 'select data from kv where key=?', [key]))
+ result = self.cursor.fetchone()
+ if not result:
+ return default
+ if record:
+ return Record(json.loads(result[0]))
+ return json.loads(result[0])
+
+ def getrange(self, key_prefix, strip=False):
+ stmt = "select key, data from kv where key like '%s%%'" % key_prefix
+ self.cursor.execute(*self._scoped_query(stmt))
+ result = self.cursor.fetchall()
+
+ if not result:
+ return None
+ if not strip:
+ key_prefix = ''
+ return dict([
+ (k[len(key_prefix):], json.loads(v)) for k, v in result])
+
+ def update(self, mapping, prefix=""):
+ for k, v in mapping.items():
+ self.set("%s%s" % (prefix, k), v)
+
+ def unset(self, key):
+ self.cursor.execute('delete from kv where key=?', [key])
+ if self.revision and self.cursor.rowcount:
+ self.cursor.execute(
+ 'insert into kv_revisions values (?, ?, ?)',
+ [key, self.revision, json.dumps('DELETED')])
+
+ def set(self, key, value):
+ serialized = json.dumps(value)
+
+ self.cursor.execute(
+ 'select data from kv where key=?', [key])
+ exists = self.cursor.fetchone()
+
+ # Skip mutations to the same value
+ if exists:
+ if exists[0] == serialized:
+ return value
+
+ if not exists:
+ self.cursor.execute(
+ 'insert into kv (key, data) values (?, ?)',
+ (key, serialized))
+ else:
+ self.cursor.execute('''
+ update kv
+ set data = ?
+ where key = ?''', [serialized, key])
+
+ # Save
+ if not self.revision:
+ return value
+
+ self.cursor.execute(
+ 'select 1 from kv_revisions where key=? and revision=?',
+ [key, self.revision])
+ exists = self.cursor.fetchone()
+
+ if not exists:
+ self.cursor.execute(
+ '''insert into kv_revisions (
+ revision, key, data) values (?, ?, ?)''',
+ (self.revision, key, serialized))
+ else:
+ self.cursor.execute(
+ '''
+ update kv_revisions
+ set data = ?
+ where key = ?
+ and revision = ?''',
+ [serialized, key, self.revision])
+
+ return value
+
+ def delta(self, mapping, prefix):
+ """
+ return a delta containing values that have changed.
+ """
+ previous = self.getrange(prefix, strip=True)
+ if not previous:
+ pk = set()
+ else:
+ pk = set(previous.keys())
+ ck = set(mapping.keys())
+ delta = DeltaSet()
+
+ # added
+ for k in ck.difference(pk):
+ delta[k] = Delta(None, mapping[k])
+
+ # removed
+ for k in pk.difference(ck):
+ delta[k] = Delta(previous[k], None)
+
+ # changed
+ for k in pk.intersection(ck):
+ c = mapping[k]
+ p = previous[k]
+ if c != p:
+ delta[k] = Delta(p, c)
+
+ return delta
+
+ @contextlib.contextmanager
+ def hook_scope(self, name=""):
+ """Scope all future interactions to the current hook execution
+ revision."""
+ assert not self.revision
+ self.cursor.execute(
+ 'insert into hooks (hook, date) values (?, ?)',
+ (name or sys.argv[0],
+ datetime.datetime.utcnow().isoformat()))
+ self.revision = self.cursor.lastrowid
+ try:
+ yield self.revision
+ self.revision = None
+ except:
+ self.flush(False)
+ self.revision = None
+ raise
+ else:
+ self.flush()
+
+ def flush(self, save=True):
+ if save:
+ self.conn.commit()
+ elif self._closed:
+ return
+ else:
+ self.conn.rollback()
+
+ def _init(self):
+ self.cursor.execute('''
+ create table if not exists kv (
+ key text,
+ data text,
+ primary key (key)
+ )''')
+ self.cursor.execute('''
+ create table if not exists kv_revisions (
+ key text,
+ revision integer,
+ data text,
+ primary key (key, revision)
+ )''')
+ self.cursor.execute('''
+ create table if not exists hooks (
+ version integer primary key autoincrement,
+ hook text,
+ date text
+ )''')
+ self.conn.commit()
+
+ def gethistory(self, key, deserialize=False):
+ self.cursor.execute(
+ '''
+ select kv.revision, kv.key, kv.data, h.hook, h.date
+ from kv_revisions kv,
+ hooks h
+ where kv.key=?
+ and kv.revision = h.version
+ ''', [key])
+ if deserialize is False:
+ return self.cursor.fetchall()
+ return map(_parse_history, self.cursor.fetchall())
+
+ def debug(self, fh=sys.stderr):
+ self.cursor.execute('select * from kv')
+ pprint.pprint(self.cursor.fetchall(), stream=fh)
+ self.cursor.execute('select * from kv_revisions')
+ pprint.pprint(self.cursor.fetchall(), stream=fh)
+
+
+def _parse_history(d):
+ return (d[0], d[1], json.loads(d[2]), d[3],
+ datetime.datetime.strptime(d[-1], "%Y-%m-%dT%H:%M:%S.%f"))
+
+
+class HookData(object):
+ """Simple integration for existing hook exec frameworks.
+
+ Records all unit information, and stores deltas for processing
+ by the hook.
+
+ Sample::
+
+ from charmhelper.core import hookenv, unitdata
+
+ changes = unitdata.HookData()
+ db = unitdata.kv()
+ hooks = hookenv.Hooks()
+
+ @hooks.hook
+ def config_changed():
+ # View all changes to configuration
+ for changed, (prev, cur) in changes.conf.items():
+ print('config changed', changed,
+ 'previous value', prev,
+ 'current value', cur)
+
+ # Get some unit specific bookeeping
+ if not db.get('pkg_key'):
+ key = urllib.urlopen('https://example.com/pkg_key').read()
+ db.set('pkg_key', key)
+
+ if __name__ == '__main__':
+ with changes():
+ hook.execute()
+
+ """
+ def __init__(self):
+ self.kv = kv()
+ self.conf = None
+ self.rels = None
+
+ @contextlib.contextmanager
+ def __call__(self):
+ from charmhelpers.core import hookenv
+ hook_name = hookenv.hook_name()
+
+ with self.kv.hook_scope(hook_name):
+ self._record_charm_version(hookenv.charm_dir())
+ delta_config, delta_relation = self._record_hook(hookenv)
+ yield self.kv, delta_config, delta_relation
+
+ def _record_charm_version(self, charm_dir):
+ # Record revisions.. charm revisions are meaningless
+ # to charm authors as they don't control the revision.
+ # so logic dependnent on revision is not particularly
+ # useful, however it is useful for debugging analysis.
+ charm_rev = open(
+ os.path.join(charm_dir, 'revision')).read().strip()
+ charm_rev = charm_rev or '0'
+ revs = self.kv.get('charm_revisions', [])
+ if charm_rev not in revs:
+ revs.append(charm_rev.strip() or '0')
+ self.kv.set('charm_revisions', revs)
+
+ def _record_hook(self, hookenv):
+ data = hookenv.execution_environment()
+ self.conf = conf_delta = self.kv.delta(data['conf'], 'config')
+ self.rels = rels_delta = self.kv.delta(data['rels'], 'rels')
+ self.kv.set('env', data['env'])
+ self.kv.set('unit', data['unit'])
+ self.kv.set('relid', data.get('relid'))
+ return conf_delta, rels_delta
+
+
+class Record(dict):
+
+ __slots__ = ()
+
+ def __getattr__(self, k):
+ if k in self:
+ return self[k]
+ raise AttributeError(k)
+
+
+class DeltaSet(Record):
+
+ __slots__ = ()
+
+
+Delta = collections.namedtuple('Delta', ['previous', 'current'])
+
+
+_KV = None
+
+
+def kv():
+ global _KV
+ if _KV is None:
+ _KV = Storage()
+ return _KV
diff --git a/hooks/charmhelpers/fetch/archiveurl.py b/hooks/charmhelpers/fetch/archiveurl.py
index d25a0ddd..8dfce505 100644
--- a/hooks/charmhelpers/fetch/archiveurl.py
+++ b/hooks/charmhelpers/fetch/archiveurl.py
@@ -18,6 +18,16 @@ import os
import hashlib
import re
+from charmhelpers.fetch import (
+ BaseFetchHandler,
+ UnhandledSource
+)
+from charmhelpers.payload.archive import (
+ get_archive_handler,
+ extract,
+)
+from charmhelpers.core.host import mkdir, check_hash
+
import six
if six.PY3:
from urllib.request import (
@@ -35,16 +45,6 @@ else:
)
from urlparse import urlparse, urlunparse, parse_qs
-from charmhelpers.fetch import (
- BaseFetchHandler,
- UnhandledSource
-)
-from charmhelpers.payload.archive import (
- get_archive_handler,
- extract,
-)
-from charmhelpers.core.host import mkdir, check_hash
-
def splituser(host):
'''urllib.splituser(), but six's support of this seems broken'''
diff --git a/hooks/charmhelpers/fetch/giturl.py b/hooks/charmhelpers/fetch/giturl.py
index 5376786b..93aae87b 100644
--- a/hooks/charmhelpers/fetch/giturl.py
+++ b/hooks/charmhelpers/fetch/giturl.py
@@ -32,7 +32,7 @@ except ImportError:
apt_install("python-git")
from git import Repo
-from git.exc import GitCommandError
+from git.exc import GitCommandError # noqa E402
class GitUrlFetchHandler(BaseFetchHandler):