From 116e28106a858e0343db22e210ce756eb9f45c46 Mon Sep 17 00:00:00 2001 From: "yolanda.robla@canonical.com" <> Date: Tue, 17 Dec 2013 11:09:56 +0100 Subject: [PATCH 01/19] synced charmhelpers, added rabbitmq_hosts --- .../contrib/openstack/alternatives.py | 17 +++++++ .../charmhelpers/contrib/openstack/context.py | 4 ++ hooks/charmhelpers/contrib/openstack/utils.py | 6 ++- .../contrib/storage/linux/utils.py | 2 +- hooks/charmhelpers/core/host.py | 44 +++++++++++++++++ hooks/charmhelpers/fetch/__init__.py | 42 ++++++++++++---- revision | 2 +- templates/grizzly/cinder.conf | 48 +++++++++++++++++++ 8 files changed, 152 insertions(+), 13 deletions(-) create mode 100644 hooks/charmhelpers/contrib/openstack/alternatives.py create mode 100644 templates/grizzly/cinder.conf diff --git a/hooks/charmhelpers/contrib/openstack/alternatives.py b/hooks/charmhelpers/contrib/openstack/alternatives.py new file mode 100644 index 00000000..b413259c --- /dev/null +++ b/hooks/charmhelpers/contrib/openstack/alternatives.py @@ -0,0 +1,17 @@ +''' Helper for managing alternatives for file conflict resolution ''' + +import subprocess +import shutil +import os + + +def install_alternative(name, target, source, priority=50): + ''' Install alternative configuration ''' + if (os.path.exists(target) and not os.path.islink(target)): + # Move existing file/directory away before installing + shutil.move(target, '{}.bak'.format(target)) + cmd = [ + 'update-alternatives', '--force', '--install', + target, name, source, str(priority) + ] + subprocess.check_call(cmd) diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 8a982ffa..4587b370 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -172,6 +172,10 @@ class AMQPContext(OSContextGenerator): else: ctxt['rabbitmq_host'] = relation_get('private-address', rid=rid, unit=unit) + # if several hosts are provided, send the rabbitmq_hosts + hosts = relation_get('hosts', rid=rid, unit=unit) + if hosts is not None: + ctxt['rabbitmq_hosts'] = hosts ctxt.update({ 'rabbitmq_user': username, 'rabbitmq_password': relation_get('password', rid=rid, diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index d66afd74..67a23378 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -41,6 +41,7 @@ UBUNTU_OPENSTACK_RELEASE = OrderedDict([ ('quantal', 'folsom'), ('raring', 'grizzly'), ('saucy', 'havana'), + ('trusty', 'icehouse') ]) @@ -201,7 +202,7 @@ def os_release(package, base='essex'): def import_key(keyid): - cmd = "apt-key adv --keyserver keyserver.ubuntu.com " \ + cmd = "apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 " \ "--recv-keys %s" % keyid try: subprocess.check_call(cmd.split(' ')) @@ -260,6 +261,9 @@ def configure_installation_source(rel): 'havana': 'precise-updates/havana', 'havana/updates': 'precise-updates/havana', 'havana/proposed': 'precise-proposed/havana', + 'icehouse': 'precise-updates/icehouse', + 'icehouse/updates': 'precise-updates/icehouse', + 'icehouse/proposed': 'precise-proposed/icehouse', } try: diff --git a/hooks/charmhelpers/contrib/storage/linux/utils.py b/hooks/charmhelpers/contrib/storage/linux/utils.py index 5b9b6d47..c40218f0 100644 --- a/hooks/charmhelpers/contrib/storage/linux/utils.py +++ b/hooks/charmhelpers/contrib/storage/linux/utils.py @@ -22,4 +22,4 @@ def zap_disk(block_device): :param block_device: str: Full path of block device to clean. ''' - check_call(['sgdisk', '--zap-all', block_device]) + check_call(['sgdisk', '--zap-all', '--mbrtogpt', block_device]) diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py index 4a6a4a8c..c8c81b28 100644 --- a/hooks/charmhelpers/core/host.py +++ b/hooks/charmhelpers/core/host.py @@ -245,3 +245,47 @@ def pwgen(length=None): random_chars = [ random.choice(alphanumeric_chars) for _ in range(length)] return(''.join(random_chars)) + + +def list_nics(nic_type): + '''Return a list of nics of given type(s)''' + if isinstance(nic_type, basestring): + int_types = [nic_type] + else: + int_types = nic_type + interfaces = [] + for int_type in int_types: + cmd = ['ip', 'addr', 'show', 'label', int_type + '*'] + ip_output = subprocess.check_output(cmd).split('\n') + ip_output = (line for line in ip_output if line) + for line in ip_output: + if line.split()[1].startswith(int_type): + interfaces.append(line.split()[1].replace(":", "")) + return interfaces + + +def set_nic_mtu(nic, mtu): + '''Set MTU on a network interface''' + cmd = ['ip', 'link', 'set', nic, 'mtu', mtu] + subprocess.check_call(cmd) + + +def get_nic_mtu(nic): + cmd = ['ip', 'addr', 'show', nic] + ip_output = subprocess.check_output(cmd).split('\n') + mtu = "" + for line in ip_output: + words = line.split() + if 'mtu' in words: + mtu = words[words.index("mtu") + 1] + return mtu + + +def get_nic_hwaddr(nic): + cmd = ['ip', '-o', '-0', 'addr', 'show', nic] + ip_output = subprocess.check_output(cmd) + hwaddr = "" + words = ip_output.split() + if 'link/ether' in words: + hwaddr = words[words.index('link/ether') + 1] + return hwaddr diff --git a/hooks/charmhelpers/fetch/__init__.py b/hooks/charmhelpers/fetch/__init__.py index fa0172a9..1f4f6315 100644 --- a/hooks/charmhelpers/fetch/__init__.py +++ b/hooks/charmhelpers/fetch/__init__.py @@ -13,6 +13,7 @@ from charmhelpers.core.hookenv import ( log, ) import apt_pkg +import os CLOUD_ARCHIVE = """# Ubuntu Cloud Archive deb http://ubuntu-cloud.archive.canonical.com/ubuntu {} main @@ -43,8 +44,16 @@ CLOUD_ARCHIVE_POCKETS = { 'precise-havana/updates': 'precise-updates/havana', 'precise-updates/havana': 'precise-updates/havana', 'havana/proposed': 'precise-proposed/havana', - 'precies-havana/proposed': 'precise-proposed/havana', + 'precise-havana/proposed': 'precise-proposed/havana', 'precise-proposed/havana': 'precise-proposed/havana', + # Icehouse + 'icehouse': 'precise-updates/icehouse', + 'precise-icehouse': 'precise-updates/icehouse', + 'precise-icehouse/updates': 'precise-updates/icehouse', + 'precise-updates/icehouse': 'precise-updates/icehouse', + 'icehouse/proposed': 'precise-proposed/icehouse', + 'precise-icehouse/proposed': 'precise-proposed/icehouse', + 'precise-proposed/icehouse': 'precise-proposed/icehouse', } @@ -66,8 +75,10 @@ def filter_installed_packages(packages): def apt_install(packages, options=None, fatal=False): """Install one or more packages""" - options = options or [] - cmd = ['apt-get', '-y'] + if options is None: + options = ['--option=Dpkg::Options::=--force-confold'] + + cmd = ['apt-get', '--assume-yes'] cmd.extend(options) cmd.append('install') if isinstance(packages, basestring): @@ -76,10 +87,14 @@ def apt_install(packages, options=None, fatal=False): cmd.extend(packages) log("Installing {} with options: {}".format(packages, options)) + env = os.environ.copy() + if 'DEBIAN_FRONTEND' not in env: + env['DEBIAN_FRONTEND'] = 'noninteractive' + if fatal: - subprocess.check_call(cmd) + subprocess.check_call(cmd, env=env) else: - subprocess.call(cmd) + subprocess.call(cmd, env=env) def apt_update(fatal=False): @@ -93,7 +108,7 @@ def apt_update(fatal=False): def apt_purge(packages, fatal=False): """Purge one or more packages""" - cmd = ['apt-get', '-y', 'purge'] + cmd = ['apt-get', '--assume-yes', 'purge'] if isinstance(packages, basestring): cmd.append(packages) else: @@ -123,14 +138,16 @@ def add_source(source, key=None): if (source.startswith('ppa:') or source.startswith('http:') or source.startswith('deb ') or - source.startswith('cloud-archive:')): + source.startswith('cloud-archive:')): subprocess.check_call(['add-apt-repository', '--yes', source]) elif source.startswith('cloud:'): apt_install(filter_installed_packages(['ubuntu-cloud-keyring']), fatal=True) pocket = source.split(':')[-1] if pocket not in CLOUD_ARCHIVE_POCKETS: - raise SourceConfigError('Unsupported cloud: source option %s' % pocket) + raise SourceConfigError( + 'Unsupported cloud: source option %s' % + pocket) actual_pocket = CLOUD_ARCHIVE_POCKETS[pocket] with open('/etc/apt/sources.list.d/cloud-archive.list', 'w') as apt: apt.write(CLOUD_ARCHIVE.format(actual_pocket)) @@ -220,7 +237,9 @@ def install_from_config(config_var_name): class BaseFetchHandler(object): + """Base class for FetchHandler implementations in fetch plugins""" + def can_handle(self, source): """Returns True if the source can be handled. Otherwise returns a string explaining why it cannot""" @@ -248,10 +267,13 @@ def plugins(fetch_handlers=None): for handler_name in fetch_handlers: package, classname = handler_name.rsplit('.', 1) try: - handler_class = getattr(importlib.import_module(package), classname) + handler_class = getattr( + importlib.import_module(package), + classname) plugin_list.append(handler_class()) except (ImportError, AttributeError): # Skip missing plugins so that they can be ommitted from # installation if desired - log("FetchHandler {} not found, skipping plugin".format(handler_name)) + log("FetchHandler {} not found, skipping plugin".format( + handler_name)) return plugin_list diff --git a/revision b/revision index b0d73241..fd03ab2a 100644 --- a/revision +++ b/revision @@ -1 +1 @@ -129 +130 diff --git a/templates/grizzly/cinder.conf b/templates/grizzly/cinder.conf new file mode 100644 index 00000000..35988f90 --- /dev/null +++ b/templates/grizzly/cinder.conf @@ -0,0 +1,48 @@ +############################################################################### +# [ WARNING ] +# cinder configuration file maintained by Juju +# local changes may be overwritten. +############################################################################### +[DEFAULT] +rootwrap_config = /etc/cinder/rootwrap.conf +api_paste_confg = /etc/cinder/api-paste.ini +iscsi_helper = tgtadm +volume_name_template = volume-%s +volume_group = cinder-volumes +verbose = True +auth_strategy = keystone +state_path = /var/lib/cinder +lock_path = /var/lock/cinder +volumes_dir = /var/lib/cinder/volumes +{% if database_host -%} +sql_connection = mysql://{{ database_user }}:{{ database_password }}@{{ database_host }}/{{ database }} +{% endif -%} +{% if rabbitmq_host or rabbitmq_hosts -%} +notification_driver = cinder.openstack.common.notifier.rabbit_notifier +control_exchange = cinder +rabbit_userid = {{ rabbitmq_user }} +rabbit_password = {{ rabbitmq_password }} +rabbit_virtual_host = {{ rabbitmq_virtual_host }} +{% if rabbitmq_hosts -%} +rabbit_hosts = {{ rabbitmq_hosts }} +{% else %} +rabbit_host = {{ rabbitmq_host }} +{% endif -%} +{% endif -%} +{% if volume_driver -%} +volume_driver = {{ volume_driver }} +{% endif -%} +{% if rbd_pool -%} +rbd_pool = {{ rbd_pool }} +host = {{ host }} +rbd_user = {{ rbd_user }} +{% endif -%} +{% if osapi_volume_listen_port -%} +osapi_volume_listen_port = {{ osapi_volume_listen_port }} +{% endif -%} +{% if glance_api_servers -%} +glance_api_servers = {{ glance_api_servers }} +{% endif -%} +{% if glance_api_version -%} +glance_api_version = {{ glance_api_version }} +{% endif -%} From ba758960e8dfcb0d872ae9e2463c2e26c4d6f637 Mon Sep 17 00:00:00 2001 From: "yolanda.robla@canonical.com" <> Date: Tue, 17 Dec 2013 12:13:41 +0100 Subject: [PATCH 02/19] correctly sending rabbitmq_hosts --- hooks/charmhelpers/contrib/openstack/context.py | 5 ----- revision | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 4587b370..04fb69ed 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -185,11 +185,6 @@ class AMQPContext(OSContextGenerator): if context_complete(ctxt): # Sufficient information found = break out! break - # Used for active/active rabbitmq >= grizzly - ctxt['rabbitmq_hosts'] = [] - for unit in related_units(rid): - ctxt['rabbitmq_hosts'].append(relation_get('private-address', - rid=rid, unit=unit)) if not context_complete(ctxt): return {} else: diff --git a/revision b/revision index fd03ab2a..a57f6ce7 100644 --- a/revision +++ b/revision @@ -1 +1 @@ -130 +131 From 18f0c085d8e93497617c60846bbefb36059f217a Mon Sep 17 00:00:00 2001 From: "yolanda.robla@canonical.com" <> Date: Tue, 17 Dec 2013 12:31:09 +0100 Subject: [PATCH 03/19] only sending hosts if not vip --- hooks/charmhelpers/contrib/openstack/context.py | 11 +++++++---- revision | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 04fb69ed..43fa3b94 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -172,10 +172,6 @@ class AMQPContext(OSContextGenerator): else: ctxt['rabbitmq_host'] = relation_get('private-address', rid=rid, unit=unit) - # if several hosts are provided, send the rabbitmq_hosts - hosts = relation_get('hosts', rid=rid, unit=unit) - if hosts is not None: - ctxt['rabbitmq_hosts'] = hosts ctxt.update({ 'rabbitmq_user': username, 'rabbitmq_password': relation_get('password', rid=rid, @@ -185,6 +181,13 @@ class AMQPContext(OSContextGenerator): if context_complete(ctxt): # Sufficient information found = break out! break + # Used for active/active rabbitmq >= grizzly + if relation_get('vip', rid=rid, unit=unit) is None: + rabbitmq_hosts = [] + for unit in related_units(rid): + rabbitmq_hosts.append(relation_get('private-address', + rid=rid, unit=unit)) + ctxt['rabbitmq_hosts'] = ','.join(rabbitmq_hosts) if not context_complete(ctxt): return {} else: diff --git a/revision b/revision index a57f6ce7..94361d49 100644 --- a/revision +++ b/revision @@ -1 +1 @@ -131 +132 From b77a2d631ce20c13f9eae74da9f9929b4dcf3757 Mon Sep 17 00:00:00 2001 From: "yolanda.robla@canonical.com" <> Date: Wed, 8 Jan 2014 11:13:13 +0100 Subject: [PATCH 04/19] refreshed charmhelpers --- hooks/charmhelpers/cli/README.rst | 57 ++ hooks/charmhelpers/cli/__init__.py | 147 +++++ hooks/charmhelpers/cli/commands.py | 2 + hooks/charmhelpers/cli/host.py | 15 + .../charmhelpers/contrib/ansible/__init__.py | 165 +++++ .../charmhelpers/contrib/charmhelpers/IMPORT | 4 + .../contrib/charmhelpers/__init__.py | 184 ++++++ .../charmhelpers/contrib/charmsupport/IMPORT | 14 + .../contrib/charmsupport/__init__.py | 0 .../charmhelpers/contrib/charmsupport/nrpe.py | 216 +++++++ .../contrib/charmsupport/volumes.py | 156 +++++ hooks/charmhelpers/contrib/jujugui/IMPORT | 4 + .../charmhelpers/contrib/jujugui/__init__.py | 0 hooks/charmhelpers/contrib/jujugui/utils.py | 602 ++++++++++++++++++ .../charmhelpers/contrib/network/__init__.py | 0 .../contrib/network/ovs/__init__.py | 75 +++ .../charmhelpers/contrib/openstack/context.py | 64 +- .../templates/openstack_https_frontend.conf | 24 +- .../contrib/saltstack/__init__.py | 102 +++ hooks/charmhelpers/contrib/ssl/__init__.py | 78 +++ .../contrib/templating/__init__.py | 0 .../contrib/templating/contexts.py | 73 +++ .../contrib/templating/pyformat.py | 13 + hooks/charmhelpers/payload/archive.py | 57 ++ 24 files changed, 2012 insertions(+), 40 deletions(-) create mode 100644 hooks/charmhelpers/cli/README.rst create mode 100644 hooks/charmhelpers/cli/__init__.py create mode 100644 hooks/charmhelpers/cli/commands.py create mode 100644 hooks/charmhelpers/cli/host.py create mode 100644 hooks/charmhelpers/contrib/ansible/__init__.py create mode 100644 hooks/charmhelpers/contrib/charmhelpers/IMPORT create mode 100644 hooks/charmhelpers/contrib/charmhelpers/__init__.py create mode 100644 hooks/charmhelpers/contrib/charmsupport/IMPORT create mode 100644 hooks/charmhelpers/contrib/charmsupport/__init__.py create mode 100644 hooks/charmhelpers/contrib/charmsupport/nrpe.py create mode 100644 hooks/charmhelpers/contrib/charmsupport/volumes.py create mode 100644 hooks/charmhelpers/contrib/jujugui/IMPORT create mode 100644 hooks/charmhelpers/contrib/jujugui/__init__.py create mode 100644 hooks/charmhelpers/contrib/jujugui/utils.py create mode 100644 hooks/charmhelpers/contrib/network/__init__.py create mode 100644 hooks/charmhelpers/contrib/network/ovs/__init__.py mode change 100644 => 120000 hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend.conf create mode 100644 hooks/charmhelpers/contrib/saltstack/__init__.py create mode 100644 hooks/charmhelpers/contrib/ssl/__init__.py create mode 100644 hooks/charmhelpers/contrib/templating/__init__.py create mode 100644 hooks/charmhelpers/contrib/templating/contexts.py create mode 100644 hooks/charmhelpers/contrib/templating/pyformat.py create mode 100644 hooks/charmhelpers/payload/archive.py diff --git a/hooks/charmhelpers/cli/README.rst b/hooks/charmhelpers/cli/README.rst new file mode 100644 index 00000000..f7901c09 --- /dev/null +++ b/hooks/charmhelpers/cli/README.rst @@ -0,0 +1,57 @@ +========== +Commandant +========== + +----------------------------------------------------- +Automatic command-line interfaces to Python functions +----------------------------------------------------- + +One of the benefits of ``libvirt`` is the uniformity of the interface: the C API (as well as the bindings in other languages) is a set of functions that accept parameters that are nearly identical to the command-line arguments. If you run ``virsh``, you get an interactive command prompt that supports all of the same commands that your shell scripts use as ``virsh`` subcommands. + +Command execution and stdio manipulation is the greatest common factor across all development systems in the POSIX environment. By exposing your functions as commands that manipulate streams of text, you can make life easier for all the Ruby and Erlang and Go programmers in your life. + +Goals +===== + +* Single decorator to expose a function as a command. + * now two decorators - one "automatic" and one that allows authors to manipulate the arguments for fine-grained control.(MW) +* Automatic analysis of function signature through ``inspect.getargspec()`` +* Command argument parser built automatically with ``argparse`` +* Interactive interpreter loop object made with ``Cmd`` +* Options to output structured return value data via ``pprint``, ``yaml`` or ``json`` dumps. + +Other Important Features that need writing +------------------------------------------ + +* Help and Usage documentation can be automatically generated, but it will be important to let users override this behaviour +* The decorator should allow specifying further parameters to the parser's add_argument() calls, to specify types or to make arguments behave as boolean flags, etc. + - Filename arguments are important, as good practice is for functions to accept file objects as parameters. + - choices arguments help to limit bad input before the function is called +* Some automatic behaviour could make for better defaults, once the user can override them. + - We could automatically detect arguments that default to False or True, and automatically support --no-foo for foo=True. + - We could automatically support hyphens as alternates for underscores + - Arguments defaulting to sequence types could support the ``append`` action. + + +----------------------------------------------------- +Implementing subcommands +----------------------------------------------------- + +(WIP) + +So as to avoid dependencies on the cli module, subcommands should be defined separately from their implementations. The recommmendation would be to place definitions into separate modules near the implementations which they expose. + +Some examples:: + + from charmhelpers.cli import CommandLine + from charmhelpers.payload import execd + from charmhelpers.foo import bar + + cli = CommandLine() + + cli.subcommand(execd.execd_run) + + @cli.subcommand_builder("bar", help="Bar baz qux") + def barcmd_builder(subparser): + subparser.add_argument('argument1', help="yackety") + return bar diff --git a/hooks/charmhelpers/cli/__init__.py b/hooks/charmhelpers/cli/__init__.py new file mode 100644 index 00000000..def53983 --- /dev/null +++ b/hooks/charmhelpers/cli/__init__.py @@ -0,0 +1,147 @@ +import inspect +import itertools +import argparse +import sys + + +class OutputFormatter(object): + def __init__(self, outfile=sys.stdout): + self.formats = ( + "raw", + "json", + "py", + "yaml", + "csv", + "tab", + ) + self.outfile = outfile + + def add_arguments(self, argument_parser): + formatgroup = argument_parser.add_mutually_exclusive_group() + choices = self.supported_formats + formatgroup.add_argument("--format", metavar='FMT', + help="Select output format for returned data, " + "where FMT is one of: {}".format(choices), + choices=choices, default='raw') + for fmt in self.formats: + fmtfunc = getattr(self, fmt) + formatgroup.add_argument("-{}".format(fmt[0]), + "--{}".format(fmt), action='store_const', + const=fmt, dest='format', + help=fmtfunc.__doc__) + + @property + def supported_formats(self): + return self.formats + + def raw(self, output): + """Output data as raw string (default)""" + self.outfile.write(str(output)) + + def py(self, output): + """Output data as a nicely-formatted python data structure""" + import pprint + pprint.pprint(output, stream=self.outfile) + + def json(self, output): + """Output data in JSON format""" + import json + json.dump(output, self.outfile) + + def yaml(self, output): + """Output data in YAML format""" + import yaml + yaml.safe_dump(output, self.outfile) + + def csv(self, output): + """Output data as excel-compatible CSV""" + import csv + csvwriter = csv.writer(self.outfile) + csvwriter.writerows(output) + + def tab(self, output): + """Output data in excel-compatible tab-delimited format""" + import csv + csvwriter = csv.writer(self.outfile, dialect=csv.excel_tab) + csvwriter.writerows(output) + + def format_output(self, output, fmt='raw'): + fmtfunc = getattr(self, fmt) + fmtfunc(output) + + +class CommandLine(object): + argument_parser = None + subparsers = None + formatter = None + + def __init__(self): + if not self.argument_parser: + self.argument_parser = argparse.ArgumentParser(description='Perform common charm tasks') + if not self.formatter: + self.formatter = OutputFormatter() + self.formatter.add_arguments(self.argument_parser) + if not self.subparsers: + self.subparsers = self.argument_parser.add_subparsers(help='Commands') + + def subcommand(self, command_name=None): + """ + Decorate a function as a subcommand. Use its arguments as the + command-line arguments""" + def wrapper(decorated): + cmd_name = command_name or decorated.__name__ + subparser = self.subparsers.add_parser(cmd_name, + description=decorated.__doc__) + for args, kwargs in describe_arguments(decorated): + subparser.add_argument(*args, **kwargs) + subparser.set_defaults(func=decorated) + return decorated + return wrapper + + def subcommand_builder(self, command_name, description=None): + """ + Decorate a function that builds a subcommand. Builders should accept a + single argument (the subparser instance) and return the function to be + run as the command.""" + def wrapper(decorated): + subparser = self.subparsers.add_parser(command_name) + func = decorated(subparser) + subparser.set_defaults(func=func) + subparser.description = description or func.__doc__ + return wrapper + + def run(self): + "Run cli, processing arguments and executing subcommands." + arguments = self.argument_parser.parse_args() + argspec = inspect.getargspec(arguments.func) + vargs = [] + kwargs = {} + if argspec.varargs: + vargs = getattr(arguments, argspec.varargs) + for arg in argspec.args: + kwargs[arg] = getattr(arguments, arg) + self.formatter.format_output(arguments.func(*vargs, **kwargs), arguments.format) + + +cmdline = CommandLine() + + +def describe_arguments(func): + """ + Analyze a function's signature and return a data structure suitable for + passing in as arguments to an argparse parser's add_argument() method.""" + + argspec = inspect.getargspec(func) + # we should probably raise an exception somewhere if func includes **kwargs + if argspec.defaults: + positional_args = argspec.args[:-len(argspec.defaults)] + keyword_names = argspec.args[-len(argspec.defaults):] + for arg, default in itertools.izip(keyword_names, argspec.defaults): + yield ('--{}'.format(arg),), {'default': default} + else: + positional_args = argspec.args + + for arg in positional_args: + yield (arg,), {} + if argspec.varargs: + yield (argspec.varargs,), {'nargs': '*'} diff --git a/hooks/charmhelpers/cli/commands.py b/hooks/charmhelpers/cli/commands.py new file mode 100644 index 00000000..5aa79705 --- /dev/null +++ b/hooks/charmhelpers/cli/commands.py @@ -0,0 +1,2 @@ +from . import CommandLine +import host diff --git a/hooks/charmhelpers/cli/host.py b/hooks/charmhelpers/cli/host.py new file mode 100644 index 00000000..bada5244 --- /dev/null +++ b/hooks/charmhelpers/cli/host.py @@ -0,0 +1,15 @@ +from . import cmdline +from charmhelpers.core import host + + +@cmdline.subcommand() +def mounts(): + "List mounts" + return host.mounts() + + +@cmdline.subcommand_builder('service', description="Control system services") +def service(subparser): + subparser.add_argument("action", help="The action to perform (start, stop, etc...)") + subparser.add_argument("service_name", help="Name of the service to control") + return host.service diff --git a/hooks/charmhelpers/contrib/ansible/__init__.py b/hooks/charmhelpers/contrib/ansible/__init__.py new file mode 100644 index 00000000..f72bae36 --- /dev/null +++ b/hooks/charmhelpers/contrib/ansible/__init__.py @@ -0,0 +1,165 @@ +# Copyright 2013 Canonical Ltd. +# +# Authors: +# Charm Helpers Developers +"""Charm Helpers ansible - declare the state of your machines. + +This helper enables you to declare your machine state, rather than +program it procedurally (and have to test each change to your procedures). +Your install hook can be as simple as: + +{{{ +import charmhelpers.contrib.ansible + + +def install(): + charmhelpers.contrib.ansible.install_ansible_support() + charmhelpers.contrib.ansible.apply_playbook('playbooks/install.yaml') +}}} + +and won't need to change (nor will its tests) when you change the machine +state. + +All of your juju config and relation-data are available as template +variables within your playbooks and templates. An install playbook looks +something like: + +{{{ +--- +- hosts: localhost + user: root + + tasks: + - name: Add private repositories. + template: + src: ../templates/private-repositories.list.jinja2 + dest: /etc/apt/sources.list.d/private.list + + - name: Update the cache. + apt: update_cache=yes + + - name: Install dependencies. + apt: pkg={{ item }} + with_items: + - python-mimeparse + - python-webob + - sunburnt + + - name: Setup groups. + group: name={{ item.name }} gid={{ item.gid }} + with_items: + - { name: 'deploy_user', gid: 1800 } + - { name: 'service_user', gid: 1500 } + + ... +}}} + +Read more online about playbooks[1] and standard ansible modules[2]. + +[1] http://www.ansibleworks.com/docs/playbooks.html +[2] http://www.ansibleworks.com/docs/modules.html +""" +import os +import subprocess + +import charmhelpers.contrib.templating.contexts +import charmhelpers.core.host +import charmhelpers.core.hookenv +import charmhelpers.fetch + + +charm_dir = os.environ.get('CHARM_DIR', '') +ansible_hosts_path = '/etc/ansible/hosts' +# Ansible will automatically include any vars in the following +# file in its inventory when run locally. +ansible_vars_path = '/etc/ansible/host_vars/localhost' + + +def install_ansible_support(from_ppa=True): + """Installs the ansible package. + + By default it is installed from the PPA [1] linked from + the ansible website [2]. + + [1] https://launchpad.net/~rquillo/+archive/ansible + [2] http://www.ansibleworks.com/docs/gettingstarted.html#ubuntu-and-debian + + If from_ppa is false, you must ensure that the package is available + from a configured repository. + """ + if from_ppa: + charmhelpers.fetch.add_source('ppa:rquillo/ansible') + charmhelpers.fetch.apt_update(fatal=True) + charmhelpers.fetch.apt_install('ansible') + with open(ansible_hosts_path, 'w+') as hosts_file: + hosts_file.write('localhost ansible_connection=local') + + +def apply_playbook(playbook, tags=None): + tags = tags or [] + tags = ",".join(tags) + charmhelpers.contrib.templating.contexts.juju_state_to_yaml( + ansible_vars_path, namespace_separator='__', + allow_hyphens_in_keys=False) + call = [ + 'ansible-playbook', + '-c', + 'local', + playbook, + ] + if tags: + call.extend(['--tags', '{}'.format(tags)]) + subprocess.check_call(call) + + +class AnsibleHooks(charmhelpers.core.hookenv.Hooks): + """Run a playbook with the hook-name as the tag. + + This helper builds on the standard hookenv.Hooks helper, + but additionally runs the playbook with the hook-name specified + using --tags (ie. running all the tasks tagged with the hook-name). + + Example: + hooks = AnsibleHooks(playbook_path='playbooks/my_machine_state.yaml') + + # All the tasks within my_machine_state.yaml tagged with 'install' + # will be run automatically after do_custom_work() + @hooks.hook() + def install(): + do_custom_work() + + # For most of your hooks, you won't need to do anything other + # than run the tagged tasks for the hook: + @hooks.hook('config-changed', 'start', 'stop') + def just_use_playbook(): + pass + + # As a convenience, you can avoid the above noop function by specifying + # the hooks which are handled by ansible-only and they'll be registered + # for you: + # hooks = AnsibleHooks( + # 'playbooks/my_machine_state.yaml', + # default_hooks=['config-changed', 'start', 'stop']) + + if __name__ == "__main__": + # execute a hook based on the name the program is called by + hooks.execute(sys.argv) + """ + + def __init__(self, playbook_path, default_hooks=None): + """Register any hooks handled by ansible.""" + super(AnsibleHooks, self).__init__() + + self.playbook_path = playbook_path + + default_hooks = default_hooks or [] + noop = lambda *args, **kwargs: None + for hook in default_hooks: + self.register(hook, noop) + + def execute(self, args): + """Execute the hook followed by the playbook using the hook as tag.""" + super(AnsibleHooks, self).execute(args) + hook_name = os.path.basename(args[0]) + charmhelpers.contrib.ansible.apply_playbook( + self.playbook_path, tags=[hook_name]) diff --git a/hooks/charmhelpers/contrib/charmhelpers/IMPORT b/hooks/charmhelpers/contrib/charmhelpers/IMPORT new file mode 100644 index 00000000..d41cb041 --- /dev/null +++ b/hooks/charmhelpers/contrib/charmhelpers/IMPORT @@ -0,0 +1,4 @@ +Source lp:charm-tools/trunk + +charm-tools/helpers/python/charmhelpers/__init__.py -> charmhelpers/charmhelpers/contrib/charmhelpers/__init__.py +charm-tools/helpers/python/charmhelpers/tests/test_charmhelpers.py -> charmhelpers/tests/contrib/charmhelpers/test_charmhelpers.py diff --git a/hooks/charmhelpers/contrib/charmhelpers/__init__.py b/hooks/charmhelpers/contrib/charmhelpers/__init__.py new file mode 100644 index 00000000..b08f33d2 --- /dev/null +++ b/hooks/charmhelpers/contrib/charmhelpers/__init__.py @@ -0,0 +1,184 @@ +# Copyright 2012 Canonical Ltd. This software is licensed under the +# GNU Affero General Public License version 3 (see the file LICENSE). + +import warnings +warnings.warn("contrib.charmhelpers is deprecated", DeprecationWarning) + +"""Helper functions for writing Juju charms in Python.""" + +__metaclass__ = type +__all__ = [ + #'get_config', # core.hookenv.config() + #'log', # core.hookenv.log() + #'log_entry', # core.hookenv.log() + #'log_exit', # core.hookenv.log() + #'relation_get', # core.hookenv.relation_get() + #'relation_set', # core.hookenv.relation_set() + #'relation_ids', # core.hookenv.relation_ids() + #'relation_list', # core.hookenv.relation_units() + #'config_get', # core.hookenv.config() + #'unit_get', # core.hookenv.unit_get() + #'open_port', # core.hookenv.open_port() + #'close_port', # core.hookenv.close_port() + #'service_control', # core.host.service() + 'unit_info', # client-side, NOT IMPLEMENTED + 'wait_for_machine', # client-side, NOT IMPLEMENTED + 'wait_for_page_contents', # client-side, NOT IMPLEMENTED + 'wait_for_relation', # client-side, NOT IMPLEMENTED + 'wait_for_unit', # client-side, NOT IMPLEMENTED +] + +import operator +from shelltoolbox import ( + command, +) +import tempfile +import time +import urllib2 +import yaml + +SLEEP_AMOUNT = 0.1 +# We create a juju_status Command here because it makes testing much, +# much easier. +juju_status = lambda: command('juju')('status') + +# re-implemented as charmhelpers.fetch.configure_sources() +#def configure_source(update=False): +# source = config_get('source') +# if ((source.startswith('ppa:') or +# source.startswith('cloud:') or +# source.startswith('http:'))): +# run('add-apt-repository', source) +# if source.startswith("http:"): +# run('apt-key', 'import', config_get('key')) +# if update: +# run('apt-get', 'update') + + +# DEPRECATED: client-side only +def make_charm_config_file(charm_config): + charm_config_file = tempfile.NamedTemporaryFile() + charm_config_file.write(yaml.dump(charm_config)) + charm_config_file.flush() + # The NamedTemporaryFile instance is returned instead of just the name + # because we want to take advantage of garbage collection-triggered + # deletion of the temp file when it goes out of scope in the caller. + return charm_config_file + + +# DEPRECATED: client-side only +def unit_info(service_name, item_name, data=None, unit=None): + if data is None: + data = yaml.safe_load(juju_status()) + service = data['services'].get(service_name) + if service is None: + # XXX 2012-02-08 gmb: + # This allows us to cope with the race condition that we + # have between deploying a service and having it come up in + # `juju status`. We could probably do with cleaning it up so + # that it fails a bit more noisily after a while. + return '' + units = service['units'] + if unit is not None: + item = units[unit][item_name] + else: + # It might seem odd to sort the units here, but we do it to + # ensure that when no unit is specified, the first unit for the + # service (or at least the one with the lowest number) is the + # one whose data gets returned. + sorted_unit_names = sorted(units.keys()) + item = units[sorted_unit_names[0]][item_name] + return item + + +# DEPRECATED: client-side only +def get_machine_data(): + return yaml.safe_load(juju_status())['machines'] + + +# DEPRECATED: client-side only +def wait_for_machine(num_machines=1, timeout=300): + """Wait `timeout` seconds for `num_machines` machines to come up. + + This wait_for... function can be called by other wait_for functions + whose timeouts might be too short in situations where only a bare + Juju setup has been bootstrapped. + + :return: A tuple of (num_machines, time_taken). This is used for + testing. + """ + # You may think this is a hack, and you'd be right. The easiest way + # to tell what environment we're working in (LXC vs EC2) is to check + # the dns-name of the first machine. If it's localhost we're in LXC + # and we can just return here. + if get_machine_data()[0]['dns-name'] == 'localhost': + return 1, 0 + start_time = time.time() + while True: + # Drop the first machine, since it's the Zookeeper and that's + # not a machine that we need to wait for. This will only work + # for EC2 environments, which is why we return early above if + # we're in LXC. + machine_data = get_machine_data() + non_zookeeper_machines = [ + machine_data[key] for key in machine_data.keys()[1:]] + if len(non_zookeeper_machines) >= num_machines: + all_machines_running = True + for machine in non_zookeeper_machines: + if machine.get('instance-state') != 'running': + all_machines_running = False + break + if all_machines_running: + break + if time.time() - start_time >= timeout: + raise RuntimeError('timeout waiting for service to start') + time.sleep(SLEEP_AMOUNT) + return num_machines, time.time() - start_time + + +# DEPRECATED: client-side only +def wait_for_unit(service_name, timeout=480): + """Wait `timeout` seconds for a given service name to come up.""" + wait_for_machine(num_machines=1) + start_time = time.time() + while True: + state = unit_info(service_name, 'agent-state') + if 'error' in state or state == 'started': + break + if time.time() - start_time >= timeout: + raise RuntimeError('timeout waiting for service to start') + time.sleep(SLEEP_AMOUNT) + if state != 'started': + raise RuntimeError('unit did not start, agent-state: ' + state) + + +# DEPRECATED: client-side only +def wait_for_relation(service_name, relation_name, timeout=120): + """Wait `timeout` seconds for a given relation to come up.""" + start_time = time.time() + while True: + relation = unit_info(service_name, 'relations').get(relation_name) + if relation is not None and relation['state'] == 'up': + break + if time.time() - start_time >= timeout: + raise RuntimeError('timeout waiting for relation to be up') + time.sleep(SLEEP_AMOUNT) + + +# DEPRECATED: client-side only +def wait_for_page_contents(url, contents, timeout=120, validate=None): + if validate is None: + validate = operator.contains + start_time = time.time() + while True: + try: + stream = urllib2.urlopen(url) + except (urllib2.HTTPError, urllib2.URLError): + pass + else: + page = stream.read() + if validate(page, contents): + return page + if time.time() - start_time >= timeout: + raise RuntimeError('timeout waiting for contents of ' + url) + time.sleep(SLEEP_AMOUNT) diff --git a/hooks/charmhelpers/contrib/charmsupport/IMPORT b/hooks/charmhelpers/contrib/charmsupport/IMPORT new file mode 100644 index 00000000..554fddda --- /dev/null +++ b/hooks/charmhelpers/contrib/charmsupport/IMPORT @@ -0,0 +1,14 @@ +Source: lp:charmsupport/trunk + +charmsupport/charmsupport/execd.py -> charm-helpers/charmhelpers/contrib/charmsupport/execd.py +charmsupport/charmsupport/hookenv.py -> charm-helpers/charmhelpers/contrib/charmsupport/hookenv.py +charmsupport/charmsupport/host.py -> charm-helpers/charmhelpers/contrib/charmsupport/host.py +charmsupport/charmsupport/nrpe.py -> charm-helpers/charmhelpers/contrib/charmsupport/nrpe.py +charmsupport/charmsupport/volumes.py -> charm-helpers/charmhelpers/contrib/charmsupport/volumes.py + +charmsupport/tests/test_execd.py -> charm-helpers/tests/contrib/charmsupport/test_execd.py +charmsupport/tests/test_hookenv.py -> charm-helpers/tests/contrib/charmsupport/test_hookenv.py +charmsupport/tests/test_host.py -> charm-helpers/tests/contrib/charmsupport/test_host.py +charmsupport/tests/test_nrpe.py -> charm-helpers/tests/contrib/charmsupport/test_nrpe.py + +charmsupport/bin/charmsupport -> charm-helpers/bin/contrib/charmsupport/charmsupport diff --git a/hooks/charmhelpers/contrib/charmsupport/__init__.py b/hooks/charmhelpers/contrib/charmsupport/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/hooks/charmhelpers/contrib/charmsupport/nrpe.py b/hooks/charmhelpers/contrib/charmsupport/nrpe.py new file mode 100644 index 00000000..26a41eb9 --- /dev/null +++ b/hooks/charmhelpers/contrib/charmsupport/nrpe.py @@ -0,0 +1,216 @@ +"""Compatibility with the nrpe-external-master charm""" +# Copyright 2012 Canonical Ltd. +# +# Authors: +# Matthew Wedgwood + +import subprocess +import pwd +import grp +import os +import re +import shlex +import yaml + +from charmhelpers.core.hookenv import ( + config, + local_unit, + log, + relation_ids, + relation_set, +) + +from charmhelpers.core.host import service + +# This module adds compatibility with the nrpe-external-master and plain nrpe +# subordinate charms. To use it in your charm: +# +# 1. Update metadata.yaml +# +# provides: +# (...) +# nrpe-external-master: +# interface: nrpe-external-master +# scope: container +# +# and/or +# +# provides: +# (...) +# local-monitors: +# interface: local-monitors +# scope: container + +# +# 2. Add the following to config.yaml +# +# nagios_context: +# default: "juju" +# type: string +# description: | +# Used by the nrpe subordinate charms. +# A string that will be prepended to instance name to set the host name +# in nagios. So for instance the hostname would be something like: +# juju-myservice-0 +# If you're running multiple environments with the same services in them +# this allows you to differentiate between them. +# +# 3. Add custom checks (Nagios plugins) to files/nrpe-external-master +# +# 4. Update your hooks.py with something like this: +# +# from charmsupport.nrpe import NRPE +# (...) +# def update_nrpe_config(): +# nrpe_compat = NRPE() +# nrpe_compat.add_check( +# shortname = "myservice", +# description = "Check MyService", +# check_cmd = "check_http -w 2 -c 10 http://localhost" +# ) +# nrpe_compat.add_check( +# "myservice_other", +# "Check for widget failures", +# check_cmd = "/srv/myapp/scripts/widget_check" +# ) +# nrpe_compat.write() +# +# def config_changed(): +# (...) +# update_nrpe_config() +# +# def nrpe_external_master_relation_changed(): +# update_nrpe_config() +# +# def local_monitors_relation_changed(): +# update_nrpe_config() +# +# 5. ln -s hooks.py nrpe-external-master-relation-changed +# ln -s hooks.py local-monitors-relation-changed + + +class CheckException(Exception): + pass + + +class Check(object): + shortname_re = '[A-Za-z0-9-_]+$' + service_template = (""" +#--------------------------------------------------- +# This file is Juju managed +#--------------------------------------------------- +define service {{ + use active-service + host_name {nagios_hostname} + service_description {nagios_hostname}[{shortname}] """ + """{description} + check_command check_nrpe!{command} + servicegroups {nagios_servicegroup} +}} +""") + + def __init__(self, shortname, description, check_cmd): + super(Check, self).__init__() + # XXX: could be better to calculate this from the service name + if not re.match(self.shortname_re, shortname): + raise CheckException("shortname must match {}".format( + Check.shortname_re)) + self.shortname = shortname + self.command = "check_{}".format(shortname) + # Note: a set of invalid characters is defined by the + # Nagios server config + # The default is: illegal_object_name_chars=`~!$%^&*"|'<>?,()= + self.description = description + self.check_cmd = self._locate_cmd(check_cmd) + + def _locate_cmd(self, check_cmd): + search_path = ( + '/usr/lib/nagios/plugins', + '/usr/local/lib/nagios/plugins', + ) + parts = shlex.split(check_cmd) + for path in search_path: + if os.path.exists(os.path.join(path, parts[0])): + command = os.path.join(path, parts[0]) + if len(parts) > 1: + command += " " + " ".join(parts[1:]) + return command + log('Check command not found: {}'.format(parts[0])) + return '' + + def write(self, nagios_context, hostname): + nrpe_check_file = '/etc/nagios/nrpe.d/{}.cfg'.format( + self.command) + with open(nrpe_check_file, 'w') as nrpe_check_config: + nrpe_check_config.write("# check {}\n".format(self.shortname)) + nrpe_check_config.write("command[{}]={}\n".format( + self.command, self.check_cmd)) + + if not os.path.exists(NRPE.nagios_exportdir): + log('Not writing service config as {} is not accessible'.format( + NRPE.nagios_exportdir)) + else: + self.write_service_config(nagios_context, hostname) + + def write_service_config(self, nagios_context, hostname): + for f in os.listdir(NRPE.nagios_exportdir): + if re.search('.*{}.cfg'.format(self.command), f): + os.remove(os.path.join(NRPE.nagios_exportdir, f)) + + templ_vars = { + 'nagios_hostname': hostname, + 'nagios_servicegroup': nagios_context, + 'description': self.description, + 'shortname': self.shortname, + 'command': self.command, + } + nrpe_service_text = Check.service_template.format(**templ_vars) + nrpe_service_file = '{}/service__{}_{}.cfg'.format( + NRPE.nagios_exportdir, hostname, self.command) + with open(nrpe_service_file, 'w') as nrpe_service_config: + nrpe_service_config.write(str(nrpe_service_text)) + + def run(self): + subprocess.call(self.check_cmd) + + +class NRPE(object): + nagios_logdir = '/var/log/nagios' + nagios_exportdir = '/var/lib/nagios/export' + nrpe_confdir = '/etc/nagios/nrpe.d' + + def __init__(self): + super(NRPE, self).__init__() + self.config = config() + self.nagios_context = self.config['nagios_context'] + self.unit_name = local_unit().replace('/', '-') + self.hostname = "{}-{}".format(self.nagios_context, self.unit_name) + self.checks = [] + + def add_check(self, *args, **kwargs): + self.checks.append(Check(*args, **kwargs)) + + def write(self): + try: + nagios_uid = pwd.getpwnam('nagios').pw_uid + nagios_gid = grp.getgrnam('nagios').gr_gid + except: + log("Nagios user not set up, nrpe checks not updated") + return + + if not os.path.exists(NRPE.nagios_logdir): + os.mkdir(NRPE.nagios_logdir) + os.chown(NRPE.nagios_logdir, nagios_uid, nagios_gid) + + nrpe_monitors = {} + monitors = {"monitors": {"remote": {"nrpe": nrpe_monitors}}} + for nrpecheck in self.checks: + nrpecheck.write(self.nagios_context, self.hostname) + nrpe_monitors[nrpecheck.shortname] = { + "command": nrpecheck.command, + } + + service('restart', 'nagios-nrpe-server') + + for rid in relation_ids("local-monitors"): + relation_set(relation_id=rid, monitors=yaml.dump(monitors)) diff --git a/hooks/charmhelpers/contrib/charmsupport/volumes.py b/hooks/charmhelpers/contrib/charmsupport/volumes.py new file mode 100644 index 00000000..0f905dff --- /dev/null +++ b/hooks/charmhelpers/contrib/charmsupport/volumes.py @@ -0,0 +1,156 @@ +''' +Functions for managing volumes in juju units. One volume is supported per unit. +Subordinates may have their own storage, provided it is on its own partition. + +Configuration stanzas: + volume-ephemeral: + type: boolean + default: true + description: > + If false, a volume is mounted as sepecified in "volume-map" + If true, ephemeral storage will be used, meaning that log data + will only exist as long as the machine. YOU HAVE BEEN WARNED. + volume-map: + type: string + default: {} + description: > + YAML map of units to device names, e.g: + "{ rsyslog/0: /dev/vdb, rsyslog/1: /dev/vdb }" + Service units will raise a configure-error if volume-ephemeral + is 'true' and no volume-map value is set. Use 'juju set' to set a + value and 'juju resolved' to complete configuration. + +Usage: + from charmsupport.volumes import configure_volume, VolumeConfigurationError + from charmsupport.hookenv import log, ERROR + def post_mount_hook(): + stop_service('myservice') + def post_mount_hook(): + start_service('myservice') + + if __name__ == '__main__': + try: + configure_volume(before_change=pre_mount_hook, + after_change=post_mount_hook) + except VolumeConfigurationError: + log('Storage could not be configured', ERROR) +''' + +# XXX: Known limitations +# - fstab is neither consulted nor updated + +import os +from charmhelpers.core import hookenv +from charmhelpers.core import host +import yaml + + +MOUNT_BASE = '/srv/juju/volumes' + + +class VolumeConfigurationError(Exception): + '''Volume configuration data is missing or invalid''' + pass + + +def get_config(): + '''Gather and sanity-check volume configuration data''' + volume_config = {} + config = hookenv.config() + + errors = False + + if config.get('volume-ephemeral') in (True, 'True', 'true', 'Yes', 'yes'): + volume_config['ephemeral'] = True + else: + volume_config['ephemeral'] = False + + try: + volume_map = yaml.safe_load(config.get('volume-map', '{}')) + except yaml.YAMLError as e: + hookenv.log("Error parsing YAML volume-map: {}".format(e), + hookenv.ERROR) + errors = True + if volume_map is None: + # probably an empty string + volume_map = {} + elif not isinstance(volume_map, dict): + hookenv.log("Volume-map should be a dictionary, not {}".format( + type(volume_map))) + errors = True + + volume_config['device'] = volume_map.get(os.environ['JUJU_UNIT_NAME']) + if volume_config['device'] and volume_config['ephemeral']: + # asked for ephemeral storage but also defined a volume ID + hookenv.log('A volume is defined for this unit, but ephemeral ' + 'storage was requested', hookenv.ERROR) + errors = True + elif not volume_config['device'] and not volume_config['ephemeral']: + # asked for permanent storage but did not define volume ID + hookenv.log('Ephemeral storage was requested, but there is no volume ' + 'defined for this unit.', hookenv.ERROR) + errors = True + + unit_mount_name = hookenv.local_unit().replace('/', '-') + volume_config['mountpoint'] = os.path.join(MOUNT_BASE, unit_mount_name) + + if errors: + return None + return volume_config + + +def mount_volume(config): + if os.path.exists(config['mountpoint']): + if not os.path.isdir(config['mountpoint']): + hookenv.log('Not a directory: {}'.format(config['mountpoint'])) + raise VolumeConfigurationError() + else: + host.mkdir(config['mountpoint']) + if os.path.ismount(config['mountpoint']): + unmount_volume(config) + if not host.mount(config['device'], config['mountpoint'], persist=True): + raise VolumeConfigurationError() + + +def unmount_volume(config): + if os.path.ismount(config['mountpoint']): + if not host.umount(config['mountpoint'], persist=True): + raise VolumeConfigurationError() + + +def managed_mounts(): + '''List of all mounted managed volumes''' + return filter(lambda mount: mount[0].startswith(MOUNT_BASE), host.mounts()) + + +def configure_volume(before_change=lambda: None, after_change=lambda: None): + '''Set up storage (or don't) according to the charm's volume configuration. + Returns the mount point or "ephemeral". before_change and after_change + are optional functions to be called if the volume configuration changes. + ''' + + config = get_config() + if not config: + hookenv.log('Failed to read volume configuration', hookenv.CRITICAL) + raise VolumeConfigurationError() + + if config['ephemeral']: + if os.path.ismount(config['mountpoint']): + before_change() + unmount_volume(config) + after_change() + return 'ephemeral' + else: + # persistent storage + if os.path.ismount(config['mountpoint']): + mounts = dict(managed_mounts()) + if mounts.get(config['mountpoint']) != config['device']: + before_change() + unmount_volume(config) + mount_volume(config) + after_change() + else: + before_change() + mount_volume(config) + after_change() + return config['mountpoint'] diff --git a/hooks/charmhelpers/contrib/jujugui/IMPORT b/hooks/charmhelpers/contrib/jujugui/IMPORT new file mode 100644 index 00000000..619a403c --- /dev/null +++ b/hooks/charmhelpers/contrib/jujugui/IMPORT @@ -0,0 +1,4 @@ +Source: lp:charms/juju-gui + +juju-gui/hooks/utils.py -> charm-helpers/charmhelpers/contrib/jujugui/utils.py +juju-gui/tests/test_utils.py -> charm-helpers/tests/contrib/jujugui/test_utils.py diff --git a/hooks/charmhelpers/contrib/jujugui/__init__.py b/hooks/charmhelpers/contrib/jujugui/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/hooks/charmhelpers/contrib/jujugui/utils.py b/hooks/charmhelpers/contrib/jujugui/utils.py new file mode 100644 index 00000000..c7d2b89b --- /dev/null +++ b/hooks/charmhelpers/contrib/jujugui/utils.py @@ -0,0 +1,602 @@ +"""Juju GUI charm utilities.""" + +__all__ = [ + 'AGENT', + 'APACHE', + 'API_PORT', + 'CURRENT_DIR', + 'HAPROXY', + 'IMPROV', + 'JUJU_DIR', + 'JUJU_GUI_DIR', + 'JUJU_GUI_SITE', + 'JUJU_PEM', + 'WEB_PORT', + 'bzr_checkout', + 'chain', + 'cmd_log', + 'fetch_api', + 'fetch_gui', + 'find_missing_packages', + 'first_path_in_dir', + 'get_api_address', + 'get_npm_cache_archive_url', + 'get_release_file_url', + 'get_staging_dependencies', + 'get_zookeeper_address', + 'legacy_juju', + 'log_hook', + 'merge', + 'parse_source', + 'prime_npm_cache', + 'render_to_file', + 'save_or_create_certificates', + 'setup_apache', + 'setup_gui', + 'start_agent', + 'start_gui', + 'start_improv', + 'write_apache_config', +] + +from contextlib import contextmanager +import errno +import json +import os +import logging +import shutil +from subprocess import CalledProcessError +import tempfile +from urlparse import urlparse + +import apt +import tempita + +from launchpadlib.launchpad import Launchpad +from shelltoolbox import ( + Serializer, + apt_get_install, + command, + environ, + install_extra_repositories, + run, + script_name, + search_file, + su, +) +from charmhelpers.core.host import ( + service_start, +) +from charmhelpers.core.hookenv import ( + log, + config, + unit_get, +) + + +AGENT = 'juju-api-agent' +APACHE = 'apache2' +IMPROV = 'juju-api-improv' +HAPROXY = 'haproxy' + +API_PORT = 8080 +WEB_PORT = 8000 + +CURRENT_DIR = os.getcwd() +JUJU_DIR = os.path.join(CURRENT_DIR, 'juju') +JUJU_GUI_DIR = os.path.join(CURRENT_DIR, 'juju-gui') +JUJU_GUI_SITE = '/etc/apache2/sites-available/juju-gui' +JUJU_GUI_PORTS = '/etc/apache2/ports.conf' +JUJU_PEM = 'juju.includes-private-key.pem' +BUILD_REPOSITORIES = ('ppa:chris-lea/node.js-legacy',) +DEB_BUILD_DEPENDENCIES = ( + 'bzr', 'imagemagick', 'make', 'nodejs', 'npm', +) +DEB_STAGE_DEPENDENCIES = ( + 'zookeeper', +) + + +# Store the configuration from on invocation to the next. +config_json = Serializer('/tmp/config.json') +# Bazaar checkout command. +bzr_checkout = command('bzr', 'co', '--lightweight') +# Whether or not the charm is deployed using juju-core. +# If juju-core has been used to deploy the charm, an agent.conf file must +# be present in the charm parent directory. +legacy_juju = lambda: not os.path.exists( + os.path.join(CURRENT_DIR, '..', 'agent.conf')) + + +def _get_build_dependencies(): + """Install deb dependencies for building.""" + log('Installing build dependencies.') + cmd_log(install_extra_repositories(*BUILD_REPOSITORIES)) + cmd_log(apt_get_install(*DEB_BUILD_DEPENDENCIES)) + + +def get_api_address(unit_dir): + """Return the Juju API address stored in the uniter agent.conf file.""" + import yaml # python-yaml is only installed if juju-core is used. + # XXX 2013-03-27 frankban bug=1161443: + # currently the uniter agent.conf file does not include the API + # address. For now retrieve it from the machine agent file. + base_dir = os.path.abspath(os.path.join(unit_dir, '..')) + for dirname in os.listdir(base_dir): + if dirname.startswith('machine-'): + agent_conf = os.path.join(base_dir, dirname, 'agent.conf') + break + else: + raise IOError('Juju agent configuration file not found.') + contents = yaml.load(open(agent_conf)) + return contents['apiinfo']['addrs'][0] + + +def get_staging_dependencies(): + """Install deb dependencies for the stage (improv) environment.""" + log('Installing stage dependencies.') + cmd_log(apt_get_install(*DEB_STAGE_DEPENDENCIES)) + + +def first_path_in_dir(directory): + """Return the full path of the first file/dir in *directory*.""" + return os.path.join(directory, os.listdir(directory)[0]) + + +def _get_by_attr(collection, attr, value): + """Return the first item in collection having attr == value. + + Return None if the item is not found. + """ + for item in collection: + if getattr(item, attr) == value: + return item + + +def get_release_file_url(project, series_name, release_version): + """Return the URL of the release file hosted in Launchpad. + + The returned URL points to a release file for the given project, series + name and release version. + The argument *project* is a project object as returned by launchpadlib. + The arguments *series_name* and *release_version* are strings. If + *release_version* is None, the URL of the latest release will be returned. + """ + series = _get_by_attr(project.series, 'name', series_name) + if series is None: + raise ValueError('%r: series not found' % series_name) + # Releases are returned by Launchpad in reverse date order. + releases = list(series.releases) + if not releases: + raise ValueError('%r: series does not contain releases' % series_name) + if release_version is not None: + release = _get_by_attr(releases, 'version', release_version) + if release is None: + raise ValueError('%r: release not found' % release_version) + releases = [release] + for release in releases: + for file_ in release.files: + if str(file_).endswith('.tgz'): + return file_.file_link + raise ValueError('%r: file not found' % release_version) + + +def get_zookeeper_address(agent_file_path): + """Retrieve the Zookeeper address contained in the given *agent_file_path*. + + The *agent_file_path* is a path to a file containing a line similar to the + following:: + + env JUJU_ZOOKEEPER="address" + """ + line = search_file('JUJU_ZOOKEEPER', agent_file_path).strip() + return line.split('=')[1].strip('"') + + +@contextmanager +def log_hook(): + """Log when a hook starts and stops its execution. + + Also log to stdout possible CalledProcessError exceptions raised executing + the hook. + """ + script = script_name() + log(">>> Entering {}".format(script)) + try: + yield + except CalledProcessError as err: + log('Exception caught:') + log(err.output) + raise + finally: + log("<<< Exiting {}".format(script)) + + +def parse_source(source): + """Parse the ``juju-gui-source`` option. + + Return a tuple of two elements representing info on how to deploy Juju GUI. + Examples: + - ('stable', None): latest stable release; + - ('stable', '0.1.0'): stable release v0.1.0; + - ('trunk', None): latest trunk release; + - ('trunk', '0.1.0+build.1'): trunk release v0.1.0 bzr revision 1; + - ('branch', 'lp:juju-gui'): release is made from a branch; + - ('url', 'http://example.com/gui'): release from a downloaded file. + """ + if source.startswith('url:'): + source = source[4:] + # Support file paths, including relative paths. + if urlparse(source).scheme == '': + if not source.startswith('/'): + source = os.path.join(os.path.abspath(CURRENT_DIR), source) + source = "file://%s" % source + return 'url', source + if source in ('stable', 'trunk'): + return source, None + if source.startswith('lp:') or source.startswith('http://'): + return 'branch', source + if 'build' in source: + return 'trunk', source + return 'stable', source + + +def render_to_file(template_name, context, destination): + """Render the given *template_name* into *destination* using *context*. + + The tempita template language is used to render contents + (see http://pythonpaste.org/tempita/). + The argument *template_name* is the name or path of the template file: + it may be either a path relative to ``../config`` or an absolute path. + The argument *destination* is a file path. + The argument *context* is a dict-like object. + """ + template_path = os.path.abspath(template_name) + template = tempita.Template.from_filename(template_path) + with open(destination, 'w') as stream: + stream.write(template.substitute(context)) + + +results_log = None + + +def _setupLogging(): + global results_log + if results_log is not None: + return + cfg = config() + logging.basicConfig( + filename=cfg['command-log-file'], + level=logging.INFO, + format="%(asctime)s: %(name)s@%(levelname)s %(message)s") + results_log = logging.getLogger('juju-gui') + + +def cmd_log(results): + global results_log + if not results: + return + if results_log is None: + _setupLogging() + # Since 'results' may be multi-line output, start it on a separate line + # from the logger timestamp, etc. + results_log.info('\n' + results) + + +def start_improv(staging_env, ssl_cert_path, + config_path='/etc/init/juju-api-improv.conf'): + """Start a simulated juju environment using ``improv.py``.""" + log('Setting up staging start up script.') + context = { + 'juju_dir': JUJU_DIR, + 'keys': ssl_cert_path, + 'port': API_PORT, + 'staging_env': staging_env, + } + render_to_file('config/juju-api-improv.conf.template', context, config_path) + log('Starting the staging backend.') + with su('root'): + service_start(IMPROV) + + +def start_agent( + ssl_cert_path, config_path='/etc/init/juju-api-agent.conf', + read_only=False): + """Start the Juju agent and connect to the current environment.""" + # Retrieve the Zookeeper address from the start up script. + unit_dir = os.path.realpath(os.path.join(CURRENT_DIR, '..')) + agent_file = '/etc/init/juju-{0}.conf'.format(os.path.basename(unit_dir)) + zookeeper = get_zookeeper_address(agent_file) + log('Setting up API agent start up script.') + context = { + 'juju_dir': JUJU_DIR, + 'keys': ssl_cert_path, + 'port': API_PORT, + 'zookeeper': zookeeper, + 'read_only': read_only + } + render_to_file('config/juju-api-agent.conf.template', context, config_path) + log('Starting API agent.') + with su('root'): + service_start(AGENT) + + +def start_gui( + console_enabled, login_help, readonly, in_staging, ssl_cert_path, + charmworld_url, serve_tests, haproxy_path='/etc/haproxy/haproxy.cfg', + config_js_path=None, secure=True, sandbox=False): + """Set up and start the Juju GUI server.""" + with su('root'): + run('chown', '-R', 'ubuntu:', JUJU_GUI_DIR) + # XXX 2013-02-05 frankban bug=1116320: + # External insecure resources are still loaded when testing in the + # debug environment. For now, switch to the production environment if + # the charm is configured to serve tests. + if in_staging and not serve_tests: + build_dirname = 'build-debug' + else: + build_dirname = 'build-prod' + build_dir = os.path.join(JUJU_GUI_DIR, build_dirname) + log('Generating the Juju GUI configuration file.') + is_legacy_juju = legacy_juju() + user, password = None, None + if (is_legacy_juju and in_staging) or sandbox: + user, password = 'admin', 'admin' + else: + user, password = None, None + + api_backend = 'python' if is_legacy_juju else 'go' + if secure: + protocol = 'wss' + else: + log('Running in insecure mode! Port 80 will serve unencrypted.') + protocol = 'ws' + + context = { + 'raw_protocol': protocol, + 'address': unit_get('public-address'), + 'console_enabled': json.dumps(console_enabled), + 'login_help': json.dumps(login_help), + 'password': json.dumps(password), + 'api_backend': json.dumps(api_backend), + 'readonly': json.dumps(readonly), + 'user': json.dumps(user), + 'protocol': json.dumps(protocol), + 'sandbox': json.dumps(sandbox), + 'charmworld_url': json.dumps(charmworld_url), + } + if config_js_path is None: + config_js_path = os.path.join( + build_dir, 'juju-ui', 'assets', 'config.js') + render_to_file('config/config.js.template', context, config_js_path) + + write_apache_config(build_dir, serve_tests) + + log('Generating haproxy configuration file.') + if is_legacy_juju: + # The PyJuju API agent is listening on localhost. + api_address = '127.0.0.1:{0}'.format(API_PORT) + else: + # Retrieve the juju-core API server address. + api_address = get_api_address(os.path.join(CURRENT_DIR, '..')) + context = { + 'api_address': api_address, + 'api_pem': JUJU_PEM, + 'legacy_juju': is_legacy_juju, + 'ssl_cert_path': ssl_cert_path, + # In PyJuju environments, use the same certificate for both HTTPS and + # WebSocket connections. In juju-core the system already has the proper + # certificate installed. + 'web_pem': JUJU_PEM, + 'web_port': WEB_PORT, + 'secure': secure + } + render_to_file('config/haproxy.cfg.template', context, haproxy_path) + log('Starting Juju GUI.') + + +def write_apache_config(build_dir, serve_tests=False): + log('Generating the apache site configuration file.') + context = { + 'port': WEB_PORT, + 'serve_tests': serve_tests, + 'server_root': build_dir, + 'tests_root': os.path.join(JUJU_GUI_DIR, 'test', ''), + } + render_to_file('config/apache-ports.template', context, JUJU_GUI_PORTS) + render_to_file('config/apache-site.template', context, JUJU_GUI_SITE) + + +def get_npm_cache_archive_url(Launchpad=Launchpad): + """Figure out the URL of the most recent NPM cache archive on Launchpad.""" + launchpad = Launchpad.login_anonymously('Juju GUI charm', 'production') + project = launchpad.projects['juju-gui'] + # Find the URL of the most recently created NPM cache archive. + npm_cache_url = get_release_file_url(project, 'npm-cache', None) + return npm_cache_url + + +def prime_npm_cache(npm_cache_url): + """Download NPM cache archive and prime the NPM cache with it.""" + # Download the cache archive and then uncompress it into the NPM cache. + npm_cache_archive = os.path.join(CURRENT_DIR, 'npm-cache.tgz') + cmd_log(run('curl', '-L', '-o', npm_cache_archive, npm_cache_url)) + npm_cache_dir = os.path.expanduser('~/.npm') + # The NPM cache directory probably does not exist, so make it if not. + try: + os.mkdir(npm_cache_dir) + except OSError, e: + # If the directory already exists then ignore the error. + if e.errno != errno.EEXIST: # File exists. + raise + uncompress = command('tar', '-x', '-z', '-C', npm_cache_dir, '-f') + cmd_log(uncompress(npm_cache_archive)) + + +def fetch_gui(juju_gui_source, logpath): + """Retrieve the Juju GUI release/branch.""" + # Retrieve a Juju GUI release. + origin, version_or_branch = parse_source(juju_gui_source) + if origin == 'branch': + # Make sure we have the dependencies necessary for us to actually make + # a build. + _get_build_dependencies() + # Create a release starting from a branch. + juju_gui_source_dir = os.path.join(CURRENT_DIR, 'juju-gui-source') + log('Retrieving Juju GUI source checkout from %s.' % version_or_branch) + cmd_log(run('rm', '-rf', juju_gui_source_dir)) + cmd_log(bzr_checkout(version_or_branch, juju_gui_source_dir)) + log('Preparing a Juju GUI release.') + logdir = os.path.dirname(logpath) + fd, name = tempfile.mkstemp(prefix='make-distfile-', dir=logdir) + log('Output from "make distfile" sent to %s' % name) + with environ(NO_BZR='1'): + run('make', '-C', juju_gui_source_dir, 'distfile', + stdout=fd, stderr=fd) + release_tarball = first_path_in_dir( + os.path.join(juju_gui_source_dir, 'releases')) + else: + log('Retrieving Juju GUI release.') + if origin == 'url': + file_url = version_or_branch + else: + # Retrieve a release from Launchpad. + launchpad = Launchpad.login_anonymously( + 'Juju GUI charm', 'production') + project = launchpad.projects['juju-gui'] + file_url = get_release_file_url(project, origin, version_or_branch) + log('Downloading release file from %s.' % file_url) + release_tarball = os.path.join(CURRENT_DIR, 'release.tgz') + cmd_log(run('curl', '-L', '-o', release_tarball, file_url)) + return release_tarball + + +def fetch_api(juju_api_branch): + """Retrieve the Juju branch.""" + # Retrieve Juju API source checkout. + log('Retrieving Juju API source checkout.') + cmd_log(run('rm', '-rf', JUJU_DIR)) + cmd_log(bzr_checkout(juju_api_branch, JUJU_DIR)) + + +def setup_gui(release_tarball): + """Set up Juju GUI.""" + # Uncompress the release tarball. + log('Installing Juju GUI.') + release_dir = os.path.join(CURRENT_DIR, 'release') + cmd_log(run('rm', '-rf', release_dir)) + os.mkdir(release_dir) + uncompress = command('tar', '-x', '-z', '-C', release_dir, '-f') + cmd_log(uncompress(release_tarball)) + # Link the Juju GUI dir to the contents of the release tarball. + cmd_log(run('ln', '-sf', first_path_in_dir(release_dir), JUJU_GUI_DIR)) + + +def setup_apache(): + """Set up apache.""" + log('Setting up apache.') + if not os.path.exists(JUJU_GUI_SITE): + cmd_log(run('touch', JUJU_GUI_SITE)) + cmd_log(run('chown', 'ubuntu:', JUJU_GUI_SITE)) + cmd_log( + run('ln', '-s', JUJU_GUI_SITE, + '/etc/apache2/sites-enabled/juju-gui')) + + if not os.path.exists(JUJU_GUI_PORTS): + cmd_log(run('touch', JUJU_GUI_PORTS)) + cmd_log(run('chown', 'ubuntu:', JUJU_GUI_PORTS)) + + with su('root'): + run('a2dissite', 'default') + run('a2ensite', 'juju-gui') + + +def save_or_create_certificates( + ssl_cert_path, ssl_cert_contents, ssl_key_contents): + """Generate the SSL certificates. + + If both *ssl_cert_contents* and *ssl_key_contents* are provided, use them + as certificates; otherwise, generate them. + + Also create a pem file, suitable for use in the haproxy configuration, + concatenating the key and the certificate files. + """ + crt_path = os.path.join(ssl_cert_path, 'juju.crt') + key_path = os.path.join(ssl_cert_path, 'juju.key') + if not os.path.exists(ssl_cert_path): + os.makedirs(ssl_cert_path) + if ssl_cert_contents and ssl_key_contents: + # Save the provided certificates. + with open(crt_path, 'w') as cert_file: + cert_file.write(ssl_cert_contents) + with open(key_path, 'w') as key_file: + key_file.write(ssl_key_contents) + else: + # Generate certificates. + # See http://superuser.com/questions/226192/openssl-without-prompt + cmd_log(run( + 'openssl', 'req', '-new', '-newkey', 'rsa:4096', + '-days', '365', '-nodes', '-x509', '-subj', + # These are arbitrary test values for the certificate. + '/C=GB/ST=Juju/L=GUI/O=Ubuntu/CN=juju.ubuntu.com', + '-keyout', key_path, '-out', crt_path)) + # Generate the pem file. + pem_path = os.path.join(ssl_cert_path, JUJU_PEM) + if os.path.exists(pem_path): + os.remove(pem_path) + with open(pem_path, 'w') as pem_file: + shutil.copyfileobj(open(key_path), pem_file) + shutil.copyfileobj(open(crt_path), pem_file) + + +def find_missing_packages(*packages): + """Given a list of packages, return the packages which are not installed. + """ + cache = apt.Cache() + missing = set() + for pkg_name in packages: + try: + pkg = cache[pkg_name] + except KeyError: + missing.add(pkg_name) + continue + if pkg.is_installed: + continue + missing.add(pkg_name) + return missing + + +## Backend support decorators + +def chain(name): + """Helper method to compose a set of mixin objects into a callable. + + Each method is called in the context of its mixin instance, and its + argument is the Backend instance. + """ + # Chain method calls through all implementing mixins. + def method(self): + for mixin in self.mixins: + a_callable = getattr(type(mixin), name, None) + if a_callable: + a_callable(mixin, self) + + method.__name__ = name + return method + + +def merge(name): + """Helper to merge a property from a set of strategy objects + into a unified set. + """ + # Return merged property from every providing mixin as a set. + @property + def method(self): + result = set() + for mixin in self.mixins: + segment = getattr(type(mixin), name, None) + if segment and isinstance(segment, (list, tuple, set)): + result |= set(segment) + + return result + return method diff --git a/hooks/charmhelpers/contrib/network/__init__.py b/hooks/charmhelpers/contrib/network/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/hooks/charmhelpers/contrib/network/ovs/__init__.py b/hooks/charmhelpers/contrib/network/ovs/__init__.py new file mode 100644 index 00000000..5eba8376 --- /dev/null +++ b/hooks/charmhelpers/contrib/network/ovs/__init__.py @@ -0,0 +1,75 @@ +''' Helpers for interacting with OpenvSwitch ''' +import subprocess +import os +from charmhelpers.core.hookenv import ( + log, WARNING +) +from charmhelpers.core.host import ( + service +) + + +def add_bridge(name): + ''' Add the named bridge to openvswitch ''' + log('Creating bridge {}'.format(name)) + subprocess.check_call(["ovs-vsctl", "--", "--may-exist", "add-br", name]) + + +def del_bridge(name): + ''' Delete the named bridge from openvswitch ''' + log('Deleting bridge {}'.format(name)) + subprocess.check_call(["ovs-vsctl", "--", "--if-exists", "del-br", name]) + + +def add_bridge_port(name, port): + ''' Add a port to the named openvswitch bridge ''' + log('Adding port {} to bridge {}'.format(port, name)) + subprocess.check_call(["ovs-vsctl", "--", "--may-exist", "add-port", + name, port]) + subprocess.check_call(["ip", "link", "set", port, "up"]) + + +def del_bridge_port(name, port): + ''' Delete a port from the named openvswitch bridge ''' + log('Deleting port {} from bridge {}'.format(port, name)) + subprocess.check_call(["ovs-vsctl", "--", "--if-exists", "del-port", + name, port]) + subprocess.check_call(["ip", "link", "set", port, "down"]) + + +def set_manager(manager): + ''' Set the controller for the local openvswitch ''' + log('Setting manager for local ovs to {}'.format(manager)) + subprocess.check_call(['ovs-vsctl', 'set-manager', + 'ssl:{}'.format(manager)]) + + +CERT_PATH = '/etc/openvswitch/ovsclient-cert.pem' + + +def get_certificate(): + ''' Read openvswitch certificate from disk ''' + if os.path.exists(CERT_PATH): + log('Reading ovs certificate from {}'.format(CERT_PATH)) + with open(CERT_PATH, 'r') as cert: + full_cert = cert.read() + begin_marker = "-----BEGIN CERTIFICATE-----" + end_marker = "-----END CERTIFICATE-----" + begin_index = full_cert.find(begin_marker) + end_index = full_cert.rfind(end_marker) + if end_index == -1 or begin_index == -1: + raise RuntimeError("Certificate does not contain valid begin" + " and end markers.") + full_cert = full_cert[begin_index:(end_index + len(end_marker))] + return full_cert + else: + log('Certificate not found', level=WARNING) + return None + + +def full_restart(): + ''' Full restart and reload of openvswitch ''' + if os.path.exists('/etc/init/openvswitch-force-reload-kmod.conf'): + service('start', 'openvswitch-force-reload-kmod') + else: + service('force-reload-kmod', 'openvswitch-switch') diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 43fa3b94..dec1bfe0 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -182,11 +182,11 @@ class AMQPContext(OSContextGenerator): # Sufficient information found = break out! break # Used for active/active rabbitmq >= grizzly - if relation_get('vip', rid=rid, unit=unit) is None: + if 'clustered' not in ctxt and len(related_units(rid))>0: rabbitmq_hosts = [] for unit in related_units(rid): rabbitmq_hosts.append(relation_get('private-address', - rid=rid, unit=unit)) + rid=rid, unit=unit)) ctxt['rabbitmq_hosts'] = ','.join(rabbitmq_hosts) if not context_complete(ctxt): return {} @@ -435,25 +435,55 @@ class NeutronContext(object): class OSConfigFlagContext(OSContextGenerator): - ''' - Responsible adding user-defined config-flags in charm config to a - to a template context. - ''' + """ + Responsible for adding user-defined config-flags in charm config to a + template context. + + NOTE: the value of config-flags may be a comma-separated list of + key=value pairs and some Openstack config files support + comma-separated lists as values. + """ def __call__(self): config_flags = config('config-flags') - if not config_flags or config_flags in ['None', '']: + if not config_flags: return {} - config_flags = config_flags.split(',') + + if config_flags.find('==') >= 0: + log("config_flags is not in expected format (key=value)", + level=ERROR) + raise OSContextError + + # strip the following from each value. + post_strippers = ' ,' + # we strip any leading/trailing '=' or ' ' from the string then + # split on '='. + split = config_flags.strip(' =').split('=') + limit = len(split) flags = {} - for flag in config_flags: - if '=' not in flag: - log('Improperly formatted config-flag, expected k=v ' - 'got %s' % flag, level=WARNING) - continue - k, v = flag.split('=') - flags[k.strip()] = v - ctxt = {'user_config_flags': flags} - return ctxt + for i in xrange(0, limit - 1): + current = split[i] + next = split[i + 1] + vindex = next.rfind(',') + if (i == limit - 2) or (vindex < 0): + value = next + else: + value = next[:vindex] + + if i == 0: + key = current + else: + # if this not the first entry, expect an embedded key. + index = current.rfind(',') + if index < 0: + log("invalid config value(s) at index %s" % (i), + level=ERROR) + raise OSContextError + key = current[index + 1:] + + # Add to collection. + flags[key.strip(post_strippers)] = value.rstrip(post_strippers) + + return {'user_config_flags': flags} class SubordinateConfigContext(OSContextGenerator): diff --git a/hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend.conf b/hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend.conf deleted file mode 100644 index e02dc751..00000000 --- a/hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend.conf +++ /dev/null @@ -1,23 +0,0 @@ -{% if endpoints -%} -{% for ext, int in endpoints -%} -Listen {{ ext }} -NameVirtualHost *:{{ ext }} - - ServerName {{ private_address }} - SSLEngine on - SSLCertificateFile /etc/apache2/ssl/{{ namespace }}/cert - SSLCertificateKeyFile /etc/apache2/ssl/{{ namespace }}/key - ProxyPass / http://localhost:{{ int }}/ - ProxyPassReverse / http://localhost:{{ int }}/ - ProxyPreserveHost on - - - Order deny,allow - Allow from all - - - Order allow,deny - Allow from all - -{% endfor -%} -{% endif -%} diff --git a/hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend.conf b/hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend.conf new file mode 120000 index 00000000..9a2f6f2b --- /dev/null +++ b/hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend.conf @@ -0,0 +1 @@ +openstack_https_frontend \ No newline at end of file diff --git a/hooks/charmhelpers/contrib/saltstack/__init__.py b/hooks/charmhelpers/contrib/saltstack/__init__.py new file mode 100644 index 00000000..19078f8e --- /dev/null +++ b/hooks/charmhelpers/contrib/saltstack/__init__.py @@ -0,0 +1,102 @@ +"""Charm Helpers saltstack - declare the state of your machines. + +This helper enables you to declare your machine state, rather than +program it procedurally (and have to test each change to your procedures). +Your install hook can be as simple as: + +{{{ +from charmhelpers.contrib.saltstack import ( + install_salt_support, + update_machine_state, +) + + +def install(): + install_salt_support() + update_machine_state('machine_states/dependencies.yaml') + update_machine_state('machine_states/installed.yaml') +}}} + +and won't need to change (nor will its tests) when you change the machine +state. + +It's using a python package called salt-minion which allows various formats for +specifying resources, such as: + +{{{ +/srv/{{ basedir }}: + file.directory: + - group: ubunet + - user: ubunet + - require: + - user: ubunet + - recurse: + - user + - group + +ubunet: + group.present: + - gid: 1500 + user.present: + - uid: 1500 + - gid: 1500 + - createhome: False + - require: + - group: ubunet +}}} + +The docs for all the different state definitions are at: + http://docs.saltstack.com/ref/states/all/ + + +TODO: + * Add test helpers which will ensure that machine state definitions + are functionally (but not necessarily logically) correct (ie. getting + salt to parse all state defs. + * Add a link to a public bootstrap charm example / blogpost. + * Find a way to obviate the need to use the grains['charm_dir'] syntax + in templates. +""" +# Copyright 2013 Canonical Ltd. +# +# Authors: +# Charm Helpers Developers +import subprocess + +import charmhelpers.contrib.templating.contexts +import charmhelpers.core.host +import charmhelpers.core.hookenv + + +salt_grains_path = '/etc/salt/grains' + + +def install_salt_support(from_ppa=True): + """Installs the salt-minion helper for machine state. + + By default the salt-minion package is installed from + the saltstack PPA. If from_ppa is False you must ensure + that the salt-minion package is available in the apt cache. + """ + if from_ppa: + subprocess.check_call([ + '/usr/bin/add-apt-repository', + '--yes', + 'ppa:saltstack/salt', + ]) + subprocess.check_call(['/usr/bin/apt-get', 'update']) + # We install salt-common as salt-minion would run the salt-minion + # daemon. + charmhelpers.fetch.apt_install('salt-common') + + +def update_machine_state(state_path): + """Update the machine state using the provided state declaration.""" + charmhelpers.contrib.templating.contexts.juju_state_to_yaml( + salt_grains_path) + subprocess.check_call([ + 'salt-call', + '--local', + 'state.template', + state_path, + ]) diff --git a/hooks/charmhelpers/contrib/ssl/__init__.py b/hooks/charmhelpers/contrib/ssl/__init__.py new file mode 100644 index 00000000..2999c0a3 --- /dev/null +++ b/hooks/charmhelpers/contrib/ssl/__init__.py @@ -0,0 +1,78 @@ +import subprocess +from charmhelpers.core import hookenv + + +def generate_selfsigned(keyfile, certfile, keysize="1024", config=None, subject=None, cn=None): + """Generate selfsigned SSL keypair + + You must provide one of the 3 optional arguments: + config, subject or cn + If more than one is provided the leftmost will be used + + Arguments: + keyfile -- (required) full path to the keyfile to be created + certfile -- (required) full path to the certfile to be created + keysize -- (optional) SSL key length + config -- (optional) openssl configuration file + subject -- (optional) dictionary with SSL subject variables + cn -- (optional) cerfificate common name + + Required keys in subject dict: + cn -- Common name (eq. FQDN) + + Optional keys in subject dict + country -- Country Name (2 letter code) + state -- State or Province Name (full name) + locality -- Locality Name (eg, city) + organization -- Organization Name (eg, company) + organizational_unit -- Organizational Unit Name (eg, section) + email -- Email Address + """ + + cmd = [] + if config: + cmd = ["/usr/bin/openssl", "req", "-new", "-newkey", + "rsa:{}".format(keysize), "-days", "365", "-nodes", "-x509", + "-keyout", keyfile, + "-out", certfile, "-config", config] + elif subject: + ssl_subject = "" + if "country" in subject: + ssl_subject = ssl_subject + "/C={}".format(subject["country"]) + if "state" in subject: + ssl_subject = ssl_subject + "/ST={}".format(subject["state"]) + if "locality" in subject: + ssl_subject = ssl_subject + "/L={}".format(subject["locality"]) + if "organization" in subject: + ssl_subject = ssl_subject + "/O={}".format(subject["organization"]) + if "organizational_unit" in subject: + ssl_subject = ssl_subject + "/OU={}".format(subject["organizational_unit"]) + if "cn" in subject: + ssl_subject = ssl_subject + "/CN={}".format(subject["cn"]) + else: + hookenv.log("When using \"subject\" argument you must " + "provide \"cn\" field at very least") + return False + if "email" in subject: + ssl_subject = ssl_subject + "/emailAddress={}".format(subject["email"]) + + cmd = ["/usr/bin/openssl", "req", "-new", "-newkey", + "rsa:{}".format(keysize), "-days", "365", "-nodes", "-x509", + "-keyout", keyfile, + "-out", certfile, "-subj", ssl_subject] + elif cn: + cmd = ["/usr/bin/openssl", "req", "-new", "-newkey", + "rsa:{}".format(keysize), "-days", "365", "-nodes", "-x509", + "-keyout", keyfile, + "-out", certfile, "-subj", "/CN={}".format(cn)] + + if not cmd: + hookenv.log("No config, subject or cn provided," + "unable to generate self signed SSL certificates") + return False + try: + subprocess.check_call(cmd) + return True + except Exception as e: + print "Execution of openssl command failed:\n{}".format(e) + return False diff --git a/hooks/charmhelpers/contrib/templating/__init__.py b/hooks/charmhelpers/contrib/templating/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/hooks/charmhelpers/contrib/templating/contexts.py b/hooks/charmhelpers/contrib/templating/contexts.py new file mode 100644 index 00000000..b117b2de --- /dev/null +++ b/hooks/charmhelpers/contrib/templating/contexts.py @@ -0,0 +1,73 @@ +# Copyright 2013 Canonical Ltd. +# +# Authors: +# Charm Helpers Developers +"""A helper to create a yaml cache of config with namespaced relation data.""" +import os +import yaml + +import charmhelpers.core.hookenv + + +charm_dir = os.environ.get('CHARM_DIR', '') + + +def juju_state_to_yaml(yaml_path, namespace_separator=':', + allow_hyphens_in_keys=True): + """Update the juju config and state in a yaml file. + + This includes any current relation-get data, and the charm + directory. + + This function was created for the ansible and saltstack + support, as those libraries can use a yaml file to supply + context to templates, but it may be useful generally to + create and update an on-disk cache of all the config, including + previous relation data. + + By default, hyphens are allowed in keys as this is supported + by yaml, but for tools like ansible, hyphens are not valid [1]. + + [1] http://www.ansibleworks.com/docs/playbooks_variables.html#what-makes-a-valid-variable-name + """ + config = charmhelpers.core.hookenv.config() + + # Add the charm_dir which we will need to refer to charm + # file resources etc. + config['charm_dir'] = charm_dir + config['local_unit'] = charmhelpers.core.hookenv.local_unit() + + # Add any relation data prefixed with the relation type. + relation_type = charmhelpers.core.hookenv.relation_type() + if relation_type is not None: + relation_data = charmhelpers.core.hookenv.relation_get() + relation_data = dict( + ("{relation_type}{namespace_separator}{key}".format( + relation_type=relation_type.replace('-', '_'), + key=key, + namespace_separator=namespace_separator), val) + for key, val in relation_data.items()) + config.update(relation_data) + + # Don't use non-standard tags for unicode which will not + # work when salt uses yaml.load_safe. + yaml.add_representer(unicode, lambda dumper, + value: dumper.represent_scalar( + u'tag:yaml.org,2002:str', value)) + + yaml_dir = os.path.dirname(yaml_path) + if not os.path.exists(yaml_dir): + os.makedirs(yaml_dir) + + if os.path.exists(yaml_path): + with open(yaml_path, "r") as existing_vars_file: + existing_vars = yaml.load(existing_vars_file.read()) + else: + existing_vars = {} + + if not allow_hyphens_in_keys: + config = dict( + (key.replace('-', '_'), val) for key, val in config.items()) + existing_vars.update(config) + with open(yaml_path, "w+") as fp: + fp.write(yaml.dump(existing_vars)) diff --git a/hooks/charmhelpers/contrib/templating/pyformat.py b/hooks/charmhelpers/contrib/templating/pyformat.py new file mode 100644 index 00000000..baaf98e7 --- /dev/null +++ b/hooks/charmhelpers/contrib/templating/pyformat.py @@ -0,0 +1,13 @@ +''' +Templating using standard Python str.format() method. +''' + +from charmhelpers.core import hookenv + + +def render(template, extra={}, **kwargs): + """Return the template rendered using Python's str.format().""" + context = hookenv.execution_environment() + context.update(extra) + context.update(kwargs) + return template.format(**context) diff --git a/hooks/charmhelpers/payload/archive.py b/hooks/charmhelpers/payload/archive.py new file mode 100644 index 00000000..de03e1b7 --- /dev/null +++ b/hooks/charmhelpers/payload/archive.py @@ -0,0 +1,57 @@ +import os +import tarfile +import zipfile +from charmhelpers.core import ( + host, + hookenv, +) + + +class ArchiveError(Exception): + pass + + +def get_archive_handler(archive_name): + if os.path.isfile(archive_name): + if tarfile.is_tarfile(archive_name): + return extract_tarfile + elif zipfile.is_zipfile(archive_name): + return extract_zipfile + else: + # look at the file name + for ext in ('.tar', '.tar.gz', '.tgz', 'tar.bz2', '.tbz2', '.tbz'): + if archive_name.endswith(ext): + return extract_tarfile + for ext in ('.zip', '.jar'): + if archive_name.endswith(ext): + return extract_zipfile + + +def archive_dest_default(archive_name): + archive_file = os.path.basename(archive_name) + return os.path.join(hookenv.charm_dir(), "archives", archive_file) + + +def extract(archive_name, destpath=None): + handler = get_archive_handler(archive_name) + if handler: + if not destpath: + destpath = archive_dest_default(archive_name) + if not os.path.isdir(destpath): + host.mkdir(destpath) + handler(archive_name, destpath) + return destpath + else: + raise ArchiveError("No handler for archive") + + +def extract_tarfile(archive_name, destpath): + "Unpack a tar archive, optionally compressed" + archive = tarfile.open(archive_name) + archive.extractall(destpath) + + +def extract_zipfile(archive_name, destpath): + "Unpack a zip file" + archive = zipfile.ZipFile(archive_name) + archive.extractall(destpath) From 80b9c4e41f83a19c1780a33d38b08a50e8164c23 Mon Sep 17 00:00:00 2001 From: "yolanda.robla@canonical.com" <> Date: Wed, 8 Jan 2014 12:01:59 +0100 Subject: [PATCH 05/19] refreshed charmhelpers --- hooks/charmhelpers/contrib/openstack/context.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index dec1bfe0..ad5fc93b 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -182,7 +182,7 @@ class AMQPContext(OSContextGenerator): # Sufficient information found = break out! break # Used for active/active rabbitmq >= grizzly - if 'clustered' not in ctxt and len(related_units(rid))>0: + if 'clustered' not in ctxt and len(related_units(rid))>1: rabbitmq_hosts = [] for unit in related_units(rid): rabbitmq_hosts.append(relation_get('private-address', From 5555644e290a2346dc6022db0171188d04ad3a32 Mon Sep 17 00:00:00 2001 From: "yolanda.robla@canonical.com" <> Date: Thu, 23 Jan 2014 14:29:17 +0100 Subject: [PATCH 06/19] added amqp-relation-departed hook to refresh hosts --- hooks/amqp-relation-departed | 1 + hooks/cinder_hooks.py | 8 ++++++++ revision | 2 +- 3 files changed, 10 insertions(+), 1 deletion(-) create mode 120000 hooks/amqp-relation-departed diff --git a/hooks/amqp-relation-departed b/hooks/amqp-relation-departed new file mode 120000 index 00000000..6dcd0084 --- /dev/null +++ b/hooks/amqp-relation-departed @@ -0,0 +1 @@ +cinder_hooks.py \ No newline at end of file diff --git a/hooks/cinder_hooks.py b/hooks/cinder_hooks.py index 1ceb504a..1b430e3d 100755 --- a/hooks/cinder_hooks.py +++ b/hooks/cinder_hooks.py @@ -122,6 +122,14 @@ def amqp_changed(): return CONFIGS.write(CINDER_CONF) +@hooks.hook('amqp-relation-departed') +@restart_on_change(restart_map()) +def amqp_departed(): + if 'amqp' not in CONFIGS.complete_contexts(): + juju_log('amqp relation incomplete. Peer not ready?') + return + CONFIGS.write(CINDER_CONF) + @hooks.hook('identity-service-relation-joined') def identity_joined(rid=None): diff --git a/revision b/revision index 94361d49..6a4573e8 100644 --- a/revision +++ b/revision @@ -1 +1 @@ -132 +133 From 6f957220e9238db64bc09a0620a41b8cb3f4c7fb Mon Sep 17 00:00:00 2001 From: "yolanda.robla@canonical.com" <> Date: Mon, 27 Jan 2014 10:03:39 +0100 Subject: [PATCH 07/19] enabling ha queues if needed --- hooks/charmhelpers/contrib/openstack/context.py | 2 ++ templates/grizzly/cinder.conf | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index ad5fc93b..42f72b61 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -177,6 +177,8 @@ class AMQPContext(OSContextGenerator): 'rabbitmq_password': relation_get('password', rid=rid, unit=unit), 'rabbitmq_virtual_host': vhost, + 'rabbitmq_ha_queues': relation_get('ha_queues', rid=rid, + unit=unit), }) if context_complete(ctxt): # Sufficient information found = break out! diff --git a/templates/grizzly/cinder.conf b/templates/grizzly/cinder.conf index 35988f90..4ce8228a 100644 --- a/templates/grizzly/cinder.conf +++ b/templates/grizzly/cinder.conf @@ -25,6 +25,11 @@ rabbit_password = {{ rabbitmq_password }} rabbit_virtual_host = {{ rabbitmq_virtual_host }} {% if rabbitmq_hosts -%} rabbit_hosts = {{ rabbitmq_hosts }} +{% if rabbitmq_ha_queues -%} +rabbit_ha_queues = true +{% else %} +rabbit_ha_queues = false +{% endif %} {% else %} rabbit_host = {{ rabbitmq_host }} {% endif -%} From e5ee544d28880d89fe95dbf7dc41a224c981b06d Mon Sep 17 00:00:00 2001 From: "yolanda.robla@canonical.com" <> Date: Mon, 27 Jan 2014 18:09:12 +0100 Subject: [PATCH 08/19] resyncing charmehlpers --- .../charmhelpers/contrib/openstack/context.py | 10 ++++---- .../templates/openstack_https_frontend.conf | 24 ++++++++++++++++++- hooks/charmhelpers/contrib/openstack/utils.py | 14 +++++++---- hooks/charmhelpers/core/hookenv.py | 6 +++++ 4 files changed, 44 insertions(+), 10 deletions(-) mode change 120000 => 100644 hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend.conf diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 42f72b61..6331a92e 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -23,7 +23,6 @@ from charmhelpers.core.hookenv import ( unit_get, unit_private_ip, ERROR, - WARNING, ) from charmhelpers.contrib.hahelpers.cluster import ( @@ -177,14 +176,12 @@ class AMQPContext(OSContextGenerator): 'rabbitmq_password': relation_get('password', rid=rid, unit=unit), 'rabbitmq_virtual_host': vhost, - 'rabbitmq_ha_queues': relation_get('ha_queues', rid=rid, - unit=unit), }) if context_complete(ctxt): # Sufficient information found = break out! break # Used for active/active rabbitmq >= grizzly - if 'clustered' not in ctxt and len(related_units(rid))>1: + if 'clustered' not in ctxt and len(related_units(rid)) > 1: rabbitmq_hosts = [] for unit in related_units(rid): rabbitmq_hosts.append(relation_get('private-address', @@ -290,6 +287,7 @@ class ImageServiceContext(OSContextGenerator): class ApacheSSLContext(OSContextGenerator): + """ Generates a context for an apache vhost configuration that configures HTTPS reverse proxying for one or many endpoints. Generated context @@ -437,6 +435,7 @@ class NeutronContext(object): class OSConfigFlagContext(OSContextGenerator): + """ Responsible for adding user-defined config-flags in charm config to a template context. @@ -445,6 +444,7 @@ class OSConfigFlagContext(OSContextGenerator): key=value pairs and some Openstack config files support comma-separated lists as values. """ + def __call__(self): config_flags = config('config-flags') if not config_flags: @@ -489,6 +489,7 @@ class OSConfigFlagContext(OSContextGenerator): class SubordinateConfigContext(OSContextGenerator): + """ Responsible for inspecting relations to subordinates that may be exporting required config via a json blob. @@ -529,6 +530,7 @@ class SubordinateConfigContext(OSContextGenerator): } """ + def __init__(self, service, config_file, interface): """ :param service : Service name key to query in any subordinate diff --git a/hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend.conf b/hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend.conf deleted file mode 120000 index 9a2f6f2b..00000000 --- a/hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend.conf +++ /dev/null @@ -1 +0,0 @@ -openstack_https_frontend \ No newline at end of file diff --git a/hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend.conf b/hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend.conf new file mode 100644 index 00000000..e02dc751 --- /dev/null +++ b/hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend.conf @@ -0,0 +1,23 @@ +{% if endpoints -%} +{% for ext, int in endpoints -%} +Listen {{ ext }} +NameVirtualHost *:{{ ext }} + + ServerName {{ private_address }} + SSLEngine on + SSLCertificateFile /etc/apache2/ssl/{{ namespace }}/cert + SSLCertificateKeyFile /etc/apache2/ssl/{{ namespace }}/key + ProxyPass / http://localhost:{{ int }}/ + ProxyPassReverse / http://localhost:{{ int }}/ + ProxyPreserveHost on + + + Order deny,allow + Allow from all + + + Order allow,deny + Allow from all + +{% endfor -%} +{% endif -%} diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 67a23378..56d04245 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -415,7 +415,7 @@ def get_host_ip(hostname): return ns_query(hostname) -def get_hostname(address): +def get_hostname(address, fqdn=True): """ Resolves hostname for given IP, or returns the input if it is already a hostname. @@ -434,7 +434,11 @@ def get_hostname(address): if not result: return None - # strip trailing . - if result.endswith('.'): - return result[:-1] - return result + if fqdn: + # strip trailing . + if result.endswith('.'): + return result[:-1] + else: + return result + else: + return result.split('.')[0] diff --git a/hooks/charmhelpers/core/hookenv.py b/hooks/charmhelpers/core/hookenv.py index bb196dfa..505c202d 100644 --- a/hooks/charmhelpers/core/hookenv.py +++ b/hooks/charmhelpers/core/hookenv.py @@ -8,6 +8,7 @@ import os import json import yaml import subprocess +import sys import UserDict from subprocess import CalledProcessError @@ -149,6 +150,11 @@ def service_name(): return local_unit().split('/')[0] +def hook_name(): + """The name of the currently executing hook""" + return os.path.basename(sys.argv[0]) + + @cached def config(scope=None): """Juju charm configuration""" From a55b5387dbd8e6aa69315b8a430363b1cf520afd Mon Sep 17 00:00:00 2001 From: "yolanda.robla@canonical.com" <> Date: Tue, 28 Jan 2014 09:46:07 +0100 Subject: [PATCH 09/19] setting durable queues to false when ha queues --- templates/grizzly/cinder.conf | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/templates/grizzly/cinder.conf b/templates/grizzly/cinder.conf index 4ce8228a..af4bd528 100644 --- a/templates/grizzly/cinder.conf +++ b/templates/grizzly/cinder.conf @@ -27,8 +27,7 @@ rabbit_virtual_host = {{ rabbitmq_virtual_host }} rabbit_hosts = {{ rabbitmq_hosts }} {% if rabbitmq_ha_queues -%} rabbit_ha_queues = true -{% else %} -rabbit_ha_queues = false +rabbit_durable_queues = false {% endif %} {% else %} rabbit_host = {{ rabbitmq_host }} From bd565b30ce6416d7f18a79f55bb014ba1d065078 Mon Sep 17 00:00:00 2001 From: "yolanda.robla@canonical.com" <> Date: Thu, 30 Jan 2014 11:24:17 +0100 Subject: [PATCH 10/19] resync charmhelpers --- hooks/charmhelpers/cli/README.rst | 57 -- hooks/charmhelpers/cli/__init__.py | 147 ----- hooks/charmhelpers/cli/commands.py | 2 - hooks/charmhelpers/cli/host.py | 15 - .../charmhelpers/contrib/ansible/__init__.py | 165 ----- .../charmhelpers/contrib/charmhelpers/IMPORT | 4 - .../contrib/charmhelpers/__init__.py | 184 ------ .../charmhelpers/contrib/charmsupport/IMPORT | 14 - .../contrib/charmsupport/__init__.py | 0 .../charmhelpers/contrib/charmsupport/nrpe.py | 216 ------- .../contrib/charmsupport/volumes.py | 156 ----- hooks/charmhelpers/contrib/jujugui/IMPORT | 4 - .../charmhelpers/contrib/jujugui/__init__.py | 0 hooks/charmhelpers/contrib/jujugui/utils.py | 602 ------------------ .../charmhelpers/contrib/network/__init__.py | 0 .../contrib/network/ovs/__init__.py | 75 --- .../contrib/saltstack/__init__.py | 102 --- hooks/charmhelpers/contrib/ssl/__init__.py | 78 --- .../contrib/templating/__init__.py | 0 .../contrib/templating/contexts.py | 73 --- .../contrib/templating/pyformat.py | 13 - hooks/charmhelpers/payload/archive.py | 57 -- revision | 2 +- 23 files changed, 1 insertion(+), 1965 deletions(-) delete mode 100644 hooks/charmhelpers/cli/README.rst delete mode 100644 hooks/charmhelpers/cli/__init__.py delete mode 100644 hooks/charmhelpers/cli/commands.py delete mode 100644 hooks/charmhelpers/cli/host.py delete mode 100644 hooks/charmhelpers/contrib/ansible/__init__.py delete mode 100644 hooks/charmhelpers/contrib/charmhelpers/IMPORT delete mode 100644 hooks/charmhelpers/contrib/charmhelpers/__init__.py delete mode 100644 hooks/charmhelpers/contrib/charmsupport/IMPORT delete mode 100644 hooks/charmhelpers/contrib/charmsupport/__init__.py delete mode 100644 hooks/charmhelpers/contrib/charmsupport/nrpe.py delete mode 100644 hooks/charmhelpers/contrib/charmsupport/volumes.py delete mode 100644 hooks/charmhelpers/contrib/jujugui/IMPORT delete mode 100644 hooks/charmhelpers/contrib/jujugui/__init__.py delete mode 100644 hooks/charmhelpers/contrib/jujugui/utils.py delete mode 100644 hooks/charmhelpers/contrib/network/__init__.py delete mode 100644 hooks/charmhelpers/contrib/network/ovs/__init__.py delete mode 100644 hooks/charmhelpers/contrib/saltstack/__init__.py delete mode 100644 hooks/charmhelpers/contrib/ssl/__init__.py delete mode 100644 hooks/charmhelpers/contrib/templating/__init__.py delete mode 100644 hooks/charmhelpers/contrib/templating/contexts.py delete mode 100644 hooks/charmhelpers/contrib/templating/pyformat.py delete mode 100644 hooks/charmhelpers/payload/archive.py diff --git a/hooks/charmhelpers/cli/README.rst b/hooks/charmhelpers/cli/README.rst deleted file mode 100644 index f7901c09..00000000 --- a/hooks/charmhelpers/cli/README.rst +++ /dev/null @@ -1,57 +0,0 @@ -========== -Commandant -========== - ------------------------------------------------------ -Automatic command-line interfaces to Python functions ------------------------------------------------------ - -One of the benefits of ``libvirt`` is the uniformity of the interface: the C API (as well as the bindings in other languages) is a set of functions that accept parameters that are nearly identical to the command-line arguments. If you run ``virsh``, you get an interactive command prompt that supports all of the same commands that your shell scripts use as ``virsh`` subcommands. - -Command execution and stdio manipulation is the greatest common factor across all development systems in the POSIX environment. By exposing your functions as commands that manipulate streams of text, you can make life easier for all the Ruby and Erlang and Go programmers in your life. - -Goals -===== - -* Single decorator to expose a function as a command. - * now two decorators - one "automatic" and one that allows authors to manipulate the arguments for fine-grained control.(MW) -* Automatic analysis of function signature through ``inspect.getargspec()`` -* Command argument parser built automatically with ``argparse`` -* Interactive interpreter loop object made with ``Cmd`` -* Options to output structured return value data via ``pprint``, ``yaml`` or ``json`` dumps. - -Other Important Features that need writing ------------------------------------------- - -* Help and Usage documentation can be automatically generated, but it will be important to let users override this behaviour -* The decorator should allow specifying further parameters to the parser's add_argument() calls, to specify types or to make arguments behave as boolean flags, etc. - - Filename arguments are important, as good practice is for functions to accept file objects as parameters. - - choices arguments help to limit bad input before the function is called -* Some automatic behaviour could make for better defaults, once the user can override them. - - We could automatically detect arguments that default to False or True, and automatically support --no-foo for foo=True. - - We could automatically support hyphens as alternates for underscores - - Arguments defaulting to sequence types could support the ``append`` action. - - ------------------------------------------------------ -Implementing subcommands ------------------------------------------------------ - -(WIP) - -So as to avoid dependencies on the cli module, subcommands should be defined separately from their implementations. The recommmendation would be to place definitions into separate modules near the implementations which they expose. - -Some examples:: - - from charmhelpers.cli import CommandLine - from charmhelpers.payload import execd - from charmhelpers.foo import bar - - cli = CommandLine() - - cli.subcommand(execd.execd_run) - - @cli.subcommand_builder("bar", help="Bar baz qux") - def barcmd_builder(subparser): - subparser.add_argument('argument1', help="yackety") - return bar diff --git a/hooks/charmhelpers/cli/__init__.py b/hooks/charmhelpers/cli/__init__.py deleted file mode 100644 index def53983..00000000 --- a/hooks/charmhelpers/cli/__init__.py +++ /dev/null @@ -1,147 +0,0 @@ -import inspect -import itertools -import argparse -import sys - - -class OutputFormatter(object): - def __init__(self, outfile=sys.stdout): - self.formats = ( - "raw", - "json", - "py", - "yaml", - "csv", - "tab", - ) - self.outfile = outfile - - def add_arguments(self, argument_parser): - formatgroup = argument_parser.add_mutually_exclusive_group() - choices = self.supported_formats - formatgroup.add_argument("--format", metavar='FMT', - help="Select output format for returned data, " - "where FMT is one of: {}".format(choices), - choices=choices, default='raw') - for fmt in self.formats: - fmtfunc = getattr(self, fmt) - formatgroup.add_argument("-{}".format(fmt[0]), - "--{}".format(fmt), action='store_const', - const=fmt, dest='format', - help=fmtfunc.__doc__) - - @property - def supported_formats(self): - return self.formats - - def raw(self, output): - """Output data as raw string (default)""" - self.outfile.write(str(output)) - - def py(self, output): - """Output data as a nicely-formatted python data structure""" - import pprint - pprint.pprint(output, stream=self.outfile) - - def json(self, output): - """Output data in JSON format""" - import json - json.dump(output, self.outfile) - - def yaml(self, output): - """Output data in YAML format""" - import yaml - yaml.safe_dump(output, self.outfile) - - def csv(self, output): - """Output data as excel-compatible CSV""" - import csv - csvwriter = csv.writer(self.outfile) - csvwriter.writerows(output) - - def tab(self, output): - """Output data in excel-compatible tab-delimited format""" - import csv - csvwriter = csv.writer(self.outfile, dialect=csv.excel_tab) - csvwriter.writerows(output) - - def format_output(self, output, fmt='raw'): - fmtfunc = getattr(self, fmt) - fmtfunc(output) - - -class CommandLine(object): - argument_parser = None - subparsers = None - formatter = None - - def __init__(self): - if not self.argument_parser: - self.argument_parser = argparse.ArgumentParser(description='Perform common charm tasks') - if not self.formatter: - self.formatter = OutputFormatter() - self.formatter.add_arguments(self.argument_parser) - if not self.subparsers: - self.subparsers = self.argument_parser.add_subparsers(help='Commands') - - def subcommand(self, command_name=None): - """ - Decorate a function as a subcommand. Use its arguments as the - command-line arguments""" - def wrapper(decorated): - cmd_name = command_name or decorated.__name__ - subparser = self.subparsers.add_parser(cmd_name, - description=decorated.__doc__) - for args, kwargs in describe_arguments(decorated): - subparser.add_argument(*args, **kwargs) - subparser.set_defaults(func=decorated) - return decorated - return wrapper - - def subcommand_builder(self, command_name, description=None): - """ - Decorate a function that builds a subcommand. Builders should accept a - single argument (the subparser instance) and return the function to be - run as the command.""" - def wrapper(decorated): - subparser = self.subparsers.add_parser(command_name) - func = decorated(subparser) - subparser.set_defaults(func=func) - subparser.description = description or func.__doc__ - return wrapper - - def run(self): - "Run cli, processing arguments and executing subcommands." - arguments = self.argument_parser.parse_args() - argspec = inspect.getargspec(arguments.func) - vargs = [] - kwargs = {} - if argspec.varargs: - vargs = getattr(arguments, argspec.varargs) - for arg in argspec.args: - kwargs[arg] = getattr(arguments, arg) - self.formatter.format_output(arguments.func(*vargs, **kwargs), arguments.format) - - -cmdline = CommandLine() - - -def describe_arguments(func): - """ - Analyze a function's signature and return a data structure suitable for - passing in as arguments to an argparse parser's add_argument() method.""" - - argspec = inspect.getargspec(func) - # we should probably raise an exception somewhere if func includes **kwargs - if argspec.defaults: - positional_args = argspec.args[:-len(argspec.defaults)] - keyword_names = argspec.args[-len(argspec.defaults):] - for arg, default in itertools.izip(keyword_names, argspec.defaults): - yield ('--{}'.format(arg),), {'default': default} - else: - positional_args = argspec.args - - for arg in positional_args: - yield (arg,), {} - if argspec.varargs: - yield (argspec.varargs,), {'nargs': '*'} diff --git a/hooks/charmhelpers/cli/commands.py b/hooks/charmhelpers/cli/commands.py deleted file mode 100644 index 5aa79705..00000000 --- a/hooks/charmhelpers/cli/commands.py +++ /dev/null @@ -1,2 +0,0 @@ -from . import CommandLine -import host diff --git a/hooks/charmhelpers/cli/host.py b/hooks/charmhelpers/cli/host.py deleted file mode 100644 index bada5244..00000000 --- a/hooks/charmhelpers/cli/host.py +++ /dev/null @@ -1,15 +0,0 @@ -from . import cmdline -from charmhelpers.core import host - - -@cmdline.subcommand() -def mounts(): - "List mounts" - return host.mounts() - - -@cmdline.subcommand_builder('service', description="Control system services") -def service(subparser): - subparser.add_argument("action", help="The action to perform (start, stop, etc...)") - subparser.add_argument("service_name", help="Name of the service to control") - return host.service diff --git a/hooks/charmhelpers/contrib/ansible/__init__.py b/hooks/charmhelpers/contrib/ansible/__init__.py deleted file mode 100644 index f72bae36..00000000 --- a/hooks/charmhelpers/contrib/ansible/__init__.py +++ /dev/null @@ -1,165 +0,0 @@ -# Copyright 2013 Canonical Ltd. -# -# Authors: -# Charm Helpers Developers -"""Charm Helpers ansible - declare the state of your machines. - -This helper enables you to declare your machine state, rather than -program it procedurally (and have to test each change to your procedures). -Your install hook can be as simple as: - -{{{ -import charmhelpers.contrib.ansible - - -def install(): - charmhelpers.contrib.ansible.install_ansible_support() - charmhelpers.contrib.ansible.apply_playbook('playbooks/install.yaml') -}}} - -and won't need to change (nor will its tests) when you change the machine -state. - -All of your juju config and relation-data are available as template -variables within your playbooks and templates. An install playbook looks -something like: - -{{{ ---- -- hosts: localhost - user: root - - tasks: - - name: Add private repositories. - template: - src: ../templates/private-repositories.list.jinja2 - dest: /etc/apt/sources.list.d/private.list - - - name: Update the cache. - apt: update_cache=yes - - - name: Install dependencies. - apt: pkg={{ item }} - with_items: - - python-mimeparse - - python-webob - - sunburnt - - - name: Setup groups. - group: name={{ item.name }} gid={{ item.gid }} - with_items: - - { name: 'deploy_user', gid: 1800 } - - { name: 'service_user', gid: 1500 } - - ... -}}} - -Read more online about playbooks[1] and standard ansible modules[2]. - -[1] http://www.ansibleworks.com/docs/playbooks.html -[2] http://www.ansibleworks.com/docs/modules.html -""" -import os -import subprocess - -import charmhelpers.contrib.templating.contexts -import charmhelpers.core.host -import charmhelpers.core.hookenv -import charmhelpers.fetch - - -charm_dir = os.environ.get('CHARM_DIR', '') -ansible_hosts_path = '/etc/ansible/hosts' -# Ansible will automatically include any vars in the following -# file in its inventory when run locally. -ansible_vars_path = '/etc/ansible/host_vars/localhost' - - -def install_ansible_support(from_ppa=True): - """Installs the ansible package. - - By default it is installed from the PPA [1] linked from - the ansible website [2]. - - [1] https://launchpad.net/~rquillo/+archive/ansible - [2] http://www.ansibleworks.com/docs/gettingstarted.html#ubuntu-and-debian - - If from_ppa is false, you must ensure that the package is available - from a configured repository. - """ - if from_ppa: - charmhelpers.fetch.add_source('ppa:rquillo/ansible') - charmhelpers.fetch.apt_update(fatal=True) - charmhelpers.fetch.apt_install('ansible') - with open(ansible_hosts_path, 'w+') as hosts_file: - hosts_file.write('localhost ansible_connection=local') - - -def apply_playbook(playbook, tags=None): - tags = tags or [] - tags = ",".join(tags) - charmhelpers.contrib.templating.contexts.juju_state_to_yaml( - ansible_vars_path, namespace_separator='__', - allow_hyphens_in_keys=False) - call = [ - 'ansible-playbook', - '-c', - 'local', - playbook, - ] - if tags: - call.extend(['--tags', '{}'.format(tags)]) - subprocess.check_call(call) - - -class AnsibleHooks(charmhelpers.core.hookenv.Hooks): - """Run a playbook with the hook-name as the tag. - - This helper builds on the standard hookenv.Hooks helper, - but additionally runs the playbook with the hook-name specified - using --tags (ie. running all the tasks tagged with the hook-name). - - Example: - hooks = AnsibleHooks(playbook_path='playbooks/my_machine_state.yaml') - - # All the tasks within my_machine_state.yaml tagged with 'install' - # will be run automatically after do_custom_work() - @hooks.hook() - def install(): - do_custom_work() - - # For most of your hooks, you won't need to do anything other - # than run the tagged tasks for the hook: - @hooks.hook('config-changed', 'start', 'stop') - def just_use_playbook(): - pass - - # As a convenience, you can avoid the above noop function by specifying - # the hooks which are handled by ansible-only and they'll be registered - # for you: - # hooks = AnsibleHooks( - # 'playbooks/my_machine_state.yaml', - # default_hooks=['config-changed', 'start', 'stop']) - - if __name__ == "__main__": - # execute a hook based on the name the program is called by - hooks.execute(sys.argv) - """ - - def __init__(self, playbook_path, default_hooks=None): - """Register any hooks handled by ansible.""" - super(AnsibleHooks, self).__init__() - - self.playbook_path = playbook_path - - default_hooks = default_hooks or [] - noop = lambda *args, **kwargs: None - for hook in default_hooks: - self.register(hook, noop) - - def execute(self, args): - """Execute the hook followed by the playbook using the hook as tag.""" - super(AnsibleHooks, self).execute(args) - hook_name = os.path.basename(args[0]) - charmhelpers.contrib.ansible.apply_playbook( - self.playbook_path, tags=[hook_name]) diff --git a/hooks/charmhelpers/contrib/charmhelpers/IMPORT b/hooks/charmhelpers/contrib/charmhelpers/IMPORT deleted file mode 100644 index d41cb041..00000000 --- a/hooks/charmhelpers/contrib/charmhelpers/IMPORT +++ /dev/null @@ -1,4 +0,0 @@ -Source lp:charm-tools/trunk - -charm-tools/helpers/python/charmhelpers/__init__.py -> charmhelpers/charmhelpers/contrib/charmhelpers/__init__.py -charm-tools/helpers/python/charmhelpers/tests/test_charmhelpers.py -> charmhelpers/tests/contrib/charmhelpers/test_charmhelpers.py diff --git a/hooks/charmhelpers/contrib/charmhelpers/__init__.py b/hooks/charmhelpers/contrib/charmhelpers/__init__.py deleted file mode 100644 index b08f33d2..00000000 --- a/hooks/charmhelpers/contrib/charmhelpers/__init__.py +++ /dev/null @@ -1,184 +0,0 @@ -# Copyright 2012 Canonical Ltd. This software is licensed under the -# GNU Affero General Public License version 3 (see the file LICENSE). - -import warnings -warnings.warn("contrib.charmhelpers is deprecated", DeprecationWarning) - -"""Helper functions for writing Juju charms in Python.""" - -__metaclass__ = type -__all__ = [ - #'get_config', # core.hookenv.config() - #'log', # core.hookenv.log() - #'log_entry', # core.hookenv.log() - #'log_exit', # core.hookenv.log() - #'relation_get', # core.hookenv.relation_get() - #'relation_set', # core.hookenv.relation_set() - #'relation_ids', # core.hookenv.relation_ids() - #'relation_list', # core.hookenv.relation_units() - #'config_get', # core.hookenv.config() - #'unit_get', # core.hookenv.unit_get() - #'open_port', # core.hookenv.open_port() - #'close_port', # core.hookenv.close_port() - #'service_control', # core.host.service() - 'unit_info', # client-side, NOT IMPLEMENTED - 'wait_for_machine', # client-side, NOT IMPLEMENTED - 'wait_for_page_contents', # client-side, NOT IMPLEMENTED - 'wait_for_relation', # client-side, NOT IMPLEMENTED - 'wait_for_unit', # client-side, NOT IMPLEMENTED -] - -import operator -from shelltoolbox import ( - command, -) -import tempfile -import time -import urllib2 -import yaml - -SLEEP_AMOUNT = 0.1 -# We create a juju_status Command here because it makes testing much, -# much easier. -juju_status = lambda: command('juju')('status') - -# re-implemented as charmhelpers.fetch.configure_sources() -#def configure_source(update=False): -# source = config_get('source') -# if ((source.startswith('ppa:') or -# source.startswith('cloud:') or -# source.startswith('http:'))): -# run('add-apt-repository', source) -# if source.startswith("http:"): -# run('apt-key', 'import', config_get('key')) -# if update: -# run('apt-get', 'update') - - -# DEPRECATED: client-side only -def make_charm_config_file(charm_config): - charm_config_file = tempfile.NamedTemporaryFile() - charm_config_file.write(yaml.dump(charm_config)) - charm_config_file.flush() - # The NamedTemporaryFile instance is returned instead of just the name - # because we want to take advantage of garbage collection-triggered - # deletion of the temp file when it goes out of scope in the caller. - return charm_config_file - - -# DEPRECATED: client-side only -def unit_info(service_name, item_name, data=None, unit=None): - if data is None: - data = yaml.safe_load(juju_status()) - service = data['services'].get(service_name) - if service is None: - # XXX 2012-02-08 gmb: - # This allows us to cope with the race condition that we - # have between deploying a service and having it come up in - # `juju status`. We could probably do with cleaning it up so - # that it fails a bit more noisily after a while. - return '' - units = service['units'] - if unit is not None: - item = units[unit][item_name] - else: - # It might seem odd to sort the units here, but we do it to - # ensure that when no unit is specified, the first unit for the - # service (or at least the one with the lowest number) is the - # one whose data gets returned. - sorted_unit_names = sorted(units.keys()) - item = units[sorted_unit_names[0]][item_name] - return item - - -# DEPRECATED: client-side only -def get_machine_data(): - return yaml.safe_load(juju_status())['machines'] - - -# DEPRECATED: client-side only -def wait_for_machine(num_machines=1, timeout=300): - """Wait `timeout` seconds for `num_machines` machines to come up. - - This wait_for... function can be called by other wait_for functions - whose timeouts might be too short in situations where only a bare - Juju setup has been bootstrapped. - - :return: A tuple of (num_machines, time_taken). This is used for - testing. - """ - # You may think this is a hack, and you'd be right. The easiest way - # to tell what environment we're working in (LXC vs EC2) is to check - # the dns-name of the first machine. If it's localhost we're in LXC - # and we can just return here. - if get_machine_data()[0]['dns-name'] == 'localhost': - return 1, 0 - start_time = time.time() - while True: - # Drop the first machine, since it's the Zookeeper and that's - # not a machine that we need to wait for. This will only work - # for EC2 environments, which is why we return early above if - # we're in LXC. - machine_data = get_machine_data() - non_zookeeper_machines = [ - machine_data[key] for key in machine_data.keys()[1:]] - if len(non_zookeeper_machines) >= num_machines: - all_machines_running = True - for machine in non_zookeeper_machines: - if machine.get('instance-state') != 'running': - all_machines_running = False - break - if all_machines_running: - break - if time.time() - start_time >= timeout: - raise RuntimeError('timeout waiting for service to start') - time.sleep(SLEEP_AMOUNT) - return num_machines, time.time() - start_time - - -# DEPRECATED: client-side only -def wait_for_unit(service_name, timeout=480): - """Wait `timeout` seconds for a given service name to come up.""" - wait_for_machine(num_machines=1) - start_time = time.time() - while True: - state = unit_info(service_name, 'agent-state') - if 'error' in state or state == 'started': - break - if time.time() - start_time >= timeout: - raise RuntimeError('timeout waiting for service to start') - time.sleep(SLEEP_AMOUNT) - if state != 'started': - raise RuntimeError('unit did not start, agent-state: ' + state) - - -# DEPRECATED: client-side only -def wait_for_relation(service_name, relation_name, timeout=120): - """Wait `timeout` seconds for a given relation to come up.""" - start_time = time.time() - while True: - relation = unit_info(service_name, 'relations').get(relation_name) - if relation is not None and relation['state'] == 'up': - break - if time.time() - start_time >= timeout: - raise RuntimeError('timeout waiting for relation to be up') - time.sleep(SLEEP_AMOUNT) - - -# DEPRECATED: client-side only -def wait_for_page_contents(url, contents, timeout=120, validate=None): - if validate is None: - validate = operator.contains - start_time = time.time() - while True: - try: - stream = urllib2.urlopen(url) - except (urllib2.HTTPError, urllib2.URLError): - pass - else: - page = stream.read() - if validate(page, contents): - return page - if time.time() - start_time >= timeout: - raise RuntimeError('timeout waiting for contents of ' + url) - time.sleep(SLEEP_AMOUNT) diff --git a/hooks/charmhelpers/contrib/charmsupport/IMPORT b/hooks/charmhelpers/contrib/charmsupport/IMPORT deleted file mode 100644 index 554fddda..00000000 --- a/hooks/charmhelpers/contrib/charmsupport/IMPORT +++ /dev/null @@ -1,14 +0,0 @@ -Source: lp:charmsupport/trunk - -charmsupport/charmsupport/execd.py -> charm-helpers/charmhelpers/contrib/charmsupport/execd.py -charmsupport/charmsupport/hookenv.py -> charm-helpers/charmhelpers/contrib/charmsupport/hookenv.py -charmsupport/charmsupport/host.py -> charm-helpers/charmhelpers/contrib/charmsupport/host.py -charmsupport/charmsupport/nrpe.py -> charm-helpers/charmhelpers/contrib/charmsupport/nrpe.py -charmsupport/charmsupport/volumes.py -> charm-helpers/charmhelpers/contrib/charmsupport/volumes.py - -charmsupport/tests/test_execd.py -> charm-helpers/tests/contrib/charmsupport/test_execd.py -charmsupport/tests/test_hookenv.py -> charm-helpers/tests/contrib/charmsupport/test_hookenv.py -charmsupport/tests/test_host.py -> charm-helpers/tests/contrib/charmsupport/test_host.py -charmsupport/tests/test_nrpe.py -> charm-helpers/tests/contrib/charmsupport/test_nrpe.py - -charmsupport/bin/charmsupport -> charm-helpers/bin/contrib/charmsupport/charmsupport diff --git a/hooks/charmhelpers/contrib/charmsupport/__init__.py b/hooks/charmhelpers/contrib/charmsupport/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/hooks/charmhelpers/contrib/charmsupport/nrpe.py b/hooks/charmhelpers/contrib/charmsupport/nrpe.py deleted file mode 100644 index 26a41eb9..00000000 --- a/hooks/charmhelpers/contrib/charmsupport/nrpe.py +++ /dev/null @@ -1,216 +0,0 @@ -"""Compatibility with the nrpe-external-master charm""" -# Copyright 2012 Canonical Ltd. -# -# Authors: -# Matthew Wedgwood - -import subprocess -import pwd -import grp -import os -import re -import shlex -import yaml - -from charmhelpers.core.hookenv import ( - config, - local_unit, - log, - relation_ids, - relation_set, -) - -from charmhelpers.core.host import service - -# This module adds compatibility with the nrpe-external-master and plain nrpe -# subordinate charms. To use it in your charm: -# -# 1. Update metadata.yaml -# -# provides: -# (...) -# nrpe-external-master: -# interface: nrpe-external-master -# scope: container -# -# and/or -# -# provides: -# (...) -# local-monitors: -# interface: local-monitors -# scope: container - -# -# 2. Add the following to config.yaml -# -# nagios_context: -# default: "juju" -# type: string -# description: | -# Used by the nrpe subordinate charms. -# A string that will be prepended to instance name to set the host name -# in nagios. So for instance the hostname would be something like: -# juju-myservice-0 -# If you're running multiple environments with the same services in them -# this allows you to differentiate between them. -# -# 3. Add custom checks (Nagios plugins) to files/nrpe-external-master -# -# 4. Update your hooks.py with something like this: -# -# from charmsupport.nrpe import NRPE -# (...) -# def update_nrpe_config(): -# nrpe_compat = NRPE() -# nrpe_compat.add_check( -# shortname = "myservice", -# description = "Check MyService", -# check_cmd = "check_http -w 2 -c 10 http://localhost" -# ) -# nrpe_compat.add_check( -# "myservice_other", -# "Check for widget failures", -# check_cmd = "/srv/myapp/scripts/widget_check" -# ) -# nrpe_compat.write() -# -# def config_changed(): -# (...) -# update_nrpe_config() -# -# def nrpe_external_master_relation_changed(): -# update_nrpe_config() -# -# def local_monitors_relation_changed(): -# update_nrpe_config() -# -# 5. ln -s hooks.py nrpe-external-master-relation-changed -# ln -s hooks.py local-monitors-relation-changed - - -class CheckException(Exception): - pass - - -class Check(object): - shortname_re = '[A-Za-z0-9-_]+$' - service_template = (""" -#--------------------------------------------------- -# This file is Juju managed -#--------------------------------------------------- -define service {{ - use active-service - host_name {nagios_hostname} - service_description {nagios_hostname}[{shortname}] """ - """{description} - check_command check_nrpe!{command} - servicegroups {nagios_servicegroup} -}} -""") - - def __init__(self, shortname, description, check_cmd): - super(Check, self).__init__() - # XXX: could be better to calculate this from the service name - if not re.match(self.shortname_re, shortname): - raise CheckException("shortname must match {}".format( - Check.shortname_re)) - self.shortname = shortname - self.command = "check_{}".format(shortname) - # Note: a set of invalid characters is defined by the - # Nagios server config - # The default is: illegal_object_name_chars=`~!$%^&*"|'<>?,()= - self.description = description - self.check_cmd = self._locate_cmd(check_cmd) - - def _locate_cmd(self, check_cmd): - search_path = ( - '/usr/lib/nagios/plugins', - '/usr/local/lib/nagios/plugins', - ) - parts = shlex.split(check_cmd) - for path in search_path: - if os.path.exists(os.path.join(path, parts[0])): - command = os.path.join(path, parts[0]) - if len(parts) > 1: - command += " " + " ".join(parts[1:]) - return command - log('Check command not found: {}'.format(parts[0])) - return '' - - def write(self, nagios_context, hostname): - nrpe_check_file = '/etc/nagios/nrpe.d/{}.cfg'.format( - self.command) - with open(nrpe_check_file, 'w') as nrpe_check_config: - nrpe_check_config.write("# check {}\n".format(self.shortname)) - nrpe_check_config.write("command[{}]={}\n".format( - self.command, self.check_cmd)) - - if not os.path.exists(NRPE.nagios_exportdir): - log('Not writing service config as {} is not accessible'.format( - NRPE.nagios_exportdir)) - else: - self.write_service_config(nagios_context, hostname) - - def write_service_config(self, nagios_context, hostname): - for f in os.listdir(NRPE.nagios_exportdir): - if re.search('.*{}.cfg'.format(self.command), f): - os.remove(os.path.join(NRPE.nagios_exportdir, f)) - - templ_vars = { - 'nagios_hostname': hostname, - 'nagios_servicegroup': nagios_context, - 'description': self.description, - 'shortname': self.shortname, - 'command': self.command, - } - nrpe_service_text = Check.service_template.format(**templ_vars) - nrpe_service_file = '{}/service__{}_{}.cfg'.format( - NRPE.nagios_exportdir, hostname, self.command) - with open(nrpe_service_file, 'w') as nrpe_service_config: - nrpe_service_config.write(str(nrpe_service_text)) - - def run(self): - subprocess.call(self.check_cmd) - - -class NRPE(object): - nagios_logdir = '/var/log/nagios' - nagios_exportdir = '/var/lib/nagios/export' - nrpe_confdir = '/etc/nagios/nrpe.d' - - def __init__(self): - super(NRPE, self).__init__() - self.config = config() - self.nagios_context = self.config['nagios_context'] - self.unit_name = local_unit().replace('/', '-') - self.hostname = "{}-{}".format(self.nagios_context, self.unit_name) - self.checks = [] - - def add_check(self, *args, **kwargs): - self.checks.append(Check(*args, **kwargs)) - - def write(self): - try: - nagios_uid = pwd.getpwnam('nagios').pw_uid - nagios_gid = grp.getgrnam('nagios').gr_gid - except: - log("Nagios user not set up, nrpe checks not updated") - return - - if not os.path.exists(NRPE.nagios_logdir): - os.mkdir(NRPE.nagios_logdir) - os.chown(NRPE.nagios_logdir, nagios_uid, nagios_gid) - - nrpe_monitors = {} - monitors = {"monitors": {"remote": {"nrpe": nrpe_monitors}}} - for nrpecheck in self.checks: - nrpecheck.write(self.nagios_context, self.hostname) - nrpe_monitors[nrpecheck.shortname] = { - "command": nrpecheck.command, - } - - service('restart', 'nagios-nrpe-server') - - for rid in relation_ids("local-monitors"): - relation_set(relation_id=rid, monitors=yaml.dump(monitors)) diff --git a/hooks/charmhelpers/contrib/charmsupport/volumes.py b/hooks/charmhelpers/contrib/charmsupport/volumes.py deleted file mode 100644 index 0f905dff..00000000 --- a/hooks/charmhelpers/contrib/charmsupport/volumes.py +++ /dev/null @@ -1,156 +0,0 @@ -''' -Functions for managing volumes in juju units. One volume is supported per unit. -Subordinates may have their own storage, provided it is on its own partition. - -Configuration stanzas: - volume-ephemeral: - type: boolean - default: true - description: > - If false, a volume is mounted as sepecified in "volume-map" - If true, ephemeral storage will be used, meaning that log data - will only exist as long as the machine. YOU HAVE BEEN WARNED. - volume-map: - type: string - default: {} - description: > - YAML map of units to device names, e.g: - "{ rsyslog/0: /dev/vdb, rsyslog/1: /dev/vdb }" - Service units will raise a configure-error if volume-ephemeral - is 'true' and no volume-map value is set. Use 'juju set' to set a - value and 'juju resolved' to complete configuration. - -Usage: - from charmsupport.volumes import configure_volume, VolumeConfigurationError - from charmsupport.hookenv import log, ERROR - def post_mount_hook(): - stop_service('myservice') - def post_mount_hook(): - start_service('myservice') - - if __name__ == '__main__': - try: - configure_volume(before_change=pre_mount_hook, - after_change=post_mount_hook) - except VolumeConfigurationError: - log('Storage could not be configured', ERROR) -''' - -# XXX: Known limitations -# - fstab is neither consulted nor updated - -import os -from charmhelpers.core import hookenv -from charmhelpers.core import host -import yaml - - -MOUNT_BASE = '/srv/juju/volumes' - - -class VolumeConfigurationError(Exception): - '''Volume configuration data is missing or invalid''' - pass - - -def get_config(): - '''Gather and sanity-check volume configuration data''' - volume_config = {} - config = hookenv.config() - - errors = False - - if config.get('volume-ephemeral') in (True, 'True', 'true', 'Yes', 'yes'): - volume_config['ephemeral'] = True - else: - volume_config['ephemeral'] = False - - try: - volume_map = yaml.safe_load(config.get('volume-map', '{}')) - except yaml.YAMLError as e: - hookenv.log("Error parsing YAML volume-map: {}".format(e), - hookenv.ERROR) - errors = True - if volume_map is None: - # probably an empty string - volume_map = {} - elif not isinstance(volume_map, dict): - hookenv.log("Volume-map should be a dictionary, not {}".format( - type(volume_map))) - errors = True - - volume_config['device'] = volume_map.get(os.environ['JUJU_UNIT_NAME']) - if volume_config['device'] and volume_config['ephemeral']: - # asked for ephemeral storage but also defined a volume ID - hookenv.log('A volume is defined for this unit, but ephemeral ' - 'storage was requested', hookenv.ERROR) - errors = True - elif not volume_config['device'] and not volume_config['ephemeral']: - # asked for permanent storage but did not define volume ID - hookenv.log('Ephemeral storage was requested, but there is no volume ' - 'defined for this unit.', hookenv.ERROR) - errors = True - - unit_mount_name = hookenv.local_unit().replace('/', '-') - volume_config['mountpoint'] = os.path.join(MOUNT_BASE, unit_mount_name) - - if errors: - return None - return volume_config - - -def mount_volume(config): - if os.path.exists(config['mountpoint']): - if not os.path.isdir(config['mountpoint']): - hookenv.log('Not a directory: {}'.format(config['mountpoint'])) - raise VolumeConfigurationError() - else: - host.mkdir(config['mountpoint']) - if os.path.ismount(config['mountpoint']): - unmount_volume(config) - if not host.mount(config['device'], config['mountpoint'], persist=True): - raise VolumeConfigurationError() - - -def unmount_volume(config): - if os.path.ismount(config['mountpoint']): - if not host.umount(config['mountpoint'], persist=True): - raise VolumeConfigurationError() - - -def managed_mounts(): - '''List of all mounted managed volumes''' - return filter(lambda mount: mount[0].startswith(MOUNT_BASE), host.mounts()) - - -def configure_volume(before_change=lambda: None, after_change=lambda: None): - '''Set up storage (or don't) according to the charm's volume configuration. - Returns the mount point or "ephemeral". before_change and after_change - are optional functions to be called if the volume configuration changes. - ''' - - config = get_config() - if not config: - hookenv.log('Failed to read volume configuration', hookenv.CRITICAL) - raise VolumeConfigurationError() - - if config['ephemeral']: - if os.path.ismount(config['mountpoint']): - before_change() - unmount_volume(config) - after_change() - return 'ephemeral' - else: - # persistent storage - if os.path.ismount(config['mountpoint']): - mounts = dict(managed_mounts()) - if mounts.get(config['mountpoint']) != config['device']: - before_change() - unmount_volume(config) - mount_volume(config) - after_change() - else: - before_change() - mount_volume(config) - after_change() - return config['mountpoint'] diff --git a/hooks/charmhelpers/contrib/jujugui/IMPORT b/hooks/charmhelpers/contrib/jujugui/IMPORT deleted file mode 100644 index 619a403c..00000000 --- a/hooks/charmhelpers/contrib/jujugui/IMPORT +++ /dev/null @@ -1,4 +0,0 @@ -Source: lp:charms/juju-gui - -juju-gui/hooks/utils.py -> charm-helpers/charmhelpers/contrib/jujugui/utils.py -juju-gui/tests/test_utils.py -> charm-helpers/tests/contrib/jujugui/test_utils.py diff --git a/hooks/charmhelpers/contrib/jujugui/__init__.py b/hooks/charmhelpers/contrib/jujugui/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/hooks/charmhelpers/contrib/jujugui/utils.py b/hooks/charmhelpers/contrib/jujugui/utils.py deleted file mode 100644 index c7d2b89b..00000000 --- a/hooks/charmhelpers/contrib/jujugui/utils.py +++ /dev/null @@ -1,602 +0,0 @@ -"""Juju GUI charm utilities.""" - -__all__ = [ - 'AGENT', - 'APACHE', - 'API_PORT', - 'CURRENT_DIR', - 'HAPROXY', - 'IMPROV', - 'JUJU_DIR', - 'JUJU_GUI_DIR', - 'JUJU_GUI_SITE', - 'JUJU_PEM', - 'WEB_PORT', - 'bzr_checkout', - 'chain', - 'cmd_log', - 'fetch_api', - 'fetch_gui', - 'find_missing_packages', - 'first_path_in_dir', - 'get_api_address', - 'get_npm_cache_archive_url', - 'get_release_file_url', - 'get_staging_dependencies', - 'get_zookeeper_address', - 'legacy_juju', - 'log_hook', - 'merge', - 'parse_source', - 'prime_npm_cache', - 'render_to_file', - 'save_or_create_certificates', - 'setup_apache', - 'setup_gui', - 'start_agent', - 'start_gui', - 'start_improv', - 'write_apache_config', -] - -from contextlib import contextmanager -import errno -import json -import os -import logging -import shutil -from subprocess import CalledProcessError -import tempfile -from urlparse import urlparse - -import apt -import tempita - -from launchpadlib.launchpad import Launchpad -from shelltoolbox import ( - Serializer, - apt_get_install, - command, - environ, - install_extra_repositories, - run, - script_name, - search_file, - su, -) -from charmhelpers.core.host import ( - service_start, -) -from charmhelpers.core.hookenv import ( - log, - config, - unit_get, -) - - -AGENT = 'juju-api-agent' -APACHE = 'apache2' -IMPROV = 'juju-api-improv' -HAPROXY = 'haproxy' - -API_PORT = 8080 -WEB_PORT = 8000 - -CURRENT_DIR = os.getcwd() -JUJU_DIR = os.path.join(CURRENT_DIR, 'juju') -JUJU_GUI_DIR = os.path.join(CURRENT_DIR, 'juju-gui') -JUJU_GUI_SITE = '/etc/apache2/sites-available/juju-gui' -JUJU_GUI_PORTS = '/etc/apache2/ports.conf' -JUJU_PEM = 'juju.includes-private-key.pem' -BUILD_REPOSITORIES = ('ppa:chris-lea/node.js-legacy',) -DEB_BUILD_DEPENDENCIES = ( - 'bzr', 'imagemagick', 'make', 'nodejs', 'npm', -) -DEB_STAGE_DEPENDENCIES = ( - 'zookeeper', -) - - -# Store the configuration from on invocation to the next. -config_json = Serializer('/tmp/config.json') -# Bazaar checkout command. -bzr_checkout = command('bzr', 'co', '--lightweight') -# Whether or not the charm is deployed using juju-core. -# If juju-core has been used to deploy the charm, an agent.conf file must -# be present in the charm parent directory. -legacy_juju = lambda: not os.path.exists( - os.path.join(CURRENT_DIR, '..', 'agent.conf')) - - -def _get_build_dependencies(): - """Install deb dependencies for building.""" - log('Installing build dependencies.') - cmd_log(install_extra_repositories(*BUILD_REPOSITORIES)) - cmd_log(apt_get_install(*DEB_BUILD_DEPENDENCIES)) - - -def get_api_address(unit_dir): - """Return the Juju API address stored in the uniter agent.conf file.""" - import yaml # python-yaml is only installed if juju-core is used. - # XXX 2013-03-27 frankban bug=1161443: - # currently the uniter agent.conf file does not include the API - # address. For now retrieve it from the machine agent file. - base_dir = os.path.abspath(os.path.join(unit_dir, '..')) - for dirname in os.listdir(base_dir): - if dirname.startswith('machine-'): - agent_conf = os.path.join(base_dir, dirname, 'agent.conf') - break - else: - raise IOError('Juju agent configuration file not found.') - contents = yaml.load(open(agent_conf)) - return contents['apiinfo']['addrs'][0] - - -def get_staging_dependencies(): - """Install deb dependencies for the stage (improv) environment.""" - log('Installing stage dependencies.') - cmd_log(apt_get_install(*DEB_STAGE_DEPENDENCIES)) - - -def first_path_in_dir(directory): - """Return the full path of the first file/dir in *directory*.""" - return os.path.join(directory, os.listdir(directory)[0]) - - -def _get_by_attr(collection, attr, value): - """Return the first item in collection having attr == value. - - Return None if the item is not found. - """ - for item in collection: - if getattr(item, attr) == value: - return item - - -def get_release_file_url(project, series_name, release_version): - """Return the URL of the release file hosted in Launchpad. - - The returned URL points to a release file for the given project, series - name and release version. - The argument *project* is a project object as returned by launchpadlib. - The arguments *series_name* and *release_version* are strings. If - *release_version* is None, the URL of the latest release will be returned. - """ - series = _get_by_attr(project.series, 'name', series_name) - if series is None: - raise ValueError('%r: series not found' % series_name) - # Releases are returned by Launchpad in reverse date order. - releases = list(series.releases) - if not releases: - raise ValueError('%r: series does not contain releases' % series_name) - if release_version is not None: - release = _get_by_attr(releases, 'version', release_version) - if release is None: - raise ValueError('%r: release not found' % release_version) - releases = [release] - for release in releases: - for file_ in release.files: - if str(file_).endswith('.tgz'): - return file_.file_link - raise ValueError('%r: file not found' % release_version) - - -def get_zookeeper_address(agent_file_path): - """Retrieve the Zookeeper address contained in the given *agent_file_path*. - - The *agent_file_path* is a path to a file containing a line similar to the - following:: - - env JUJU_ZOOKEEPER="address" - """ - line = search_file('JUJU_ZOOKEEPER', agent_file_path).strip() - return line.split('=')[1].strip('"') - - -@contextmanager -def log_hook(): - """Log when a hook starts and stops its execution. - - Also log to stdout possible CalledProcessError exceptions raised executing - the hook. - """ - script = script_name() - log(">>> Entering {}".format(script)) - try: - yield - except CalledProcessError as err: - log('Exception caught:') - log(err.output) - raise - finally: - log("<<< Exiting {}".format(script)) - - -def parse_source(source): - """Parse the ``juju-gui-source`` option. - - Return a tuple of two elements representing info on how to deploy Juju GUI. - Examples: - - ('stable', None): latest stable release; - - ('stable', '0.1.0'): stable release v0.1.0; - - ('trunk', None): latest trunk release; - - ('trunk', '0.1.0+build.1'): trunk release v0.1.0 bzr revision 1; - - ('branch', 'lp:juju-gui'): release is made from a branch; - - ('url', 'http://example.com/gui'): release from a downloaded file. - """ - if source.startswith('url:'): - source = source[4:] - # Support file paths, including relative paths. - if urlparse(source).scheme == '': - if not source.startswith('/'): - source = os.path.join(os.path.abspath(CURRENT_DIR), source) - source = "file://%s" % source - return 'url', source - if source in ('stable', 'trunk'): - return source, None - if source.startswith('lp:') or source.startswith('http://'): - return 'branch', source - if 'build' in source: - return 'trunk', source - return 'stable', source - - -def render_to_file(template_name, context, destination): - """Render the given *template_name* into *destination* using *context*. - - The tempita template language is used to render contents - (see http://pythonpaste.org/tempita/). - The argument *template_name* is the name or path of the template file: - it may be either a path relative to ``../config`` or an absolute path. - The argument *destination* is a file path. - The argument *context* is a dict-like object. - """ - template_path = os.path.abspath(template_name) - template = tempita.Template.from_filename(template_path) - with open(destination, 'w') as stream: - stream.write(template.substitute(context)) - - -results_log = None - - -def _setupLogging(): - global results_log - if results_log is not None: - return - cfg = config() - logging.basicConfig( - filename=cfg['command-log-file'], - level=logging.INFO, - format="%(asctime)s: %(name)s@%(levelname)s %(message)s") - results_log = logging.getLogger('juju-gui') - - -def cmd_log(results): - global results_log - if not results: - return - if results_log is None: - _setupLogging() - # Since 'results' may be multi-line output, start it on a separate line - # from the logger timestamp, etc. - results_log.info('\n' + results) - - -def start_improv(staging_env, ssl_cert_path, - config_path='/etc/init/juju-api-improv.conf'): - """Start a simulated juju environment using ``improv.py``.""" - log('Setting up staging start up script.') - context = { - 'juju_dir': JUJU_DIR, - 'keys': ssl_cert_path, - 'port': API_PORT, - 'staging_env': staging_env, - } - render_to_file('config/juju-api-improv.conf.template', context, config_path) - log('Starting the staging backend.') - with su('root'): - service_start(IMPROV) - - -def start_agent( - ssl_cert_path, config_path='/etc/init/juju-api-agent.conf', - read_only=False): - """Start the Juju agent and connect to the current environment.""" - # Retrieve the Zookeeper address from the start up script. - unit_dir = os.path.realpath(os.path.join(CURRENT_DIR, '..')) - agent_file = '/etc/init/juju-{0}.conf'.format(os.path.basename(unit_dir)) - zookeeper = get_zookeeper_address(agent_file) - log('Setting up API agent start up script.') - context = { - 'juju_dir': JUJU_DIR, - 'keys': ssl_cert_path, - 'port': API_PORT, - 'zookeeper': zookeeper, - 'read_only': read_only - } - render_to_file('config/juju-api-agent.conf.template', context, config_path) - log('Starting API agent.') - with su('root'): - service_start(AGENT) - - -def start_gui( - console_enabled, login_help, readonly, in_staging, ssl_cert_path, - charmworld_url, serve_tests, haproxy_path='/etc/haproxy/haproxy.cfg', - config_js_path=None, secure=True, sandbox=False): - """Set up and start the Juju GUI server.""" - with su('root'): - run('chown', '-R', 'ubuntu:', JUJU_GUI_DIR) - # XXX 2013-02-05 frankban bug=1116320: - # External insecure resources are still loaded when testing in the - # debug environment. For now, switch to the production environment if - # the charm is configured to serve tests. - if in_staging and not serve_tests: - build_dirname = 'build-debug' - else: - build_dirname = 'build-prod' - build_dir = os.path.join(JUJU_GUI_DIR, build_dirname) - log('Generating the Juju GUI configuration file.') - is_legacy_juju = legacy_juju() - user, password = None, None - if (is_legacy_juju and in_staging) or sandbox: - user, password = 'admin', 'admin' - else: - user, password = None, None - - api_backend = 'python' if is_legacy_juju else 'go' - if secure: - protocol = 'wss' - else: - log('Running in insecure mode! Port 80 will serve unencrypted.') - protocol = 'ws' - - context = { - 'raw_protocol': protocol, - 'address': unit_get('public-address'), - 'console_enabled': json.dumps(console_enabled), - 'login_help': json.dumps(login_help), - 'password': json.dumps(password), - 'api_backend': json.dumps(api_backend), - 'readonly': json.dumps(readonly), - 'user': json.dumps(user), - 'protocol': json.dumps(protocol), - 'sandbox': json.dumps(sandbox), - 'charmworld_url': json.dumps(charmworld_url), - } - if config_js_path is None: - config_js_path = os.path.join( - build_dir, 'juju-ui', 'assets', 'config.js') - render_to_file('config/config.js.template', context, config_js_path) - - write_apache_config(build_dir, serve_tests) - - log('Generating haproxy configuration file.') - if is_legacy_juju: - # The PyJuju API agent is listening on localhost. - api_address = '127.0.0.1:{0}'.format(API_PORT) - else: - # Retrieve the juju-core API server address. - api_address = get_api_address(os.path.join(CURRENT_DIR, '..')) - context = { - 'api_address': api_address, - 'api_pem': JUJU_PEM, - 'legacy_juju': is_legacy_juju, - 'ssl_cert_path': ssl_cert_path, - # In PyJuju environments, use the same certificate for both HTTPS and - # WebSocket connections. In juju-core the system already has the proper - # certificate installed. - 'web_pem': JUJU_PEM, - 'web_port': WEB_PORT, - 'secure': secure - } - render_to_file('config/haproxy.cfg.template', context, haproxy_path) - log('Starting Juju GUI.') - - -def write_apache_config(build_dir, serve_tests=False): - log('Generating the apache site configuration file.') - context = { - 'port': WEB_PORT, - 'serve_tests': serve_tests, - 'server_root': build_dir, - 'tests_root': os.path.join(JUJU_GUI_DIR, 'test', ''), - } - render_to_file('config/apache-ports.template', context, JUJU_GUI_PORTS) - render_to_file('config/apache-site.template', context, JUJU_GUI_SITE) - - -def get_npm_cache_archive_url(Launchpad=Launchpad): - """Figure out the URL of the most recent NPM cache archive on Launchpad.""" - launchpad = Launchpad.login_anonymously('Juju GUI charm', 'production') - project = launchpad.projects['juju-gui'] - # Find the URL of the most recently created NPM cache archive. - npm_cache_url = get_release_file_url(project, 'npm-cache', None) - return npm_cache_url - - -def prime_npm_cache(npm_cache_url): - """Download NPM cache archive and prime the NPM cache with it.""" - # Download the cache archive and then uncompress it into the NPM cache. - npm_cache_archive = os.path.join(CURRENT_DIR, 'npm-cache.tgz') - cmd_log(run('curl', '-L', '-o', npm_cache_archive, npm_cache_url)) - npm_cache_dir = os.path.expanduser('~/.npm') - # The NPM cache directory probably does not exist, so make it if not. - try: - os.mkdir(npm_cache_dir) - except OSError, e: - # If the directory already exists then ignore the error. - if e.errno != errno.EEXIST: # File exists. - raise - uncompress = command('tar', '-x', '-z', '-C', npm_cache_dir, '-f') - cmd_log(uncompress(npm_cache_archive)) - - -def fetch_gui(juju_gui_source, logpath): - """Retrieve the Juju GUI release/branch.""" - # Retrieve a Juju GUI release. - origin, version_or_branch = parse_source(juju_gui_source) - if origin == 'branch': - # Make sure we have the dependencies necessary for us to actually make - # a build. - _get_build_dependencies() - # Create a release starting from a branch. - juju_gui_source_dir = os.path.join(CURRENT_DIR, 'juju-gui-source') - log('Retrieving Juju GUI source checkout from %s.' % version_or_branch) - cmd_log(run('rm', '-rf', juju_gui_source_dir)) - cmd_log(bzr_checkout(version_or_branch, juju_gui_source_dir)) - log('Preparing a Juju GUI release.') - logdir = os.path.dirname(logpath) - fd, name = tempfile.mkstemp(prefix='make-distfile-', dir=logdir) - log('Output from "make distfile" sent to %s' % name) - with environ(NO_BZR='1'): - run('make', '-C', juju_gui_source_dir, 'distfile', - stdout=fd, stderr=fd) - release_tarball = first_path_in_dir( - os.path.join(juju_gui_source_dir, 'releases')) - else: - log('Retrieving Juju GUI release.') - if origin == 'url': - file_url = version_or_branch - else: - # Retrieve a release from Launchpad. - launchpad = Launchpad.login_anonymously( - 'Juju GUI charm', 'production') - project = launchpad.projects['juju-gui'] - file_url = get_release_file_url(project, origin, version_or_branch) - log('Downloading release file from %s.' % file_url) - release_tarball = os.path.join(CURRENT_DIR, 'release.tgz') - cmd_log(run('curl', '-L', '-o', release_tarball, file_url)) - return release_tarball - - -def fetch_api(juju_api_branch): - """Retrieve the Juju branch.""" - # Retrieve Juju API source checkout. - log('Retrieving Juju API source checkout.') - cmd_log(run('rm', '-rf', JUJU_DIR)) - cmd_log(bzr_checkout(juju_api_branch, JUJU_DIR)) - - -def setup_gui(release_tarball): - """Set up Juju GUI.""" - # Uncompress the release tarball. - log('Installing Juju GUI.') - release_dir = os.path.join(CURRENT_DIR, 'release') - cmd_log(run('rm', '-rf', release_dir)) - os.mkdir(release_dir) - uncompress = command('tar', '-x', '-z', '-C', release_dir, '-f') - cmd_log(uncompress(release_tarball)) - # Link the Juju GUI dir to the contents of the release tarball. - cmd_log(run('ln', '-sf', first_path_in_dir(release_dir), JUJU_GUI_DIR)) - - -def setup_apache(): - """Set up apache.""" - log('Setting up apache.') - if not os.path.exists(JUJU_GUI_SITE): - cmd_log(run('touch', JUJU_GUI_SITE)) - cmd_log(run('chown', 'ubuntu:', JUJU_GUI_SITE)) - cmd_log( - run('ln', '-s', JUJU_GUI_SITE, - '/etc/apache2/sites-enabled/juju-gui')) - - if not os.path.exists(JUJU_GUI_PORTS): - cmd_log(run('touch', JUJU_GUI_PORTS)) - cmd_log(run('chown', 'ubuntu:', JUJU_GUI_PORTS)) - - with su('root'): - run('a2dissite', 'default') - run('a2ensite', 'juju-gui') - - -def save_or_create_certificates( - ssl_cert_path, ssl_cert_contents, ssl_key_contents): - """Generate the SSL certificates. - - If both *ssl_cert_contents* and *ssl_key_contents* are provided, use them - as certificates; otherwise, generate them. - - Also create a pem file, suitable for use in the haproxy configuration, - concatenating the key and the certificate files. - """ - crt_path = os.path.join(ssl_cert_path, 'juju.crt') - key_path = os.path.join(ssl_cert_path, 'juju.key') - if not os.path.exists(ssl_cert_path): - os.makedirs(ssl_cert_path) - if ssl_cert_contents and ssl_key_contents: - # Save the provided certificates. - with open(crt_path, 'w') as cert_file: - cert_file.write(ssl_cert_contents) - with open(key_path, 'w') as key_file: - key_file.write(ssl_key_contents) - else: - # Generate certificates. - # See http://superuser.com/questions/226192/openssl-without-prompt - cmd_log(run( - 'openssl', 'req', '-new', '-newkey', 'rsa:4096', - '-days', '365', '-nodes', '-x509', '-subj', - # These are arbitrary test values for the certificate. - '/C=GB/ST=Juju/L=GUI/O=Ubuntu/CN=juju.ubuntu.com', - '-keyout', key_path, '-out', crt_path)) - # Generate the pem file. - pem_path = os.path.join(ssl_cert_path, JUJU_PEM) - if os.path.exists(pem_path): - os.remove(pem_path) - with open(pem_path, 'w') as pem_file: - shutil.copyfileobj(open(key_path), pem_file) - shutil.copyfileobj(open(crt_path), pem_file) - - -def find_missing_packages(*packages): - """Given a list of packages, return the packages which are not installed. - """ - cache = apt.Cache() - missing = set() - for pkg_name in packages: - try: - pkg = cache[pkg_name] - except KeyError: - missing.add(pkg_name) - continue - if pkg.is_installed: - continue - missing.add(pkg_name) - return missing - - -## Backend support decorators - -def chain(name): - """Helper method to compose a set of mixin objects into a callable. - - Each method is called in the context of its mixin instance, and its - argument is the Backend instance. - """ - # Chain method calls through all implementing mixins. - def method(self): - for mixin in self.mixins: - a_callable = getattr(type(mixin), name, None) - if a_callable: - a_callable(mixin, self) - - method.__name__ = name - return method - - -def merge(name): - """Helper to merge a property from a set of strategy objects - into a unified set. - """ - # Return merged property from every providing mixin as a set. - @property - def method(self): - result = set() - for mixin in self.mixins: - segment = getattr(type(mixin), name, None) - if segment and isinstance(segment, (list, tuple, set)): - result |= set(segment) - - return result - return method diff --git a/hooks/charmhelpers/contrib/network/__init__.py b/hooks/charmhelpers/contrib/network/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/hooks/charmhelpers/contrib/network/ovs/__init__.py b/hooks/charmhelpers/contrib/network/ovs/__init__.py deleted file mode 100644 index 5eba8376..00000000 --- a/hooks/charmhelpers/contrib/network/ovs/__init__.py +++ /dev/null @@ -1,75 +0,0 @@ -''' Helpers for interacting with OpenvSwitch ''' -import subprocess -import os -from charmhelpers.core.hookenv import ( - log, WARNING -) -from charmhelpers.core.host import ( - service -) - - -def add_bridge(name): - ''' Add the named bridge to openvswitch ''' - log('Creating bridge {}'.format(name)) - subprocess.check_call(["ovs-vsctl", "--", "--may-exist", "add-br", name]) - - -def del_bridge(name): - ''' Delete the named bridge from openvswitch ''' - log('Deleting bridge {}'.format(name)) - subprocess.check_call(["ovs-vsctl", "--", "--if-exists", "del-br", name]) - - -def add_bridge_port(name, port): - ''' Add a port to the named openvswitch bridge ''' - log('Adding port {} to bridge {}'.format(port, name)) - subprocess.check_call(["ovs-vsctl", "--", "--may-exist", "add-port", - name, port]) - subprocess.check_call(["ip", "link", "set", port, "up"]) - - -def del_bridge_port(name, port): - ''' Delete a port from the named openvswitch bridge ''' - log('Deleting port {} from bridge {}'.format(port, name)) - subprocess.check_call(["ovs-vsctl", "--", "--if-exists", "del-port", - name, port]) - subprocess.check_call(["ip", "link", "set", port, "down"]) - - -def set_manager(manager): - ''' Set the controller for the local openvswitch ''' - log('Setting manager for local ovs to {}'.format(manager)) - subprocess.check_call(['ovs-vsctl', 'set-manager', - 'ssl:{}'.format(manager)]) - - -CERT_PATH = '/etc/openvswitch/ovsclient-cert.pem' - - -def get_certificate(): - ''' Read openvswitch certificate from disk ''' - if os.path.exists(CERT_PATH): - log('Reading ovs certificate from {}'.format(CERT_PATH)) - with open(CERT_PATH, 'r') as cert: - full_cert = cert.read() - begin_marker = "-----BEGIN CERTIFICATE-----" - end_marker = "-----END CERTIFICATE-----" - begin_index = full_cert.find(begin_marker) - end_index = full_cert.rfind(end_marker) - if end_index == -1 or begin_index == -1: - raise RuntimeError("Certificate does not contain valid begin" - " and end markers.") - full_cert = full_cert[begin_index:(end_index + len(end_marker))] - return full_cert - else: - log('Certificate not found', level=WARNING) - return None - - -def full_restart(): - ''' Full restart and reload of openvswitch ''' - if os.path.exists('/etc/init/openvswitch-force-reload-kmod.conf'): - service('start', 'openvswitch-force-reload-kmod') - else: - service('force-reload-kmod', 'openvswitch-switch') diff --git a/hooks/charmhelpers/contrib/saltstack/__init__.py b/hooks/charmhelpers/contrib/saltstack/__init__.py deleted file mode 100644 index 19078f8e..00000000 --- a/hooks/charmhelpers/contrib/saltstack/__init__.py +++ /dev/null @@ -1,102 +0,0 @@ -"""Charm Helpers saltstack - declare the state of your machines. - -This helper enables you to declare your machine state, rather than -program it procedurally (and have to test each change to your procedures). -Your install hook can be as simple as: - -{{{ -from charmhelpers.contrib.saltstack import ( - install_salt_support, - update_machine_state, -) - - -def install(): - install_salt_support() - update_machine_state('machine_states/dependencies.yaml') - update_machine_state('machine_states/installed.yaml') -}}} - -and won't need to change (nor will its tests) when you change the machine -state. - -It's using a python package called salt-minion which allows various formats for -specifying resources, such as: - -{{{ -/srv/{{ basedir }}: - file.directory: - - group: ubunet - - user: ubunet - - require: - - user: ubunet - - recurse: - - user - - group - -ubunet: - group.present: - - gid: 1500 - user.present: - - uid: 1500 - - gid: 1500 - - createhome: False - - require: - - group: ubunet -}}} - -The docs for all the different state definitions are at: - http://docs.saltstack.com/ref/states/all/ - - -TODO: - * Add test helpers which will ensure that machine state definitions - are functionally (but not necessarily logically) correct (ie. getting - salt to parse all state defs. - * Add a link to a public bootstrap charm example / blogpost. - * Find a way to obviate the need to use the grains['charm_dir'] syntax - in templates. -""" -# Copyright 2013 Canonical Ltd. -# -# Authors: -# Charm Helpers Developers -import subprocess - -import charmhelpers.contrib.templating.contexts -import charmhelpers.core.host -import charmhelpers.core.hookenv - - -salt_grains_path = '/etc/salt/grains' - - -def install_salt_support(from_ppa=True): - """Installs the salt-minion helper for machine state. - - By default the salt-minion package is installed from - the saltstack PPA. If from_ppa is False you must ensure - that the salt-minion package is available in the apt cache. - """ - if from_ppa: - subprocess.check_call([ - '/usr/bin/add-apt-repository', - '--yes', - 'ppa:saltstack/salt', - ]) - subprocess.check_call(['/usr/bin/apt-get', 'update']) - # We install salt-common as salt-minion would run the salt-minion - # daemon. - charmhelpers.fetch.apt_install('salt-common') - - -def update_machine_state(state_path): - """Update the machine state using the provided state declaration.""" - charmhelpers.contrib.templating.contexts.juju_state_to_yaml( - salt_grains_path) - subprocess.check_call([ - 'salt-call', - '--local', - 'state.template', - state_path, - ]) diff --git a/hooks/charmhelpers/contrib/ssl/__init__.py b/hooks/charmhelpers/contrib/ssl/__init__.py deleted file mode 100644 index 2999c0a3..00000000 --- a/hooks/charmhelpers/contrib/ssl/__init__.py +++ /dev/null @@ -1,78 +0,0 @@ -import subprocess -from charmhelpers.core import hookenv - - -def generate_selfsigned(keyfile, certfile, keysize="1024", config=None, subject=None, cn=None): - """Generate selfsigned SSL keypair - - You must provide one of the 3 optional arguments: - config, subject or cn - If more than one is provided the leftmost will be used - - Arguments: - keyfile -- (required) full path to the keyfile to be created - certfile -- (required) full path to the certfile to be created - keysize -- (optional) SSL key length - config -- (optional) openssl configuration file - subject -- (optional) dictionary with SSL subject variables - cn -- (optional) cerfificate common name - - Required keys in subject dict: - cn -- Common name (eq. FQDN) - - Optional keys in subject dict - country -- Country Name (2 letter code) - state -- State or Province Name (full name) - locality -- Locality Name (eg, city) - organization -- Organization Name (eg, company) - organizational_unit -- Organizational Unit Name (eg, section) - email -- Email Address - """ - - cmd = [] - if config: - cmd = ["/usr/bin/openssl", "req", "-new", "-newkey", - "rsa:{}".format(keysize), "-days", "365", "-nodes", "-x509", - "-keyout", keyfile, - "-out", certfile, "-config", config] - elif subject: - ssl_subject = "" - if "country" in subject: - ssl_subject = ssl_subject + "/C={}".format(subject["country"]) - if "state" in subject: - ssl_subject = ssl_subject + "/ST={}".format(subject["state"]) - if "locality" in subject: - ssl_subject = ssl_subject + "/L={}".format(subject["locality"]) - if "organization" in subject: - ssl_subject = ssl_subject + "/O={}".format(subject["organization"]) - if "organizational_unit" in subject: - ssl_subject = ssl_subject + "/OU={}".format(subject["organizational_unit"]) - if "cn" in subject: - ssl_subject = ssl_subject + "/CN={}".format(subject["cn"]) - else: - hookenv.log("When using \"subject\" argument you must " - "provide \"cn\" field at very least") - return False - if "email" in subject: - ssl_subject = ssl_subject + "/emailAddress={}".format(subject["email"]) - - cmd = ["/usr/bin/openssl", "req", "-new", "-newkey", - "rsa:{}".format(keysize), "-days", "365", "-nodes", "-x509", - "-keyout", keyfile, - "-out", certfile, "-subj", ssl_subject] - elif cn: - cmd = ["/usr/bin/openssl", "req", "-new", "-newkey", - "rsa:{}".format(keysize), "-days", "365", "-nodes", "-x509", - "-keyout", keyfile, - "-out", certfile, "-subj", "/CN={}".format(cn)] - - if not cmd: - hookenv.log("No config, subject or cn provided," - "unable to generate self signed SSL certificates") - return False - try: - subprocess.check_call(cmd) - return True - except Exception as e: - print "Execution of openssl command failed:\n{}".format(e) - return False diff --git a/hooks/charmhelpers/contrib/templating/__init__.py b/hooks/charmhelpers/contrib/templating/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/hooks/charmhelpers/contrib/templating/contexts.py b/hooks/charmhelpers/contrib/templating/contexts.py deleted file mode 100644 index b117b2de..00000000 --- a/hooks/charmhelpers/contrib/templating/contexts.py +++ /dev/null @@ -1,73 +0,0 @@ -# Copyright 2013 Canonical Ltd. -# -# Authors: -# Charm Helpers Developers -"""A helper to create a yaml cache of config with namespaced relation data.""" -import os -import yaml - -import charmhelpers.core.hookenv - - -charm_dir = os.environ.get('CHARM_DIR', '') - - -def juju_state_to_yaml(yaml_path, namespace_separator=':', - allow_hyphens_in_keys=True): - """Update the juju config and state in a yaml file. - - This includes any current relation-get data, and the charm - directory. - - This function was created for the ansible and saltstack - support, as those libraries can use a yaml file to supply - context to templates, but it may be useful generally to - create and update an on-disk cache of all the config, including - previous relation data. - - By default, hyphens are allowed in keys as this is supported - by yaml, but for tools like ansible, hyphens are not valid [1]. - - [1] http://www.ansibleworks.com/docs/playbooks_variables.html#what-makes-a-valid-variable-name - """ - config = charmhelpers.core.hookenv.config() - - # Add the charm_dir which we will need to refer to charm - # file resources etc. - config['charm_dir'] = charm_dir - config['local_unit'] = charmhelpers.core.hookenv.local_unit() - - # Add any relation data prefixed with the relation type. - relation_type = charmhelpers.core.hookenv.relation_type() - if relation_type is not None: - relation_data = charmhelpers.core.hookenv.relation_get() - relation_data = dict( - ("{relation_type}{namespace_separator}{key}".format( - relation_type=relation_type.replace('-', '_'), - key=key, - namespace_separator=namespace_separator), val) - for key, val in relation_data.items()) - config.update(relation_data) - - # Don't use non-standard tags for unicode which will not - # work when salt uses yaml.load_safe. - yaml.add_representer(unicode, lambda dumper, - value: dumper.represent_scalar( - u'tag:yaml.org,2002:str', value)) - - yaml_dir = os.path.dirname(yaml_path) - if not os.path.exists(yaml_dir): - os.makedirs(yaml_dir) - - if os.path.exists(yaml_path): - with open(yaml_path, "r") as existing_vars_file: - existing_vars = yaml.load(existing_vars_file.read()) - else: - existing_vars = {} - - if not allow_hyphens_in_keys: - config = dict( - (key.replace('-', '_'), val) for key, val in config.items()) - existing_vars.update(config) - with open(yaml_path, "w+") as fp: - fp.write(yaml.dump(existing_vars)) diff --git a/hooks/charmhelpers/contrib/templating/pyformat.py b/hooks/charmhelpers/contrib/templating/pyformat.py deleted file mode 100644 index baaf98e7..00000000 --- a/hooks/charmhelpers/contrib/templating/pyformat.py +++ /dev/null @@ -1,13 +0,0 @@ -''' -Templating using standard Python str.format() method. -''' - -from charmhelpers.core import hookenv - - -def render(template, extra={}, **kwargs): - """Return the template rendered using Python's str.format().""" - context = hookenv.execution_environment() - context.update(extra) - context.update(kwargs) - return template.format(**context) diff --git a/hooks/charmhelpers/payload/archive.py b/hooks/charmhelpers/payload/archive.py deleted file mode 100644 index de03e1b7..00000000 --- a/hooks/charmhelpers/payload/archive.py +++ /dev/null @@ -1,57 +0,0 @@ -import os -import tarfile -import zipfile -from charmhelpers.core import ( - host, - hookenv, -) - - -class ArchiveError(Exception): - pass - - -def get_archive_handler(archive_name): - if os.path.isfile(archive_name): - if tarfile.is_tarfile(archive_name): - return extract_tarfile - elif zipfile.is_zipfile(archive_name): - return extract_zipfile - else: - # look at the file name - for ext in ('.tar', '.tar.gz', '.tgz', 'tar.bz2', '.tbz2', '.tbz'): - if archive_name.endswith(ext): - return extract_tarfile - for ext in ('.zip', '.jar'): - if archive_name.endswith(ext): - return extract_zipfile - - -def archive_dest_default(archive_name): - archive_file = os.path.basename(archive_name) - return os.path.join(hookenv.charm_dir(), "archives", archive_file) - - -def extract(archive_name, destpath=None): - handler = get_archive_handler(archive_name) - if handler: - if not destpath: - destpath = archive_dest_default(archive_name) - if not os.path.isdir(destpath): - host.mkdir(destpath) - handler(archive_name, destpath) - return destpath - else: - raise ArchiveError("No handler for archive") - - -def extract_tarfile(archive_name, destpath): - "Unpack a tar archive, optionally compressed" - archive = tarfile.open(archive_name) - archive.extractall(destpath) - - -def extract_zipfile(archive_name, destpath): - "Unpack a zip file" - archive = zipfile.ZipFile(archive_name) - archive.extractall(destpath) diff --git a/revision b/revision index 6a4573e8..405e2afe 100644 --- a/revision +++ b/revision @@ -1 +1 @@ -133 +134 From 60af3b499b55f55c331191d4f88f349961ebd4ba Mon Sep 17 00:00:00 2001 From: "yolanda.robla@canonical.com" <> Date: Thu, 30 Jan 2014 14:18:53 +0100 Subject: [PATCH 11/19] sending rabbitmq_hosts when ha-vip-only is True --- hooks/charmhelpers/contrib/openstack/context.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 6331a92e..c995432e 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -181,7 +181,8 @@ class AMQPContext(OSContextGenerator): # Sufficient information found = break out! break # Used for active/active rabbitmq >= grizzly - if 'clustered' not in ctxt and len(related_units(rid)) > 1: + if ('clustered' not in ctxt or conf['ha-vip-only'] == True) and + len(related_units(rid)) > 1: rabbitmq_hosts = [] for unit in related_units(rid): rabbitmq_hosts.append(relation_get('private-address', From 96b6bcab08d3104afe229df8ffd1cd092df3b33e Mon Sep 17 00:00:00 2001 From: "yolanda.robla@canonical.com" <> Date: Thu, 30 Jan 2014 14:39:25 +0100 Subject: [PATCH 12/19] lint fixes --- hooks/charmhelpers/contrib/openstack/context.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index c995432e..a5ee7669 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -181,7 +181,7 @@ class AMQPContext(OSContextGenerator): # Sufficient information found = break out! break # Used for active/active rabbitmq >= grizzly - if ('clustered' not in ctxt or conf['ha-vip-only'] == True) and + if ('clustered' not in ctxt or conf['ha-vip-only'] is True) and \ len(related_units(rid)) > 1: rabbitmq_hosts = [] for unit in related_units(rid): From c0e595867d266669aca0a25d2ecb276fe6b02f8f Mon Sep 17 00:00:00 2001 From: "yolanda.robla@canonical.com" <> Date: Fri, 31 Jan 2014 09:40:40 +0100 Subject: [PATCH 13/19] reading ha-vip-only from relation --- hooks/charmhelpers/contrib/openstack/context.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index a5ee7669..1c1a40b5 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -181,8 +181,9 @@ class AMQPContext(OSContextGenerator): # Sufficient information found = break out! break # Used for active/active rabbitmq >= grizzly - if ('clustered' not in ctxt or conf['ha-vip-only'] is True) and \ + if ('clustered' not in ctxt or relation_get('ha-vip-only') == 'True') and \ len(related_units(rid)) > 1: + ctxt['rabbitmq_ha_queues'] = relation_get('ha_queues') rabbitmq_hosts = [] for unit in related_units(rid): rabbitmq_hosts.append(relation_get('private-address', From 77d68b74ce5ae6a4a895965d4079026a11ee3b45 Mon Sep 17 00:00:00 2001 From: "yolanda.robla@canonical.com" <> Date: Fri, 31 Jan 2014 14:04:10 +0100 Subject: [PATCH 14/19] refreshed charmhelper --- hooks/charmhelpers/contrib/openstack/context.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 1c1a40b5..a4452c7b 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -183,7 +183,10 @@ class AMQPContext(OSContextGenerator): # Used for active/active rabbitmq >= grizzly if ('clustered' not in ctxt or relation_get('ha-vip-only') == 'True') and \ len(related_units(rid)) > 1: - ctxt['rabbitmq_ha_queues'] = relation_get('ha_queues') + if relation_get('ha_queues'): + ctxt['rabbitmq_ha_queues'] = relation_get('ha_queues') + else: + ctxt['rabbitmq_ha_queues'] = 'False' rabbitmq_hosts = [] for unit in related_units(rid): rabbitmq_hosts.append(relation_get('private-address', From 80a11f519d908b8a55b3f27ee5d74d5461ed8f30 Mon Sep 17 00:00:00 2001 From: "yolanda.robla@canonical.com" <> Date: Wed, 5 Mar 2014 15:59:55 +0100 Subject: [PATCH 15/19] adding syslog and user_config to cinder.conf --- templates/grizzly/cinder.conf | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/templates/grizzly/cinder.conf b/templates/grizzly/cinder.conf index af4bd528..9139d94b 100644 --- a/templates/grizzly/cinder.conf +++ b/templates/grizzly/cinder.conf @@ -10,6 +10,7 @@ iscsi_helper = tgtadm volume_name_template = volume-%s volume_group = cinder-volumes verbose = True +use_syslog = {{ use_syslog }} auth_strategy = keystone state_path = /var/lib/cinder lock_path = /var/lock/cinder @@ -45,8 +46,15 @@ rbd_user = {{ rbd_user }} osapi_volume_listen_port = {{ osapi_volume_listen_port }} {% endif -%} {% if glance_api_servers -%} -glance_api_servers = {{ glance_api_servers }} +dglance_api_servers = {{ glance_api_servers }} {% endif -%} {% if glance_api_version -%} glance_api_version = {{ glance_api_version }} {% endif -%} + +{% if user_config_flags -%} +{% for key, value in user_config_flags.iteritems() -%} +{{ key }} = {{ value }} +{% endfor -%} +{% endif -%} + From 067d3a7efedafc38c71c5871567b357dfba324cc Mon Sep 17 00:00:00 2001 From: "yolanda.robla@canonical.com" <> Date: Fri, 7 Mar 2014 10:56:48 +0100 Subject: [PATCH 16/19] resync amqp context --- hooks/charmhelpers/contrib/openstack/context.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index a2d4636b..865e4201 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -199,6 +199,7 @@ class AMQPContext(OSContextGenerator): ctxt = {} for rid in relation_ids('amqp'): + ha_vip_only = False for unit in related_units(rid): if relation_get('clustered', rid=rid, unit=unit): ctxt['clustered'] = True @@ -213,16 +214,16 @@ class AMQPContext(OSContextGenerator): unit=unit), 'rabbitmq_virtual_host': vhost, }) + if relation_get('ha_queues', rid=rid, unit=unit): + ctxt['rabbitmq_ha_queues'] = True + + ha_vip_only = (relation_get('ha-vip-only', rid=rid, unit=unit) == 'True') + if context_complete(ctxt): # Sufficient information found = break out! break # Used for active/active rabbitmq >= grizzly - if ('clustered' not in ctxt or relation_get('ha-vip-only') == 'True') and \ - len(related_units(rid)) > 1: - if relation_get('ha_queues'): - ctxt['rabbitmq_ha_queues'] = relation_get('ha_queues') - else: - ctxt['rabbitmq_ha_queues'] = False + if ('clustered' not in ctxt or ha_vip_only) and len(related_units(rid)) > 1: rabbitmq_hosts = [] for unit in related_units(rid): rabbitmq_hosts.append(relation_get('private-address', From cd12aa2ff19c5a4c32e6c88998e78c7d8b8dd3c7 Mon Sep 17 00:00:00 2001 From: "yolanda.robla@canonical.com" <> Date: Wed, 12 Mar 2014 13:09:23 +0100 Subject: [PATCH 17/19] resync charmhelper --- hooks/charmhelpers/contrib/openstack/utils.py | 24 +++++++++---------- revision | 2 +- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 56d04245..de95a4b6 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -420,19 +420,19 @@ def get_hostname(address, fqdn=True): Resolves hostname for given IP, or returns the input if it is already a hostname. """ - if not is_ip(address): - return address + if is_ip(address): + try: + import dns.reversename + except ImportError: + apt_install('python-dnspython') + import dns.reversename - try: - import dns.reversename - except ImportError: - apt_install('python-dnspython') - import dns.reversename - - rev = dns.reversename.from_address(address) - result = ns_query(rev) - if not result: - return None + rev = dns.reversename.from_address(address) + result = ns_query(rev) + if not result: + return None + else: + result = address if fqdn: # strip trailing . diff --git a/revision b/revision index 405e2afe..c8b255fc 100644 --- a/revision +++ b/revision @@ -1 +1 @@ -134 +135 From 53eef715edb39854168c58df8730300df473b6e3 Mon Sep 17 00:00:00 2001 From: James Page Date: Mon, 17 Mar 2014 11:42:56 +0000 Subject: [PATCH 18/19] Resync helpers --- charm-helpers.yaml | 2 +- hooks/charmhelpers/contrib/openstack/context.py | 8 +++++--- hooks/charmhelpers/fetch/__init__.py | 4 ++++ 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/charm-helpers.yaml b/charm-helpers.yaml index b4a77d14..6331da67 100644 --- a/charm-helpers.yaml +++ b/charm-helpers.yaml @@ -1,4 +1,4 @@ -branch: lp:charm-helpers +branch: lp:~openstack-charmers/charm-helpers/active-active destination: hooks/charmhelpers include: - core diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 865e4201..6014ee1b 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -214,16 +214,18 @@ class AMQPContext(OSContextGenerator): unit=unit), 'rabbitmq_virtual_host': vhost, }) - if relation_get('ha_queues', rid=rid, unit=unit): + if relation_get('ha_queues', rid=rid, unit=unit) is not None: ctxt['rabbitmq_ha_queues'] = True - ha_vip_only = (relation_get('ha-vip-only', rid=rid, unit=unit) == 'True') + ha_vip_only = relation_get('ha-vip-only', + rid=rid, unit=unit) is not None if context_complete(ctxt): # Sufficient information found = break out! break # Used for active/active rabbitmq >= grizzly - if ('clustered' not in ctxt or ha_vip_only) and len(related_units(rid)) > 1: + if ('clustered' not in ctxt or ha_vip_only) \ + and len(related_units(rid)) > 1: rabbitmq_hosts = [] for unit in related_units(rid): rabbitmq_hosts.append(relation_get('private-address', diff --git a/hooks/charmhelpers/fetch/__init__.py b/hooks/charmhelpers/fetch/__init__.py index 07bb707d..c05e0335 100644 --- a/hooks/charmhelpers/fetch/__init__.py +++ b/hooks/charmhelpers/fetch/__init__.py @@ -135,6 +135,10 @@ def apt_hold(packages, fatal=False): def add_source(source, key=None): + if source is None: + log('Source is not present. Skipping') + return + if (source.startswith('ppa:') or source.startswith('http') or source.startswith('deb ') or From a0e27f7bfe58535886899f06b482a36e7ca8ac0a Mon Sep 17 00:00:00 2001 From: James Page Date: Mon, 17 Mar 2014 14:22:53 +0000 Subject: [PATCH 19/19] Fixup problem with cinder.conf --- templates/grizzly/cinder.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/grizzly/cinder.conf b/templates/grizzly/cinder.conf index 9139d94b..18b09da4 100644 --- a/templates/grizzly/cinder.conf +++ b/templates/grizzly/cinder.conf @@ -46,7 +46,7 @@ rbd_user = {{ rbd_user }} osapi_volume_listen_port = {{ osapi_volume_listen_port }} {% endif -%} {% if glance_api_servers -%} -dglance_api_servers = {{ glance_api_servers }} +glance_api_servers = {{ glance_api_servers }} {% endif -%} {% if glance_api_version -%} glance_api_version = {{ glance_api_version }}