From e8685516c863bc7ee881686784cd61bdca11a26c Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Mon, 22 Sep 2014 21:21:48 +0100 Subject: [PATCH] synced lp:~cts-engineering/charms/trusty/charm-helpers/ipv6 --- charm-helpers-hooks.yaml | 2 +- charm-helpers-tests.yaml | 2 +- hooks/charmhelpers/contrib/network/ip.py | 8 +- .../charmhelpers/contrib/openstack/context.py | 2 +- hooks/charmhelpers/contrib/openstack/ip.py | 5 +- hooks/charmhelpers/core/host.py | 31 ++++- hooks/charmhelpers/core/services/helpers.py | 124 +++++++++++++++++- hooks/charmhelpers/fetch/__init__.py | 21 ++- hooks/charmhelpers/fetch/archiveurl.py | 88 +++++++------ 9 files changed, 219 insertions(+), 64 deletions(-) diff --git a/charm-helpers-hooks.yaml b/charm-helpers-hooks.yaml index 214b2457..d32ea839 100644 --- a/charm-helpers-hooks.yaml +++ b/charm-helpers-hooks.yaml @@ -1,4 +1,4 @@ -branch: lp:charm-helpers +branch: lp:~cts-engineering/charms/trusty/charm-helpers/ipv6 destination: hooks/charmhelpers include: - core diff --git a/charm-helpers-tests.yaml b/charm-helpers-tests.yaml index 48b12f6f..25fc96d2 100644 --- a/charm-helpers-tests.yaml +++ b/charm-helpers-tests.yaml @@ -1,4 +1,4 @@ -branch: lp:charm-helpers +branch: lp:~cts-engineering/charms/trusty/charm-helpers/ipv6 destination: tests/charmhelpers include: - contrib.amulet diff --git a/hooks/charmhelpers/contrib/network/ip.py b/hooks/charmhelpers/contrib/network/ip.py index b859a097..37ecbbed 100644 --- a/hooks/charmhelpers/contrib/network/ip.py +++ b/hooks/charmhelpers/contrib/network/ip.py @@ -5,7 +5,9 @@ from functools import partial from charmhelpers.fetch import apt_install from charmhelpers.core.hookenv import ( - ERROR, log, + WARNING, + ERROR, + log ) try: @@ -164,9 +166,9 @@ def format_ipv6_addr(address): if is_ipv6(address): address = "[%s]" % address else: - log("Not an valid ipv6 address: %s" % address, - level=ERROR) + log("Not a valid ipv6 address: %s" % address, level=WARNING) address = None + return address diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 902a0b1f..210238e6 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -47,7 +47,7 @@ from charmhelpers.contrib.openstack.neutron import ( from charmhelpers.contrib.network.ip import ( get_address_in_network, get_ipv6_addr, - format_ipv6_addr, + format_ipv6_addr ) CA_CERT_PATH = '/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt' diff --git a/hooks/charmhelpers/contrib/openstack/ip.py b/hooks/charmhelpers/contrib/openstack/ip.py index 51a11e63..bc84fc45 100644 --- a/hooks/charmhelpers/contrib/openstack/ip.py +++ b/hooks/charmhelpers/contrib/openstack/ip.py @@ -66,10 +66,7 @@ def resolve_address(endpoint_type=PUBLIC): resolved_address = vip else: if config('prefer-ipv6'): - list = [] - if config('vip'): - list.append(config('vip')) - fallback_addr = get_ipv6_addr(exc_list=list)[0] + fallback_addr = get_ipv6_addr(exc_list=[config('vip')])[0] else: fallback_addr = unit_get(_address_map[endpoint_type]['fallback']) resolved_address = get_address_in_network( diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py index b85b0280..3ac70143 100644 --- a/hooks/charmhelpers/core/host.py +++ b/hooks/charmhelpers/core/host.py @@ -209,10 +209,15 @@ def mounts(): return system_mounts -def file_hash(path): - """Generate a md5 hash of the contents of 'path' or None if not found """ +def file_hash(path, hash_type='md5'): + """ + Generate a hash checksum of the contents of 'path' or None if not found. + + :param str hash_type: Any hash alrgorithm supported by :mod:`hashlib`, + such as md5, sha1, sha256, sha512, etc. + """ if os.path.exists(path): - h = hashlib.md5() + h = getattr(hashlib, hash_type)() with open(path, 'r') as source: h.update(source.read()) # IGNORE:E1101 - it does have update return h.hexdigest() @@ -220,6 +225,26 @@ def file_hash(path): return None +def check_hash(path, checksum, hash_type='md5'): + """ + Validate a file using a cryptographic checksum. + + + :param str checksum: Value of the checksum used to validate the file. + :param str hash_type: Hash algorithm used to generate :param:`checksum`. + Can be any hash alrgorithm supported by :mod:`hashlib`, + such as md5, sha1, sha256, sha512, etc. + :raises ChecksumError: If the file fails the checksum + """ + actual_checksum = file_hash(path, hash_type) + if checksum != actual_checksum: + raise ChecksumError("'%s' != '%s'" % (checksum, actual_checksum)) + + +class ChecksumError(ValueError): + pass + + def restart_on_change(restart_map, stopstart=False): """Restart services based on configuration files changing diff --git a/hooks/charmhelpers/core/services/helpers.py b/hooks/charmhelpers/core/services/helpers.py index 4b90589b..7067b94b 100644 --- a/hooks/charmhelpers/core/services/helpers.py +++ b/hooks/charmhelpers/core/services/helpers.py @@ -1,3 +1,5 @@ +import os +import yaml from charmhelpers.core import hookenv from charmhelpers.core import templating @@ -19,15 +21,21 @@ class RelationContext(dict): the `name` attribute that are complete will used to populate the dictionary values (see `get_data`, below). - The generated context will be namespaced under the interface type, to prevent - potential naming conflicts. + The generated context will be namespaced under the relation :attr:`name`, + to prevent potential naming conflicts. + + :param str name: Override the relation :attr:`name`, since it can vary from charm to charm + :param list additional_required_keys: Extend the list of :attr:`required_keys` """ name = None interface = None required_keys = [] - def __init__(self, *args, **kwargs): - super(RelationContext, self).__init__(*args, **kwargs) + def __init__(self, name=None, additional_required_keys=None): + if name is not None: + self.name = name + if additional_required_keys is not None: + self.required_keys.extend(additional_required_keys) self.get_data() def __bool__(self): @@ -101,9 +109,115 @@ class RelationContext(dict): return {} +class MysqlRelation(RelationContext): + """ + Relation context for the `mysql` interface. + + :param str name: Override the relation :attr:`name`, since it can vary from charm to charm + :param list additional_required_keys: Extend the list of :attr:`required_keys` + """ + name = 'db' + interface = 'mysql' + required_keys = ['host', 'user', 'password', 'database'] + + +class HttpRelation(RelationContext): + """ + Relation context for the `http` interface. + + :param str name: Override the relation :attr:`name`, since it can vary from charm to charm + :param list additional_required_keys: Extend the list of :attr:`required_keys` + """ + name = 'website' + interface = 'http' + required_keys = ['host', 'port'] + + def provide_data(self): + return { + 'host': hookenv.unit_get('private-address'), + 'port': 80, + } + + +class RequiredConfig(dict): + """ + Data context that loads config options with one or more mandatory options. + + Once the required options have been changed from their default values, all + config options will be available, namespaced under `config` to prevent + potential naming conflicts (for example, between a config option and a + relation property). + + :param list *args: List of options that must be changed from their default values. + """ + + def __init__(self, *args): + self.required_options = args + self['config'] = hookenv.config() + with open(os.path.join(hookenv.charm_dir(), 'config.yaml')) as fp: + self.config = yaml.load(fp).get('options', {}) + + def __bool__(self): + for option in self.required_options: + if option not in self['config']: + return False + current_value = self['config'][option] + default_value = self.config[option].get('default') + if current_value == default_value: + return False + if current_value in (None, '') and default_value in (None, ''): + return False + return True + + def __nonzero__(self): + return self.__bool__() + + +class StoredContext(dict): + """ + A data context that always returns the data that it was first created with. + + This is useful to do a one-time generation of things like passwords, that + will thereafter use the same value that was originally generated, instead + of generating a new value each time it is run. + """ + def __init__(self, file_name, config_data): + """ + If the file exists, populate `self` with the data from the file. + Otherwise, populate with the given data and persist it to the file. + """ + if os.path.exists(file_name): + self.update(self.read_context(file_name)) + else: + self.store_context(file_name, config_data) + self.update(config_data) + + def store_context(self, file_name, config_data): + if not os.path.isabs(file_name): + file_name = os.path.join(hookenv.charm_dir(), file_name) + with open(file_name, 'w') as file_stream: + os.fchmod(file_stream.fileno(), 0600) + yaml.dump(config_data, file_stream) + + def read_context(self, file_name): + if not os.path.isabs(file_name): + file_name = os.path.join(hookenv.charm_dir(), file_name) + with open(file_name, 'r') as file_stream: + data = yaml.load(file_stream) + if not data: + raise OSError("%s is empty" % file_name) + return data + + class TemplateCallback(ManagerCallback): """ - Callback class that will render a template, for use as a ready action. + Callback class that will render a Jinja2 template, for use as a ready action. + + :param str source: The template source file, relative to `$CHARM_DIR/templates` + :param str target: The target to write the rendered template to + :param str owner: The owner of the rendered file + :param str group: The group of the rendered file + :param int perms: The permissions of the rendered file """ def __init__(self, source, target, owner='root', group='root', perms=0444): self.source = source diff --git a/hooks/charmhelpers/fetch/__init__.py b/hooks/charmhelpers/fetch/__init__.py index 8e9d3804..20a20ac6 100644 --- a/hooks/charmhelpers/fetch/__init__.py +++ b/hooks/charmhelpers/fetch/__init__.py @@ -311,22 +311,35 @@ def configure_sources(update=False, apt_update(fatal=True) -def install_remote(source): +def install_remote(source, *args, **kwargs): """ Install a file tree from a remote source The specified source should be a url of the form: scheme://[host]/path[#[option=value][&...]] - Schemes supported are based on this modules submodules - Options supported are submodule-specific""" + Schemes supported are based on this modules submodules. + Options supported are submodule-specific. + Additional arguments are passed through to the submodule. + + For example:: + + dest = install_remote('http://example.com/archive.tgz', + checksum='deadbeef', + hash_type='sha1') + + This will download `archive.tgz`, validate it using SHA1 and, if + the file is ok, extract it and return the directory in which it + was extracted. If the checksum fails, it will raise + :class:`charmhelpers.core.host.ChecksumError`. + """ # We ONLY check for True here because can_handle may return a string # explaining why it can't handle a given source. handlers = [h for h in plugins() if h.can_handle(source) is True] installed_to = None for handler in handlers: try: - installed_to = handler.install(source) + installed_to = handler.install(source, *args, **kwargs) except UnhandledSource: pass if not installed_to: diff --git a/hooks/charmhelpers/fetch/archiveurl.py b/hooks/charmhelpers/fetch/archiveurl.py index 1b11fa03..d1dcbc33 100644 --- a/hooks/charmhelpers/fetch/archiveurl.py +++ b/hooks/charmhelpers/fetch/archiveurl.py @@ -12,21 +12,19 @@ from charmhelpers.payload.archive import ( get_archive_handler, extract, ) -from charmhelpers.core.host import mkdir +from charmhelpers.core.host import mkdir, check_hash -""" -This class is a plugin for charmhelpers.fetch.install_remote. -It grabs, validates and installs remote archives fetched over "http", "https", "ftp" or "file" protocols. The contents of the archive are installed in $CHARM_DIR/fetched/. - -Example usage: -install_remote("https://example.com/some/archive.tar.gz") -# Installs the contents of archive.tar.gz in $CHARM_DIR/fetched/. - -See charmhelpers.fetch.archiveurl.get_archivehandler for supported archive types. -""" class ArchiveUrlFetchHandler(BaseFetchHandler): - """Handler for archives via generic URLs""" + """ + Handler to download archive files from arbitrary URLs. + + Can fetch from http, https, ftp, and file URLs. + + Can install either tarballs (.tar, .tgz, .tbz2, etc) or zip files. + + Installs the contents of the archive in $CHARM_DIR/fetched/. + """ def can_handle(self, source): url_parts = self.parse_url(source) if url_parts.scheme not in ('http', 'https', 'ftp', 'file'): @@ -36,6 +34,12 @@ class ArchiveUrlFetchHandler(BaseFetchHandler): return False def download(self, source, dest): + """ + Download an archive file. + + :param str source: URL pointing to an archive file. + :param str dest: Local path location to download archive file to. + """ # propogate all exceptions # URLError, OSError, etc proto, netloc, path, params, query, fragment = urlparse.urlparse(source) @@ -60,7 +64,29 @@ class ArchiveUrlFetchHandler(BaseFetchHandler): os.unlink(dest) raise e - def install(self, source): + # Mandatory file validation via Sha1 or MD5 hashing. + def download_and_validate(self, url, hashsum, validate="sha1"): + tempfile, headers = urlretrieve(url) + check_hash(tempfile, hashsum, validate) + return tempfile + + def install(self, source, dest=None, checksum=None, hash_type='sha1'): + """ + Download and install an archive file, with optional checksum validation. + + The checksum can also be given on the :param:`source` URL's fragment. + For example:: + + handler.install('http://example.com/file.tgz#sha1=deadbeef') + + :param str source: URL pointing to an archive file. + :param str dest: Local destination path to install to. If not given, + installs to `$CHARM_DIR/archives/archive_file_name`. + :param str checksum: If given, validate the archive file after download. + :param str hash_type: Algorithm used to generate :param:`checksum`. + Can be any hash alrgorithm supported by :mod:`hashlib`, + such as md5, sha1, sha256, sha512, etc. + """ url_parts = self.parse_url(source) dest_dir = os.path.join(os.environ.get('CHARM_DIR'), 'fetched') if not os.path.exists(dest_dir): @@ -72,32 +98,10 @@ class ArchiveUrlFetchHandler(BaseFetchHandler): raise UnhandledSource(e.reason) except OSError as e: raise UnhandledSource(e.strerror) - return extract(dld_file) - - # Mandatory file validation via Sha1 or MD5 hashing. - def download_and_validate(self, url, hashsum, validate="sha1"): - if validate == 'sha1' and len(hashsum) != 40: - raise ValueError("HashSum must be = 40 characters when using sha1" - " validation") - if validate == 'md5' and len(hashsum) != 32: - raise ValueError("HashSum must be = 32 characters when using md5" - " validation") - tempfile, headers = urlretrieve(url) - self.validate_file(tempfile, hashsum, validate) - return tempfile - - # Predicate method that returns status of hash matching expected hash. - def validate_file(self, source, hashsum, vmethod='sha1'): - if vmethod != 'sha1' and vmethod != 'md5': - raise ValueError("Validation Method not supported") - - if vmethod == 'md5': - m = hashlib.md5() - if vmethod == 'sha1': - m = hashlib.sha1() - with open(source) as f: - for line in f: - m.update(line) - if hashsum != m.hexdigest(): - msg = "Hash Mismatch on {} expected {} got {}" - raise ValueError(msg.format(source, hashsum, m.hexdigest())) + options = urlparse.parse_qs(url_parts.fragment) + for key, value in options.items(): + if key in hashlib.algorithms: + check_hash(dld_file, value, key) + if checksum: + check_hash(dld_file, checksum, hash_type) + return extract(dld_file, dest)