From b6b03952b86c3c7b13d092655feb95208858e6f2 Mon Sep 17 00:00:00 2001 From: Andrew Wilkins Date: Mon, 23 Nov 2015 17:12:54 +0800 Subject: [PATCH 1/3] Sync charmhelpers to update storage helpers --- hooks/charmhelpers/core/hookenv.py | 4 ++-- hooks/charmhelpers/fetch/bzrurl.py | 10 +++++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/hooks/charmhelpers/core/hookenv.py b/hooks/charmhelpers/core/hookenv.py index 454b52a..e70f1ca 100644 --- a/hooks/charmhelpers/core/hookenv.py +++ b/hooks/charmhelpers/core/hookenv.py @@ -637,7 +637,7 @@ def unit_private_ip(): @cached -def storage_get(attribute="", storage_id=""): +def storage_get(attribute=None, storage_id=None): """Get storage attributes""" _args = ['storage-get', '--format=json'] if storage_id: @@ -651,7 +651,7 @@ def storage_get(attribute="", storage_id=""): @cached -def storage_list(storage_name=""): +def storage_list(storage_name=None): """List the storage IDs for the unit""" _args = ['storage-list', '--format=json'] if storage_name: diff --git a/hooks/charmhelpers/fetch/bzrurl.py b/hooks/charmhelpers/fetch/bzrurl.py index 3531315..8ec6969 100644 --- a/hooks/charmhelpers/fetch/bzrurl.py +++ b/hooks/charmhelpers/fetch/bzrurl.py @@ -64,11 +64,15 @@ class BzrUrlFetchHandler(BaseFetchHandler): except Exception as e: raise e - def install(self, source): + def install(self, source, dest=None): url_parts = self.parse_url(source) branch_name = url_parts.path.strip("/").split("/")[-1] - dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched", - branch_name) + if dest: + dest_dir = os.path.join(dest, branch_name) + else: + dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched", + branch_name) + if not os.path.exists(dest_dir): mkdir(dest_dir, perms=0o755) try: From 669fd96b11a21fbe0e68b6f632b9c776f2a25b21 Mon Sep 17 00:00:00 2001 From: Andrew Wilkins Date: Mon, 23 Nov 2015 17:13:18 +0800 Subject: [PATCH 2/3] Add osd-devices and osd-journal block storage Add the "osd-devices" block-type storage, with minimum of 0 and no maximum. Volumes assigned to a unit will be added as OSDs. Also, add the "osd-journal" block-type storage, with minimum of 0 and maximum of 1. The osd-journal storage, if supplied, will be used in favour of osd-journal configuration if supplied. For now, handling of osd-journal is static, just as the configuration method was before. Removing or adding the journal after deployment is not currently supported. This is possible with further changes, but requires stopping Ceph and migrating the existing journal, and so out of scope here. --- config.yaml | 3 +- hooks/ceph_hooks.py | 46 +++++++++++++++++++++++------ hooks/osd-devices-storage-attached | 1 + hooks/osd-devices-storage-detaching | 1 + metadata.yaml | 9 ++++++ 5 files changed, 50 insertions(+), 10 deletions(-) create mode 120000 hooks/osd-devices-storage-attached create mode 120000 hooks/osd-devices-storage-detaching diff --git a/config.yaml b/config.yaml index b1880de..891c740 100644 --- a/config.yaml +++ b/config.yaml @@ -40,7 +40,8 @@ options: The devices to format and set up as osd volumes. . These devices are the range of devices that will be checked for and - used across all service units. + used across all service units, in addition to any volumes attached + via the --storage flag during deployment. . For ceph >= 0.56.6 these can also be directories instead of devices - the charm assumes anything not starting with /dev is a directory instead. diff --git a/hooks/ceph_hooks.py b/hooks/ceph_hooks.py index 9d73363..8a041cb 100755 --- a/hooks/ceph_hooks.py +++ b/hooks/ceph_hooks.py @@ -29,6 +29,8 @@ from charmhelpers.core.hookenv import ( relations_of_type, status_set, local_unit, + storage_get, + storage_list ) from charmhelpers.core.host import ( service_restart, @@ -145,7 +147,7 @@ def config_changed(): if e_mountpoint and ceph.filesystem_mounted(e_mountpoint): umount(e_mountpoint) - osd_journal = config('osd-journal') + osd_journal = get_osd_journal() if (osd_journal and not os.path.exists(JOURNAL_ZAPPED) and os.path.exists(osd_journal)): ceph.zap_disk(osd_journal) @@ -158,16 +160,36 @@ def config_changed(): ceph.bootstrap_monitor_cluster(config('monitor-secret')) ceph.wait_for_bootstrap() - if ceph.is_bootstrapped(): - for dev in get_devices(): - ceph.osdize(dev, config('osd-format'), config('osd-journal'), - reformat_osd(), config('ignore-device-errors')) - ceph.start_osds(get_devices()) + storage_changed() if relations_of_type('nrpe-external-master'): update_nrpe_config() +@hooks.hook('osd-devices-storage-attached', 'osd-devices-storage-detaching') +def storage_changed(): + if ceph.is_bootstrapped(): + for dev in get_devices(): + ceph.osdize(dev, config('osd-format'), get_osd_journal(), + reformat_osd(), config('ignore-device-errors')) + ceph.start_osds(get_devices()) + + +def get_osd_journal(): + ''' + Returns the block device path to use for the OSD journal, if any. + + If there is an osd-journal storage instance attached, it will be + used as the journal. Otherwise, the osd-journal configuration will + be returned. + ''' + storage_ids = storage_list('osd-journal') + if storage_ids: + # There can be at most one osd-journal storage instance. + return storage_get('location', storage_ids[0]) + return config('osd-journal') + + def get_mon_hosts(): hosts = [] addr = get_public_addr() @@ -207,9 +229,15 @@ def reformat_osd(): def get_devices(): if config('osd-devices'): - return config('osd-devices').split(' ') + devices = config('osd-devices').split(' ') else: - return [] + devices = [] + # List storage instances for the 'osd-devices' + # store declared for this charm too, and add + # their block device paths to the list. + storage_ids = storage_list('osd-devices') + devices.extend((storage_get('location', s) for s in storage_ids)) + return devices @hooks.hook('mon-relation-joined') @@ -231,7 +259,7 @@ def mon_relation(): ceph.bootstrap_monitor_cluster(config('monitor-secret')) ceph.wait_for_bootstrap() for dev in get_devices(): - ceph.osdize(dev, config('osd-format'), config('osd-journal'), + ceph.osdize(dev, config('osd-format'), get_osd_journal(), reformat_osd(), config('ignore-device-errors')) ceph.start_osds(get_devices()) notify_osds() diff --git a/hooks/osd-devices-storage-attached b/hooks/osd-devices-storage-attached new file mode 120000 index 0000000..52d9663 --- /dev/null +++ b/hooks/osd-devices-storage-attached @@ -0,0 +1 @@ +ceph_hooks.py \ No newline at end of file diff --git a/hooks/osd-devices-storage-detaching b/hooks/osd-devices-storage-detaching new file mode 120000 index 0000000..52d9663 --- /dev/null +++ b/hooks/osd-devices-storage-detaching @@ -0,0 +1 @@ +ceph_hooks.py \ No newline at end of file diff --git a/metadata.yaml b/metadata.yaml index 3c5f926..b884330 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -25,3 +25,12 @@ provides: nrpe-external-master: interface: nrpe-external-master scope: container +storage: + osd-devices: + type: block + multiple: + range: 0- + osd-journal: + type: block + multiple: + range: 0-1 From 4d49a1b96dff7526caf2fef8f197504eef9a39b7 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Mon, 4 Jan 2016 16:25:48 -0500 Subject: [PATCH 3/3] [corey.bryant,r=trivial] Sync charm-helpers. --- .../charmhelpers/contrib/charmsupport/nrpe.py | 14 +- hooks/charmhelpers/contrib/network/ip.py | 40 +- .../contrib/storage/linux/ceph.py | 416 ++++++++++++++++-- hooks/charmhelpers/core/hookenv.py | 44 +- hooks/charmhelpers/core/host.py | 28 +- hooks/charmhelpers/core/services/helpers.py | 16 +- hooks/charmhelpers/core/templating.py | 20 +- hooks/charmhelpers/fetch/__init__.py | 10 +- hooks/charmhelpers/fetch/archiveurl.py | 2 +- hooks/charmhelpers/fetch/bzrurl.py | 44 +- hooks/charmhelpers/fetch/giturl.py | 38 +- .../contrib/openstack/amulet/deployment.py | 8 +- 12 files changed, 558 insertions(+), 122 deletions(-) diff --git a/hooks/charmhelpers/contrib/charmsupport/nrpe.py b/hooks/charmhelpers/contrib/charmsupport/nrpe.py index 65b1a27..2f24642 100644 --- a/hooks/charmhelpers/contrib/charmsupport/nrpe.py +++ b/hooks/charmhelpers/contrib/charmsupport/nrpe.py @@ -296,7 +296,7 @@ def get_nagios_hostcontext(relation_name='nrpe-external-master'): :param str relation_name: Name of relation nrpe sub joined to """ for rel in relations_of_type(relation_name): - if 'nagios_hostname' in rel: + if 'nagios_host_context' in rel: return rel['nagios_host_context'] @@ -337,11 +337,13 @@ def add_init_service_checks(nrpe, services, unit_name): upstart_init = '/etc/init/%s.conf' % svc sysv_init = '/etc/init.d/%s' % svc if os.path.exists(upstart_init): - nrpe.add_check( - shortname=svc, - description='process check {%s}' % unit_name, - check_cmd='check_upstart_job %s' % svc - ) + # Don't add a check for these services from neutron-gateway + if svc not in ['ext-port', 'os-charm-phy-nic-mtu']: + nrpe.add_check( + shortname=svc, + description='process check {%s}' % unit_name, + check_cmd='check_upstart_job %s' % svc + ) elif os.path.exists(sysv_init): cronpath = '/etc/cron.d/nagios-service-check-%s' % svc cron_file = ('*/5 * * * * root ' diff --git a/hooks/charmhelpers/contrib/network/ip.py b/hooks/charmhelpers/contrib/network/ip.py index 7f3b66b..998f00c 100644 --- a/hooks/charmhelpers/contrib/network/ip.py +++ b/hooks/charmhelpers/contrib/network/ip.py @@ -53,7 +53,7 @@ def _validate_cidr(network): def no_ip_found_error_out(network): - errmsg = ("No IP address found in network: %s" % network) + errmsg = ("No IP address found in network(s): %s" % network) raise ValueError(errmsg) @@ -61,7 +61,7 @@ def get_address_in_network(network, fallback=None, fatal=False): """Get an IPv4 or IPv6 address within the network from the host. :param network (str): CIDR presentation format. For example, - '192.168.1.0/24'. + '192.168.1.0/24'. Supports multiple networks as a space-delimited list. :param fallback (str): If no address is found, return fallback. :param fatal (boolean): If no address is found, fallback is not set and fatal is True then exit(1). @@ -75,24 +75,26 @@ def get_address_in_network(network, fallback=None, fatal=False): else: return None - _validate_cidr(network) - network = netaddr.IPNetwork(network) - for iface in netifaces.interfaces(): - addresses = netifaces.ifaddresses(iface) - if network.version == 4 and netifaces.AF_INET in addresses: - addr = addresses[netifaces.AF_INET][0]['addr'] - netmask = addresses[netifaces.AF_INET][0]['netmask'] - cidr = netaddr.IPNetwork("%s/%s" % (addr, netmask)) - if cidr in network: - return str(cidr.ip) + networks = network.split() or [network] + for network in networks: + _validate_cidr(network) + network = netaddr.IPNetwork(network) + for iface in netifaces.interfaces(): + addresses = netifaces.ifaddresses(iface) + if network.version == 4 and netifaces.AF_INET in addresses: + addr = addresses[netifaces.AF_INET][0]['addr'] + netmask = addresses[netifaces.AF_INET][0]['netmask'] + cidr = netaddr.IPNetwork("%s/%s" % (addr, netmask)) + if cidr in network: + return str(cidr.ip) - if network.version == 6 and netifaces.AF_INET6 in addresses: - for addr in addresses[netifaces.AF_INET6]: - if not addr['addr'].startswith('fe80'): - cidr = netaddr.IPNetwork("%s/%s" % (addr['addr'], - addr['netmask'])) - if cidr in network: - return str(cidr.ip) + if network.version == 6 and netifaces.AF_INET6 in addresses: + for addr in addresses[netifaces.AF_INET6]: + if not addr['addr'].startswith('fe80'): + cidr = netaddr.IPNetwork("%s/%s" % (addr['addr'], + addr['netmask'])) + if cidr in network: + return str(cidr.ip) if fallback is not None: return fallback diff --git a/hooks/charmhelpers/contrib/storage/linux/ceph.py b/hooks/charmhelpers/contrib/storage/linux/ceph.py index 1235389..60ae52b 100644 --- a/hooks/charmhelpers/contrib/storage/linux/ceph.py +++ b/hooks/charmhelpers/contrib/storage/linux/ceph.py @@ -23,10 +23,11 @@ # James Page # Adam Gandelman # +import bisect +import six import os import shutil -import six import json import time import uuid @@ -73,6 +74,394 @@ log to syslog = {use_syslog} err to syslog = {use_syslog} clog to syslog = {use_syslog} """ +# For 50 < osds < 240,000 OSDs (Roughly 1 Exabyte at 6T OSDs) +powers_of_two = [8192, 16384, 32768, 65536, 131072, 262144, 524288, 1048576, 2097152, 4194304, 8388608] + + +def validator(value, valid_type, valid_range=None): + """ + Used to validate these: http://docs.ceph.com/docs/master/rados/operations/pools/#set-pool-values + Example input: + validator(value=1, + valid_type=int, + valid_range=[0, 2]) + This says I'm testing value=1. It must be an int inclusive in [0,2] + + :param value: The value to validate + :param valid_type: The type that value should be. + :param valid_range: A range of values that value can assume. + :return: + """ + assert isinstance(value, valid_type), "{} is not a {}".format( + value, + valid_type) + if valid_range is not None: + assert isinstance(valid_range, list), \ + "valid_range must be a list, was given {}".format(valid_range) + # If we're dealing with strings + if valid_type is six.string_types: + assert value in valid_range, \ + "{} is not in the list {}".format(value, valid_range) + # Integer, float should have a min and max + else: + if len(valid_range) != 2: + raise ValueError( + "Invalid valid_range list of {} for {}. " + "List must be [min,max]".format(valid_range, value)) + assert value >= valid_range[0], \ + "{} is less than minimum allowed value of {}".format( + value, valid_range[0]) + assert value <= valid_range[1], \ + "{} is greater than maximum allowed value of {}".format( + value, valid_range[1]) + + +class PoolCreationError(Exception): + """ + A custom error to inform the caller that a pool creation failed. Provides an error message + """ + def __init__(self, message): + super(PoolCreationError, self).__init__(message) + + +class Pool(object): + """ + An object oriented approach to Ceph pool creation. This base class is inherited by ReplicatedPool and ErasurePool. + Do not call create() on this base class as it will not do anything. Instantiate a child class and call create(). + """ + def __init__(self, service, name): + self.service = service + self.name = name + + # Create the pool if it doesn't exist already + # To be implemented by subclasses + def create(self): + pass + + def add_cache_tier(self, cache_pool, mode): + """ + Adds a new cache tier to an existing pool. + :param cache_pool: six.string_types. The cache tier pool name to add. + :param mode: six.string_types. The caching mode to use for this pool. valid range = ["readonly", "writeback"] + :return: None + """ + # Check the input types and values + validator(value=cache_pool, valid_type=six.string_types) + validator(value=mode, valid_type=six.string_types, valid_range=["readonly", "writeback"]) + + check_call(['ceph', '--id', self.service, 'osd', 'tier', 'add', self.name, cache_pool]) + check_call(['ceph', '--id', self.service, 'osd', 'tier', 'cache-mode', cache_pool, mode]) + check_call(['ceph', '--id', self.service, 'osd', 'tier', 'set-overlay', self.name, cache_pool]) + check_call(['ceph', '--id', self.service, 'osd', 'pool', 'set', cache_pool, 'hit_set_type', 'bloom']) + + def remove_cache_tier(self, cache_pool): + """ + Removes a cache tier from Ceph. Flushes all dirty objects from writeback pools and waits for that to complete. + :param cache_pool: six.string_types. The cache tier pool name to remove. + :return: None + """ + # read-only is easy, writeback is much harder + mode = get_cache_mode(cache_pool) + if mode == 'readonly': + check_call(['ceph', '--id', self.service, 'osd', 'tier', 'cache-mode', cache_pool, 'none']) + check_call(['ceph', '--id', self.service, 'osd', 'tier', 'remove', self.name, cache_pool]) + + elif mode == 'writeback': + check_call(['ceph', '--id', self.service, 'osd', 'tier', 'cache-mode', cache_pool, 'forward']) + # Flush the cache and wait for it to return + check_call(['ceph', '--id', self.service, '-p', cache_pool, 'cache-flush-evict-all']) + check_call(['ceph', '--id', self.service, 'osd', 'tier', 'remove-overlay', self.name]) + check_call(['ceph', '--id', self.service, 'osd', 'tier', 'remove', self.name, cache_pool]) + + def get_pgs(self, pool_size): + """ + :param pool_size: int. pool_size is either the number of replicas for replicated pools or the K+M sum for + erasure coded pools + :return: int. The number of pgs to use. + """ + validator(value=pool_size, valid_type=int) + osds = get_osds(self.service) + if not osds: + # NOTE(james-page): Default to 200 for older ceph versions + # which don't support OSD query from cli + return 200 + + # Calculate based on Ceph best practices + if osds < 5: + return 128 + elif 5 < osds < 10: + return 512 + elif 10 < osds < 50: + return 4096 + else: + estimate = (osds * 100) / pool_size + # Return the next nearest power of 2 + index = bisect.bisect_right(powers_of_two, estimate) + return powers_of_two[index] + + +class ReplicatedPool(Pool): + def __init__(self, service, name, replicas=2): + super(ReplicatedPool, self).__init__(service=service, name=name) + self.replicas = replicas + + def create(self): + if not pool_exists(self.service, self.name): + # Create it + pgs = self.get_pgs(self.replicas) + cmd = ['ceph', '--id', self.service, 'osd', 'pool', 'create', self.name, str(pgs)] + try: + check_call(cmd) + except CalledProcessError: + raise + + +# Default jerasure erasure coded pool +class ErasurePool(Pool): + def __init__(self, service, name, erasure_code_profile="default"): + super(ErasurePool, self).__init__(service=service, name=name) + self.erasure_code_profile = erasure_code_profile + + def create(self): + if not pool_exists(self.service, self.name): + # Try to find the erasure profile information so we can properly size the pgs + erasure_profile = get_erasure_profile(service=self.service, name=self.erasure_code_profile) + + # Check for errors + if erasure_profile is None: + log(message='Failed to discover erasure_profile named={}'.format(self.erasure_code_profile), + level=ERROR) + raise PoolCreationError(message='unable to find erasure profile {}'.format(self.erasure_code_profile)) + if 'k' not in erasure_profile or 'm' not in erasure_profile: + # Error + log(message='Unable to find k (data chunks) or m (coding chunks) in {}'.format(erasure_profile), + level=ERROR) + raise PoolCreationError( + message='unable to find k (data chunks) or m (coding chunks) in {}'.format(erasure_profile)) + + pgs = self.get_pgs(int(erasure_profile['k']) + int(erasure_profile['m'])) + # Create it + cmd = ['ceph', '--id', self.service, 'osd', 'pool', 'create', self.name, str(pgs), + 'erasure', self.erasure_code_profile] + try: + check_call(cmd) + except CalledProcessError: + raise + + """Get an existing erasure code profile if it already exists. + Returns json formatted output""" + + +def get_erasure_profile(service, name): + """ + :param service: six.string_types. The Ceph user name to run the command under + :param name: + :return: + """ + try: + out = check_output(['ceph', '--id', service, + 'osd', 'erasure-code-profile', 'get', + name, '--format=json']) + return json.loads(out) + except (CalledProcessError, OSError, ValueError): + return None + + +def pool_set(service, pool_name, key, value): + """ + Sets a value for a RADOS pool in ceph. + :param service: six.string_types. The Ceph user name to run the command under + :param pool_name: six.string_types + :param key: six.string_types + :param value: + :return: None. Can raise CalledProcessError + """ + cmd = ['ceph', '--id', service, 'osd', 'pool', 'set', pool_name, key, value] + try: + check_call(cmd) + except CalledProcessError: + raise + + +def snapshot_pool(service, pool_name, snapshot_name): + """ + Snapshots a RADOS pool in ceph. + :param service: six.string_types. The Ceph user name to run the command under + :param pool_name: six.string_types + :param snapshot_name: six.string_types + :return: None. Can raise CalledProcessError + """ + cmd = ['ceph', '--id', service, 'osd', 'pool', 'mksnap', pool_name, snapshot_name] + try: + check_call(cmd) + except CalledProcessError: + raise + + +def remove_pool_snapshot(service, pool_name, snapshot_name): + """ + Remove a snapshot from a RADOS pool in ceph. + :param service: six.string_types. The Ceph user name to run the command under + :param pool_name: six.string_types + :param snapshot_name: six.string_types + :return: None. Can raise CalledProcessError + """ + cmd = ['ceph', '--id', service, 'osd', 'pool', 'rmsnap', pool_name, snapshot_name] + try: + check_call(cmd) + except CalledProcessError: + raise + + +# max_bytes should be an int or long +def set_pool_quota(service, pool_name, max_bytes): + """ + :param service: six.string_types. The Ceph user name to run the command under + :param pool_name: six.string_types + :param max_bytes: int or long + :return: None. Can raise CalledProcessError + """ + # Set a byte quota on a RADOS pool in ceph. + cmd = ['ceph', '--id', service, 'osd', 'pool', 'set-quota', pool_name, 'max_bytes', max_bytes] + try: + check_call(cmd) + except CalledProcessError: + raise + + +def remove_pool_quota(service, pool_name): + """ + Set a byte quota on a RADOS pool in ceph. + :param service: six.string_types. The Ceph user name to run the command under + :param pool_name: six.string_types + :return: None. Can raise CalledProcessError + """ + cmd = ['ceph', '--id', service, 'osd', 'pool', 'set-quota', pool_name, 'max_bytes', '0'] + try: + check_call(cmd) + except CalledProcessError: + raise + + +def create_erasure_profile(service, profile_name, erasure_plugin_name='jerasure', failure_domain='host', + data_chunks=2, coding_chunks=1, + locality=None, durability_estimator=None): + """ + Create a new erasure code profile if one does not already exist for it. Updates + the profile if it exists. Please see http://docs.ceph.com/docs/master/rados/operations/erasure-code-profile/ + for more details + :param service: six.string_types. The Ceph user name to run the command under + :param profile_name: six.string_types + :param erasure_plugin_name: six.string_types + :param failure_domain: six.string_types. One of ['chassis', 'datacenter', 'host', 'osd', 'pdu', 'pod', 'rack', 'region', + 'room', 'root', 'row']) + :param data_chunks: int + :param coding_chunks: int + :param locality: int + :param durability_estimator: int + :return: None. Can raise CalledProcessError + """ + # Ensure this failure_domain is allowed by Ceph + validator(failure_domain, six.string_types, + ['chassis', 'datacenter', 'host', 'osd', 'pdu', 'pod', 'rack', 'region', 'room', 'root', 'row']) + + cmd = ['ceph', '--id', service, 'osd', 'erasure-code-profile', 'set', profile_name, + 'plugin=' + erasure_plugin_name, 'k=' + str(data_chunks), 'm=' + str(coding_chunks), + 'ruleset_failure_domain=' + failure_domain] + if locality is not None and durability_estimator is not None: + raise ValueError("create_erasure_profile should be called with k, m and one of l or c but not both.") + + # Add plugin specific information + if locality is not None: + # For local erasure codes + cmd.append('l=' + str(locality)) + if durability_estimator is not None: + # For Shec erasure codes + cmd.append('c=' + str(durability_estimator)) + + if erasure_profile_exists(service, profile_name): + cmd.append('--force') + + try: + check_call(cmd) + except CalledProcessError: + raise + + +def rename_pool(service, old_name, new_name): + """ + Rename a Ceph pool from old_name to new_name + :param service: six.string_types. The Ceph user name to run the command under + :param old_name: six.string_types + :param new_name: six.string_types + :return: None + """ + validator(value=old_name, valid_type=six.string_types) + validator(value=new_name, valid_type=six.string_types) + + cmd = ['ceph', '--id', service, 'osd', 'pool', 'rename', old_name, new_name] + check_call(cmd) + + +def erasure_profile_exists(service, name): + """ + Check to see if an Erasure code profile already exists. + :param service: six.string_types. The Ceph user name to run the command under + :param name: six.string_types + :return: int or None + """ + validator(value=name, valid_type=six.string_types) + try: + check_call(['ceph', '--id', service, + 'osd', 'erasure-code-profile', 'get', + name]) + return True + except CalledProcessError: + return False + + +def get_cache_mode(service, pool_name): + """ + Find the current caching mode of the pool_name given. + :param service: six.string_types. The Ceph user name to run the command under + :param pool_name: six.string_types + :return: int or None + """ + validator(value=service, valid_type=six.string_types) + validator(value=pool_name, valid_type=six.string_types) + out = check_output(['ceph', '--id', service, 'osd', 'dump', '--format=json']) + try: + osd_json = json.loads(out) + for pool in osd_json['pools']: + if pool['pool_name'] == pool_name: + return pool['cache_mode'] + return None + except ValueError: + raise + + +def pool_exists(service, name): + """Check to see if a RADOS pool already exists.""" + try: + out = check_output(['rados', '--id', service, + 'lspools']).decode('UTF-8') + except CalledProcessError: + return False + + return name in out + + +def get_osds(service): + """Return a list of all Ceph Object Storage Daemons currently in the + cluster. + """ + version = ceph_version() + if version and version >= '0.56': + return json.loads(check_output(['ceph', '--id', service, + 'osd', 'ls', + '--format=json']).decode('UTF-8')) + + return None def install(): @@ -102,30 +491,6 @@ def create_rbd_image(service, pool, image, sizemb): check_call(cmd) -def pool_exists(service, name): - """Check to see if a RADOS pool already exists.""" - try: - out = check_output(['rados', '--id', service, - 'lspools']).decode('UTF-8') - except CalledProcessError: - return False - - return name in out - - -def get_osds(service): - """Return a list of all Ceph Object Storage Daemons currently in the - cluster. - """ - version = ceph_version() - if version and version >= '0.56': - return json.loads(check_output(['ceph', '--id', service, - 'osd', 'ls', - '--format=json']).decode('UTF-8')) - - return None - - def update_pool(client, pool, settings): cmd = ['ceph', '--id', client, 'osd', 'pool', 'set', pool] for k, v in six.iteritems(settings): @@ -414,6 +779,7 @@ class CephBrokerRq(object): The API is versioned and defaults to version 1. """ + def __init__(self, api_version=1, request_id=None): self.api_version = api_version if request_id: diff --git a/hooks/charmhelpers/core/hookenv.py b/hooks/charmhelpers/core/hookenv.py index e70f1ca..2dd70bc 100644 --- a/hooks/charmhelpers/core/hookenv.py +++ b/hooks/charmhelpers/core/hookenv.py @@ -492,7 +492,7 @@ def relation_types(): @cached def peer_relation_id(): - '''Get a peer relation id if a peer relation has been joined, else None.''' + '''Get the peers relation id if a peers relation has been joined, else None.''' md = metadata() section = md.get('peers') if section: @@ -517,12 +517,12 @@ def relation_to_interface(relation_name): def relation_to_role_and_interface(relation_name): """ Given the name of a relation, return the role and the name of the interface - that relation uses (where role is one of ``provides``, ``requires``, or ``peer``). + that relation uses (where role is one of ``provides``, ``requires``, or ``peers``). :returns: A tuple containing ``(role, interface)``, or ``(None, None)``. """ _metadata = metadata() - for role in ('provides', 'requires', 'peer'): + for role in ('provides', 'requires', 'peers'): interface = _metadata.get(role, {}).get(relation_name, {}).get('interface') if interface: return role, interface @@ -534,7 +534,7 @@ def role_and_interface_to_relations(role, interface_name): """ Given a role and interface name, return a list of relation names for the current charm that use that interface under that role (where role is one - of ``provides``, ``requires``, or ``peer``). + of ``provides``, ``requires``, or ``peers``). :returns: A list of relation names. """ @@ -555,7 +555,7 @@ def interface_to_relations(interface_name): :returns: A list of relation names. """ results = [] - for role in ('provides', 'requires', 'peer'): + for role in ('provides', 'requires', 'peers'): results.extend(role_and_interface_to_relations(role, interface_name)) return results @@ -878,6 +878,40 @@ def leader_set(settings=None, **kwargs): subprocess.check_call(cmd) +@translate_exc(from_exc=OSError, to_exc=NotImplementedError) +def payload_register(ptype, klass, pid): + """ is used while a hook is running to let Juju know that a + payload has been started.""" + cmd = ['payload-register'] + for x in [ptype, klass, pid]: + cmd.append(x) + subprocess.check_call(cmd) + + +@translate_exc(from_exc=OSError, to_exc=NotImplementedError) +def payload_unregister(klass, pid): + """ is used while a hook is running to let Juju know + that a payload has been manually stopped. The and provided + must match a payload that has been previously registered with juju using + payload-register.""" + cmd = ['payload-unregister'] + for x in [klass, pid]: + cmd.append(x) + subprocess.check_call(cmd) + + +@translate_exc(from_exc=OSError, to_exc=NotImplementedError) +def payload_status_set(klass, pid, status): + """is used to update the current status of a registered payload. + The and provided must match a payload that has been previously + registered with juju using payload-register. The must be one of the + follow: starting, started, stopping, stopped""" + cmd = ['payload-status-set'] + for x in [klass, pid, status]: + cmd.append(x) + subprocess.check_call(cmd) + + @cached def juju_version(): """Full version string (eg. '1.23.3.1-trusty-amd64')""" diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py index 579871b..c5fd81c 100644 --- a/hooks/charmhelpers/core/host.py +++ b/hooks/charmhelpers/core/host.py @@ -146,8 +146,22 @@ def service_available(service_name): return True -def adduser(username, password=None, shell='/bin/bash', system_user=False): - """Add a user to the system""" +def adduser(username, password=None, shell='/bin/bash', system_user=False, + primary_group=None, secondary_groups=None): + """ + Add a user to the system. + + Will log but otherwise succeed if the user already exists. + + :param str username: Username to create + :param str password: Password for user; if ``None``, create a system user + :param str shell: The default shell for the user + :param bool system_user: Whether to create a login or system user + :param str primary_group: Primary group for user; defaults to their username + :param list secondary_groups: Optional list of additional groups + + :returns: The password database entry struct, as returned by `pwd.getpwnam` + """ try: user_info = pwd.getpwnam(username) log('user {0} already exists!'.format(username)) @@ -162,6 +176,16 @@ def adduser(username, password=None, shell='/bin/bash', system_user=False): '--shell', shell, '--password', password, ]) + if not primary_group: + try: + grp.getgrnam(username) + primary_group = username # avoid "group exists" error + except KeyError: + pass + if primary_group: + cmd.extend(['-g', primary_group]) + if secondary_groups: + cmd.extend(['-G', ','.join(secondary_groups)]) cmd.append(username) subprocess.check_call(cmd) user_info = pwd.getpwnam(username) diff --git a/hooks/charmhelpers/core/services/helpers.py b/hooks/charmhelpers/core/services/helpers.py index 12d768e..2423704 100644 --- a/hooks/charmhelpers/core/services/helpers.py +++ b/hooks/charmhelpers/core/services/helpers.py @@ -243,13 +243,15 @@ class TemplateCallback(ManagerCallback): :param str source: The template source file, relative to `$CHARM_DIR/templates` - :param str target: The target to write the rendered template to + :param str target: The target to write the rendered template to (or None) :param str owner: The owner of the rendered file :param str group: The group of the rendered file :param int perms: The permissions of the rendered file :param partial on_change_action: functools partial to be executed when rendered file changes :param jinja2 loader template_loader: A jinja2 template loader + + :return str: The rendered template """ def __init__(self, source, target, owner='root', group='root', perms=0o444, @@ -267,12 +269,14 @@ class TemplateCallback(ManagerCallback): if self.on_change_action and os.path.isfile(self.target): pre_checksum = host.file_hash(self.target) service = manager.get_service(service_name) - context = {} + context = {'ctx': {}} for ctx in service.get('required_data', []): context.update(ctx) - templating.render(self.source, self.target, context, - self.owner, self.group, self.perms, - template_loader=self.template_loader) + context['ctx'].update(ctx) + + result = templating.render(self.source, self.target, context, + self.owner, self.group, self.perms, + template_loader=self.template_loader) if self.on_change_action: if pre_checksum == host.file_hash(self.target): hookenv.log( @@ -281,6 +285,8 @@ class TemplateCallback(ManagerCallback): else: self.on_change_action() + return result + # Convenience aliases for templates render_template = template = TemplateCallback diff --git a/hooks/charmhelpers/core/templating.py b/hooks/charmhelpers/core/templating.py index 239719d..d2d8eaf 100644 --- a/hooks/charmhelpers/core/templating.py +++ b/hooks/charmhelpers/core/templating.py @@ -27,7 +27,8 @@ def render(source, target, context, owner='root', group='root', The `source` path, if not absolute, is relative to the `templates_dir`. - The `target` path should be absolute. + The `target` path should be absolute. It can also be `None`, in which + case no file will be written. The context should be a dict containing the values to be replaced in the template. @@ -36,6 +37,9 @@ def render(source, target, context, owner='root', group='root', If omitted, `templates_dir` defaults to the `templates` folder in the charm. + The rendered template will be written to the file as well as being returned + as a string. + Note: Using this requires python-jinja2; if it is not installed, calling this will attempt to use charmhelpers.fetch.apt_install to install it. """ @@ -67,9 +71,11 @@ def render(source, target, context, owner='root', group='root', level=hookenv.ERROR) raise e content = template.render(context) - target_dir = os.path.dirname(target) - if not os.path.exists(target_dir): - # This is a terrible default directory permission, as the file - # or its siblings will often contain secrets. - host.mkdir(os.path.dirname(target), owner, group, perms=0o755) - host.write_file(target, content.encode(encoding), owner, group, perms) + if target is not None: + target_dir = os.path.dirname(target) + if not os.path.exists(target_dir): + # This is a terrible default directory permission, as the file + # or its siblings will often contain secrets. + host.mkdir(os.path.dirname(target), owner, group, perms=0o755) + host.write_file(target, content.encode(encoding), owner, group, perms) + return content diff --git a/hooks/charmhelpers/fetch/__init__.py b/hooks/charmhelpers/fetch/__init__.py index 5f831c3..db0d86a 100644 --- a/hooks/charmhelpers/fetch/__init__.py +++ b/hooks/charmhelpers/fetch/__init__.py @@ -98,6 +98,14 @@ CLOUD_ARCHIVE_POCKETS = { 'liberty/proposed': 'trusty-proposed/liberty', 'trusty-liberty/proposed': 'trusty-proposed/liberty', 'trusty-proposed/liberty': 'trusty-proposed/liberty', + # Mitaka + 'mitaka': 'trusty-updates/mitaka', + 'trusty-mitaka': 'trusty-updates/mitaka', + 'trusty-mitaka/updates': 'trusty-updates/mitaka', + 'trusty-updates/mitaka': 'trusty-updates/mitaka', + 'mitaka/proposed': 'trusty-proposed/mitaka', + 'trusty-mitaka/proposed': 'trusty-proposed/mitaka', + 'trusty-proposed/mitaka': 'trusty-proposed/mitaka', } # The order of this list is very important. Handlers should be listed in from @@ -411,7 +419,7 @@ def plugins(fetch_handlers=None): importlib.import_module(package), classname) plugin_list.append(handler_class()) - except (ImportError, AttributeError): + except NotImplementedError: # Skip missing plugins so that they can be ommitted from # installation if desired log("FetchHandler {} not found, skipping plugin".format( diff --git a/hooks/charmhelpers/fetch/archiveurl.py b/hooks/charmhelpers/fetch/archiveurl.py index efd7f9f..b8e0943 100644 --- a/hooks/charmhelpers/fetch/archiveurl.py +++ b/hooks/charmhelpers/fetch/archiveurl.py @@ -108,7 +108,7 @@ class ArchiveUrlFetchHandler(BaseFetchHandler): install_opener(opener) response = urlopen(source) try: - with open(dest, 'w') as dest_file: + with open(dest, 'wb') as dest_file: dest_file.write(response.read()) except Exception as e: if os.path.isfile(dest): diff --git a/hooks/charmhelpers/fetch/bzrurl.py b/hooks/charmhelpers/fetch/bzrurl.py index 8ec6969..cafd27f 100644 --- a/hooks/charmhelpers/fetch/bzrurl.py +++ b/hooks/charmhelpers/fetch/bzrurl.py @@ -15,54 +15,40 @@ # along with charm-helpers. If not, see . import os +from subprocess import check_call from charmhelpers.fetch import ( BaseFetchHandler, - UnhandledSource + UnhandledSource, + filter_installed_packages, + apt_install, ) from charmhelpers.core.host import mkdir -import six -if six.PY3: - raise ImportError('bzrlib does not support Python3') -try: - from bzrlib.branch import Branch - from bzrlib import bzrdir, workingtree, errors -except ImportError: - from charmhelpers.fetch import apt_install - apt_install("python-bzrlib") - from bzrlib.branch import Branch - from bzrlib import bzrdir, workingtree, errors +if filter_installed_packages(['bzr']) != []: + apt_install(['bzr']) + if filter_installed_packages(['bzr']) != []: + raise NotImplementedError('Unable to install bzr') class BzrUrlFetchHandler(BaseFetchHandler): """Handler for bazaar branches via generic and lp URLs""" def can_handle(self, source): url_parts = self.parse_url(source) - if url_parts.scheme not in ('bzr+ssh', 'lp'): + if url_parts.scheme not in ('bzr+ssh', 'lp', ''): return False + elif not url_parts.scheme: + return os.path.exists(os.path.join(source, '.bzr')) else: return True def branch(self, source, dest): - url_parts = self.parse_url(source) - # If we use lp:branchname scheme we need to load plugins if not self.can_handle(source): raise UnhandledSource("Cannot handle {}".format(source)) - if url_parts.scheme == "lp": - from bzrlib.plugin import load_plugins - load_plugins() - try: - local_branch = bzrdir.BzrDir.create_branch_convenience(dest) - except errors.AlreadyControlDirError: - local_branch = Branch.open(dest) - try: - remote_branch = Branch.open(source) - remote_branch.push(local_branch) - tree = workingtree.WorkingTree.open(dest) - tree.update() - except Exception as e: - raise e + if os.path.exists(dest): + check_call(['bzr', 'pull', '--overwrite', '-d', dest, source]) + else: + check_call(['bzr', 'branch', source, dest]) def install(self, source, dest=None): url_parts = self.parse_url(source) diff --git a/hooks/charmhelpers/fetch/giturl.py b/hooks/charmhelpers/fetch/giturl.py index f023b26..bbf89d5 100644 --- a/hooks/charmhelpers/fetch/giturl.py +++ b/hooks/charmhelpers/fetch/giturl.py @@ -15,24 +15,19 @@ # along with charm-helpers. If not, see . import os +from subprocess import check_call from charmhelpers.fetch import ( BaseFetchHandler, - UnhandledSource + UnhandledSource, + filter_installed_packages, + apt_install, ) from charmhelpers.core.host import mkdir -import six -if six.PY3: - raise ImportError('GitPython does not support Python 3') - -try: - from git import Repo -except ImportError: - from charmhelpers.fetch import apt_install - apt_install("python-git") - from git import Repo - -from git.exc import GitCommandError # noqa E402 +if filter_installed_packages(['git']) != []: + apt_install(['git']) + if filter_installed_packages(['git']) != []: + raise NotImplementedError('Unable to install git') class GitUrlFetchHandler(BaseFetchHandler): @@ -40,19 +35,24 @@ class GitUrlFetchHandler(BaseFetchHandler): def can_handle(self, source): url_parts = self.parse_url(source) # TODO (mattyw) no support for ssh git@ yet - if url_parts.scheme not in ('http', 'https', 'git'): + if url_parts.scheme not in ('http', 'https', 'git', ''): return False + elif not url_parts.scheme: + return os.path.exists(os.path.join(source, '.git')) else: return True - def clone(self, source, dest, branch, depth=None): + def clone(self, source, dest, branch="master", depth=None): if not self.can_handle(source): raise UnhandledSource("Cannot handle {}".format(source)) - if depth: - Repo.clone_from(source, dest, branch=branch, depth=depth) + if os.path.exists(dest): + cmd = ['git', '-C', dest, 'pull', source, branch] else: - Repo.clone_from(source, dest, branch=branch) + cmd = ['git', 'clone', source, dest, '--branch', branch] + if depth: + cmd.extend(['--depth', depth]) + check_call(cmd) def install(self, source, branch="master", dest=None, depth=None): url_parts = self.parse_url(source) @@ -66,8 +66,6 @@ class GitUrlFetchHandler(BaseFetchHandler): mkdir(dest_dir, perms=0o755) try: self.clone(source, dest_dir, branch, depth) - except GitCommandError as e: - raise UnhandledSource(e) except OSError as e: raise UnhandledSource(e.strerror) return dest_dir diff --git a/tests/charmhelpers/contrib/openstack/amulet/deployment.py b/tests/charmhelpers/contrib/openstack/amulet/deployment.py index 0506491..58b1a79 100644 --- a/tests/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/tests/charmhelpers/contrib/openstack/amulet/deployment.py @@ -225,7 +225,8 @@ class OpenStackAmuletDeployment(AmuletDeployment): self.precise_havana, self.precise_icehouse, self.trusty_icehouse, self.trusty_juno, self.utopic_juno, self.trusty_kilo, self.vivid_kilo, self.trusty_liberty, - self.wily_liberty) = range(12) + self.wily_liberty, self.trusty_mitaka, + self.xenial_mitaka) = range(14) releases = { ('precise', None): self.precise_essex, @@ -237,9 +238,11 @@ class OpenStackAmuletDeployment(AmuletDeployment): ('trusty', 'cloud:trusty-juno'): self.trusty_juno, ('trusty', 'cloud:trusty-kilo'): self.trusty_kilo, ('trusty', 'cloud:trusty-liberty'): self.trusty_liberty, + ('trusty', 'cloud:trusty-mitaka'): self.trusty_mitaka, ('utopic', None): self.utopic_juno, ('vivid', None): self.vivid_kilo, - ('wily', None): self.wily_liberty} + ('wily', None): self.wily_liberty, + ('xenial', None): self.xenial_mitaka} return releases[(self.series, self.openstack)] def _get_openstack_release_string(self): @@ -256,6 +259,7 @@ class OpenStackAmuletDeployment(AmuletDeployment): ('utopic', 'juno'), ('vivid', 'kilo'), ('wily', 'liberty'), + ('xenial', 'mitaka'), ]) if self.openstack: os_origin = self.openstack.split(':')[1]