From f84e71392c32931e2d0e5c7e4bc1b4b704d80808 Mon Sep 17 00:00:00 2001 From: James Page Date: Thu, 6 Aug 2020 06:54:41 +0100 Subject: [PATCH] Use charms.ceph for Ceph broker Drop use of local copy of ceph_broker.py in preference to the centrally maintained copy in charms.ceph. Change-Id: I89aa0f9fc7d5d2d480ebabc1cb17a86dcbef21bf --- Makefile | 7 + charmhelpers/contrib/storage/linux/ceph.py | 1407 ++++--- hooks/ceph_hooks.py | 4 +- hooks/install | 3 +- hooks/install_deps | 18 + hooks/upgrade-charm | 6 + lib/.keep | 3 - lib/charms_ceph/__init__.py | 0 .../charms_ceph/broker.py | 840 +++-- lib/charms_ceph/crush_utils.py | 154 + lib/charms_ceph/utils.py | 3349 +++++++++++++++++ unit_tests/test_ceph_broker.py | 136 - 12 files changed, 5103 insertions(+), 824 deletions(-) create mode 100755 hooks/install_deps create mode 100755 hooks/upgrade-charm delete mode 100644 lib/.keep create mode 100644 lib/charms_ceph/__init__.py rename hooks/ceph_broker.py => lib/charms_ceph/broker.py (53%) create mode 100644 lib/charms_ceph/crush_utils.py create mode 100644 lib/charms_ceph/utils.py delete mode 100644 unit_tests/test_ceph_broker.py diff --git a/Makefile b/Makefile index 39458a3..09b701f 100644 --- a/Makefile +++ b/Makefile @@ -18,3 +18,10 @@ bin/charm_helpers_sync.py: sync: bin/charm_helpers_sync.py $(PYTHON) bin/charm_helpers_sync.py -c charm-helpers-hooks.yaml + +bin/git_sync.py: + @mkdir -p bin + @wget -O bin/git_sync.py https://raw.githubusercontent.com/CanonicalLtd/git-sync/master/git_sync.py + +ceph-sync: bin/git_sync.py + $(PYTHON) bin/git_sync.py -d lib -s https://github.com/openstack/charms.ceph.git diff --git a/charmhelpers/contrib/storage/linux/ceph.py b/charmhelpers/contrib/storage/linux/ceph.py index 814d5c7..526b95a 100644 --- a/charmhelpers/contrib/storage/linux/ceph.py +++ b/charmhelpers/contrib/storage/linux/ceph.py @@ -39,6 +39,7 @@ from subprocess import ( check_output, CalledProcessError, ) +from charmhelpers import deprecate from charmhelpers.core.hookenv import ( config, service_name, @@ -178,94 +179,293 @@ def send_osd_settings(): def validator(value, valid_type, valid_range=None): - """ - Used to validate these: http://docs.ceph.com/docs/master/rados/operations/pools/#set-pool-values + """Helper function for type validation. + + Used to validate these: + https://docs.ceph.com/docs/master/rados/operations/pools/#set-pool-values + https://docs.ceph.com/docs/master/rados/configuration/bluestore-config-ref/#inline-compression + Example input: validator(value=1, valid_type=int, valid_range=[0, 2]) + This says I'm testing value=1. It must be an int inclusive in [0,2] - :param value: The value to validate + :param value: The value to validate. + :type value: any :param valid_type: The type that value should be. + :type valid_type: any :param valid_range: A range of values that value can assume. - :return: + :type valid_range: Optional[Union[List,Tuple]] + :raises: AssertionError, ValueError """ - assert isinstance(value, valid_type), "{} is not a {}".format( - value, - valid_type) + assert isinstance(value, valid_type), ( + "{} is not a {}".format(value, valid_type)) if valid_range is not None: - assert isinstance(valid_range, list), \ - "valid_range must be a list, was given {}".format(valid_range) + assert isinstance( + valid_range, list) or isinstance(valid_range, tuple), ( + "valid_range must be of type List or Tuple, " + "was given {} of type {}" + .format(valid_range, type(valid_range))) # If we're dealing with strings if isinstance(value, six.string_types): - assert value in valid_range, \ - "{} is not in the list {}".format(value, valid_range) + assert value in valid_range, ( + "{} is not in the list {}".format(value, valid_range)) # Integer, float should have a min and max else: if len(valid_range) != 2: raise ValueError( - "Invalid valid_range list of {} for {}. " + "Invalid valid_range list of {} for {}. " "List must be [min,max]".format(valid_range, value)) - assert value >= valid_range[0], \ - "{} is less than minimum allowed value of {}".format( - value, valid_range[0]) - assert value <= valid_range[1], \ - "{} is greater than maximum allowed value of {}".format( - value, valid_range[1]) + assert value >= valid_range[0], ( + "{} is less than minimum allowed value of {}" + .format(value, valid_range[0])) + assert value <= valid_range[1], ( + "{} is greater than maximum allowed value of {}" + .format(value, valid_range[1])) class PoolCreationError(Exception): - """ - A custom error to inform the caller that a pool creation failed. Provides an error message + """A custom exception to inform the caller that a pool creation failed. + + Provides an error message """ def __init__(self, message): super(PoolCreationError, self).__init__(message) -class Pool(object): - """ - An object oriented approach to Ceph pool creation. This base class is inherited by ReplicatedPool and ErasurePool. - Do not call create() on this base class as it will not do anything. Instantiate a child class and call create(). - """ +class BasePool(object): + """An object oriented approach to Ceph pool creation. - def __init__(self, service, name): + This base class is inherited by ReplicatedPool and ErasurePool. Do not call + create() on this base class as it will raise an exception. + + Instantiate a child class and call create(). + """ + # Dictionary that maps pool operation properties to Tuples with valid type + # and valid range + op_validation_map = { + 'compression-algorithm': (str, ('lz4', 'snappy', 'zlib', 'zstd')), + 'compression-mode': (str, ('none', 'passive', 'aggressive', 'force')), + 'compression-required-ratio': (float, None), + 'compression-min-blob-size': (int, None), + 'compression-min-blob-size-hdd': (int, None), + 'compression-min-blob-size-ssd': (int, None), + 'compression-max-blob-size': (int, None), + 'compression-max-blob-size-hdd': (int, None), + 'compression-max-blob-size-ssd': (int, None), + } + + def __init__(self, service, name=None, percent_data=None, app_name=None, + op=None): + """Initialize BasePool object. + + Pool information is either initialized from individual keyword + arguments or from a individual CephBrokerRq operation Dict. + + :param service: The Ceph user name to run commands under. + :type service: str + :param name: Name of pool to operate on. + :type name: str + :param percent_data: The expected pool size in relation to all + available resources in the Ceph cluster. Will be + used to set the ``target_size_ratio`` pool + property. (default: 10.0) + :type percent_data: Optional[float] + :param app_name: Ceph application name, usually one of: + ('cephfs', 'rbd', 'rgw') (default: 'unknown') + :type app_name: Optional[str] + :param op: Broker request Op to compile pool data from. + :type op: Optional[Dict[str,any]] + :raises: KeyError + """ + # NOTE: Do not perform initialization steps that require live data from + # a running cluster here. The *Pool classes may be used for validation. self.service = service - self.name = name + self.nautilus_or_later = cmp_pkgrevno('ceph-common', '14.2.0') >= 0 + self.op = op or {} + + if op: + # When initializing from op the `name` attribute is required and we + # will fail with KeyError if it is not provided. + self.name = op['name'] + self.percent_data = op.get('weight') + self.app_name = op.get('app-name') + else: + self.name = name + self.percent_data = percent_data + self.app_name = app_name + + # Set defaults for these if they are not provided + self.percent_data = self.percent_data or 10.0 + self.app_name = self.app_name or 'unknown' + + def validate(self): + """Check that value of supplied operation parameters are valid. + + :raises: ValueError + """ + for op_key, op_value in self.op.items(): + if op_key in self.op_validation_map and op_value is not None: + valid_type, valid_range = self.op_validation_map[op_key] + try: + validator(op_value, valid_type, valid_range) + except (AssertionError, ValueError) as e: + # Normalize on ValueError, also add information about which + # variable we had an issue with. + raise ValueError("'{}': {}".format(op_key, str(e))) + + def _create(self): + """Perform the pool creation, method MUST be overridden by child class. + """ + raise NotImplementedError + + def _post_create(self): + """Perform common post pool creation tasks. + + Note that pool properties subject to change during the lifetime of a + pool / deployment should go into the ``update`` method. + + Do not add calls for a specific pool type here, those should go into + one of the pool specific classes. + """ + if self.nautilus_or_later: + # Ensure we set the expected pool ratio + update_pool( + client=self.service, + pool=self.name, + settings={ + 'target_size_ratio': str( + self.percent_data / 100.0), + }) + try: + set_app_name_for_pool(client=self.service, + pool=self.name, + name=self.app_name) + except CalledProcessError: + log('Could not set app name for pool {}' + .format(self.name), + level=WARNING) + if 'pg_autoscaler' in enabled_manager_modules(): + try: + enable_pg_autoscale(self.service, self.name) + except CalledProcessError as e: + log('Could not configure auto scaling for pool {}: {}' + .format(self.name, e), + level=WARNING) - # Create the pool if it doesn't exist already - # To be implemented by subclasses def create(self): - pass + """Create pool and perform any post pool creation tasks. + + To allow for sharing of common code among pool specific classes the + processing has been broken out into the private methods ``_create`` + and ``_post_create``. + + Do not add any pool type specific handling here, that should go into + one of the pool specific classes. + """ + if not pool_exists(self.service, self.name): + self.validate() + self._create() + self._post_create() + self.update() + + def set_quota(self): + """Set a quota if requested. + + :raises: CalledProcessError + """ + max_bytes = self.op.get('max-bytes') + max_objects = self.op.get('max-objects') + if max_bytes or max_objects: + set_pool_quota(service=self.service, pool_name=self.name, + max_bytes=max_bytes, max_objects=max_objects) + + def set_compression(self): + """Set compression properties if requested. + + :raises: CalledProcessError + """ + compression_properties = { + key.replace('-', '_'): value + for key, value in self.op.items() + if key in ( + 'compression-algorithm', + 'compression-mode', + 'compression-required-ratio', + 'compression-min-blob-size', + 'compression-min-blob-size-hdd', + 'compression-min-blob-size-ssd', + 'compression-max-blob-size', + 'compression-max-blob-size-hdd', + 'compression-max-blob-size-ssd') and value} + if compression_properties: + update_pool(self.service, self.name, compression_properties) + + def update(self): + """Update properties for an already existing pool. + + Do not add calls for a specific pool type here, those should go into + one of the pool specific classes. + """ + self.validate() + self.set_quota() + self.set_compression() def add_cache_tier(self, cache_pool, mode): - """ - Adds a new cache tier to an existing pool. - :param cache_pool: six.string_types. The cache tier pool name to add. - :param mode: six.string_types. The caching mode to use for this pool. valid range = ["readonly", "writeback"] - :return: None + """Adds a new cache tier to an existing pool. + + :param cache_pool: The cache tier pool name to add. + :type cache_pool: str + :param mode: The caching mode to use for this pool. + valid range = ["readonly", "writeback"] + :type mode: str """ # Check the input types and values validator(value=cache_pool, valid_type=six.string_types) - validator(value=mode, valid_type=six.string_types, valid_range=["readonly", "writeback"]) + validator( + value=mode, valid_type=six.string_types, + valid_range=["readonly", "writeback"]) - check_call(['ceph', '--id', self.service, 'osd', 'tier', 'add', self.name, cache_pool]) - check_call(['ceph', '--id', self.service, 'osd', 'tier', 'cache-mode', cache_pool, mode]) - check_call(['ceph', '--id', self.service, 'osd', 'tier', 'set-overlay', self.name, cache_pool]) - check_call(['ceph', '--id', self.service, 'osd', 'pool', 'set', cache_pool, 'hit_set_type', 'bloom']) + check_call([ + 'ceph', '--id', self.service, + 'osd', 'tier', 'add', self.name, cache_pool, + ]) + check_call([ + 'ceph', '--id', self.service, + 'osd', 'tier', 'cache-mode', cache_pool, mode, + ]) + check_call([ + 'ceph', '--id', self.service, + 'osd', 'tier', 'set-overlay', self.name, cache_pool, + ]) + check_call([ + 'ceph', '--id', self.service, + 'osd', 'pool', 'set', cache_pool, 'hit_set_type', 'bloom', + ]) def remove_cache_tier(self, cache_pool): - """ - Removes a cache tier from Ceph. Flushes all dirty objects from writeback pools and waits for that to complete. - :param cache_pool: six.string_types. The cache tier pool name to remove. - :return: None + """Removes a cache tier from Ceph. + + Flushes all dirty objects from writeback pools and waits for that to + complete. + + :param cache_pool: The cache tier pool name to remove. + :type cache_pool: str """ # read-only is easy, writeback is much harder mode = get_cache_mode(self.service, cache_pool) if mode == 'readonly': - check_call(['ceph', '--id', self.service, 'osd', 'tier', 'cache-mode', cache_pool, 'none']) - check_call(['ceph', '--id', self.service, 'osd', 'tier', 'remove', self.name, cache_pool]) + check_call([ + 'ceph', '--id', self.service, + 'osd', 'tier', 'cache-mode', cache_pool, 'none' + ]) + check_call([ + 'ceph', '--id', self.service, + 'osd', 'tier', 'remove', self.name, cache_pool, + ]) elif mode == 'writeback': pool_forward_cmd = ['ceph', '--id', self.service, 'osd', 'tier', @@ -276,9 +476,15 @@ class Pool(object): check_call(pool_forward_cmd) # Flush the cache and wait for it to return - check_call(['rados', '--id', self.service, '-p', cache_pool, 'cache-flush-evict-all']) - check_call(['ceph', '--id', self.service, 'osd', 'tier', 'remove-overlay', self.name]) - check_call(['ceph', '--id', self.service, 'osd', 'tier', 'remove', self.name, cache_pool]) + check_call([ + 'rados', '--id', self.service, + '-p', cache_pool, 'cache-flush-evict-all']) + check_call([ + 'ceph', '--id', self.service, + 'osd', 'tier', 'remove-overlay', self.name]) + check_call([ + 'ceph', '--id', self.service, + 'osd', 'tier', 'remove', self.name, cache_pool]) def get_pgs(self, pool_size, percent_data=DEFAULT_POOL_WEIGHT, device_class=None): @@ -305,19 +511,23 @@ class Pool(object): selected for the specific rule, rather it is left to the user to tune in the form of 'expected-osd-count' config option. - :param pool_size: int. pool_size is either the number of replicas for + :param pool_size: pool_size is either the number of replicas for replicated pools or the K+M sum for erasure coded pools - :param percent_data: float. the percentage of data that is expected to + :type pool_size: int + :param percent_data: the percentage of data that is expected to be contained in the pool for the specific OSD set. Default value is to assume 10% of the data is for this pool, which is a relatively low % of the data but allows for the pg_num to be increased. NOTE: the default is primarily to handle the scenario where related charms requiring pools has not been upgraded to include an update to indicate their relative usage of the pools. - :param device_class: str. class of storage to use for basis of pgs + :type percent_data: float + :param device_class: class of storage to use for basis of pgs calculation; ceph supports nvme, ssd and hdd by default based on presence of devices of each type in the deployment. - :return: int. The number of pgs to use. + :type device_class: str + :returns: The number of pgs to use. + :rtype: int """ # Note: This calculation follows the approach that is provided @@ -357,7 +567,8 @@ class Pool(object): return LEGACY_PG_COUNT percent_data /= 100.0 - target_pgs_per_osd = config('pgs-per-osd') or DEFAULT_PGS_PER_OSD_TARGET + target_pgs_per_osd = config( + 'pgs-per-osd') or DEFAULT_PGS_PER_OSD_TARGET num_pg = (target_pgs_per_osd * osd_count * percent_data) // pool_size # NOTE: ensure a sane minimum number of PGS otherwise we don't get any @@ -380,147 +591,174 @@ class Pool(object): return int(nearest) -class ReplicatedPool(Pool): - def __init__(self, service, name, pg_num=None, replicas=2, - percent_data=10.0, app_name=None): - super(ReplicatedPool, self).__init__(service=service, name=name) - self.replicas = replicas - self.percent_data = percent_data - if pg_num: +class Pool(BasePool): + """Compability shim for any descendents external to this library.""" + + @deprecate( + 'The ``Pool`` baseclass has been replaced by ``BasePool`` class.') + def __init__(self, service, name): + super(Pool, self).__init__(service, name=name) + + def create(self): + pass + + +class ReplicatedPool(BasePool): + def __init__(self, service, name=None, pg_num=None, replicas=None, + percent_data=None, app_name=None, op=None): + """Initialize ReplicatedPool object. + + Pool information is either initialized from individual keyword + arguments or from a individual CephBrokerRq operation Dict. + + Please refer to the docstring of the ``BasePool`` class for + documentation of the common parameters. + + :param pg_num: Express wish for number of Placement Groups (this value + is subject to validation against a running cluster prior + to use to avoid creating a pool with too many PGs) + :type pg_num: int + :param replicas: Number of copies there should be of each object added + to this replicated pool. + :type replicas: int + :raises: KeyError + """ + # NOTE: Do not perform initialization steps that require live data from + # a running cluster here. The *Pool classes may be used for validation. + + # The common parameters are handled in our parents initializer + super(ReplicatedPool, self).__init__( + service=service, name=name, percent_data=percent_data, + app_name=app_name, op=op) + + if op: + # When initializing from op `replicas` is a required attribute, and + # we will fail with KeyError if it is not provided. + self.replicas = op['replicas'] + self.pg_num = op.get('pg_num') + else: + self.replicas = replicas or 2 + self.pg_num = pg_num + + def _create(self): + # Do extra validation on pg_num with data from live cluster + if self.pg_num: # Since the number of placement groups were specified, ensure # that there aren't too many created. max_pgs = self.get_pgs(self.replicas, 100.0) - self.pg_num = min(pg_num, max_pgs) + self.pg_num = min(self.pg_num, max_pgs) else: - self.pg_num = self.get_pgs(self.replicas, percent_data) - if app_name: - self.app_name = app_name + self.pg_num = self.get_pgs(self.replicas, self.percent_data) + + # Create it + if self.nautilus_or_later: + cmd = [ + 'ceph', '--id', self.service, 'osd', 'pool', 'create', + '--pg-num-min={}'.format( + min(AUTOSCALER_DEFAULT_PGS, self.pg_num) + ), + self.name, str(self.pg_num) + ] else: - self.app_name = 'unknown' + cmd = [ + 'ceph', '--id', self.service, 'osd', 'pool', 'create', + self.name, str(self.pg_num) + ] + check_call(cmd) - def create(self): - if not pool_exists(self.service, self.name): - nautilus_or_later = cmp_pkgrevno('ceph-common', '14.2.0') >= 0 - # Create it - if nautilus_or_later: - cmd = [ - 'ceph', '--id', self.service, 'osd', 'pool', 'create', - '--pg-num-min={}'.format( - min(AUTOSCALER_DEFAULT_PGS, self.pg_num) - ), - self.name, str(self.pg_num) - ] - else: - cmd = [ - 'ceph', '--id', self.service, 'osd', 'pool', 'create', - self.name, str(self.pg_num) - ] - - try: - check_call(cmd) - # Set the pool replica size - update_pool(client=self.service, - pool=self.name, - settings={'size': str(self.replicas)}) - if nautilus_or_later: - # Ensure we set the expected pool ratio - update_pool(client=self.service, - pool=self.name, - settings={'target_size_ratio': str(self.percent_data / 100.0)}) - try: - set_app_name_for_pool(client=self.service, - pool=self.name, - name=self.app_name) - except CalledProcessError: - log('Could not set app name for pool {}'.format(self.name), level=WARNING) - if 'pg_autoscaler' in enabled_manager_modules(): - try: - enable_pg_autoscale(self.service, self.name) - except CalledProcessError as e: - log('Could not configure auto scaling for pool {}: {}'.format( - self.name, e), level=WARNING) - except CalledProcessError: - raise + def _post_create(self): + # Set the pool replica size + update_pool(client=self.service, + pool=self.name, + settings={'size': str(self.replicas)}) + # Perform other common post pool creation tasks + super(ReplicatedPool, self)._post_create() -# Default jerasure erasure coded pool -class ErasurePool(Pool): - def __init__(self, service, name, erasure_code_profile="default", - percent_data=10.0, app_name=None): - super(ErasurePool, self).__init__(service=service, name=name) - self.erasure_code_profile = erasure_code_profile - self.percent_data = percent_data - if app_name: - self.app_name = app_name +class ErasurePool(BasePool): + """Default jerasure erasure coded pool.""" + + def __init__(self, service, name=None, erasure_code_profile=None, + percent_data=None, app_name=None, op=None, + allow_ec_overwrites=False): + """Initialize ReplicatedPool object. + + Pool information is either initialized from individual keyword + arguments or from a individual CephBrokerRq operation Dict. + + Please refer to the docstring of the ``BasePool`` class for + documentation of the common parameters. + + :param erasure_code_profile: EC Profile to use (default: 'default') + :type erasure_code_profile: Optional[str] + """ + # NOTE: Do not perform initialization steps that require live data from + # a running cluster here. The *Pool classes may be used for validation. + + # The common parameters are handled in our parents initializer + super(ErasurePool, self).__init__( + service=service, name=name, percent_data=percent_data, + app_name=app_name, op=op) + + if op: + # Note that the different default when initializing from op stems + # from different handling of this in the `charms.ceph` library. + self.erasure_code_profile = op.get('erasure-profile', + 'default-canonical') + self.allow_ec_overwrites = op.get('allow-ec-overwrites') else: - self.app_name = 'unknown' + # We keep the class default when initialized from keyword arguments + # to not break the API for any other consumers. + self.erasure_code_profile = erasure_code_profile or 'default' + self.allow_ec_overwrites = allow_ec_overwrites - def create(self): - if not pool_exists(self.service, self.name): - # Try to find the erasure profile information in order to properly - # size the number of placement groups. The size of an erasure - # coded placement group is calculated as k+m. - erasure_profile = get_erasure_profile(self.service, - self.erasure_code_profile) + def _create(self): + # Try to find the erasure profile information in order to properly + # size the number of placement groups. The size of an erasure + # coded placement group is calculated as k+m. + erasure_profile = get_erasure_profile(self.service, + self.erasure_code_profile) - # Check for errors - if erasure_profile is None: - msg = ("Failed to discover erasure profile named " - "{}".format(self.erasure_code_profile)) - log(msg, level=ERROR) - raise PoolCreationError(msg) - if 'k' not in erasure_profile or 'm' not in erasure_profile: - # Error - msg = ("Unable to find k (data chunks) or m (coding chunks) " - "in erasure profile {}".format(erasure_profile)) - log(msg, level=ERROR) - raise PoolCreationError(msg) + # Check for errors + if erasure_profile is None: + msg = ("Failed to discover erasure profile named " + "{}".format(self.erasure_code_profile)) + log(msg, level=ERROR) + raise PoolCreationError(msg) + if 'k' not in erasure_profile or 'm' not in erasure_profile: + # Error + msg = ("Unable to find k (data chunks) or m (coding chunks) " + "in erasure profile {}".format(erasure_profile)) + log(msg, level=ERROR) + raise PoolCreationError(msg) - k = int(erasure_profile['k']) - m = int(erasure_profile['m']) - pgs = self.get_pgs(k + m, self.percent_data) - nautilus_or_later = cmp_pkgrevno('ceph-common', '14.2.0') >= 0 - # Create it - if nautilus_or_later: - cmd = [ - 'ceph', '--id', self.service, 'osd', 'pool', 'create', - '--pg-num-min={}'.format( - min(AUTOSCALER_DEFAULT_PGS, pgs) - ), - self.name, str(pgs), str(pgs), - 'erasure', self.erasure_code_profile - ] - else: - cmd = [ - 'ceph', '--id', self.service, 'osd', 'pool', 'create', - self.name, str(pgs), str(pgs), - 'erasure', self.erasure_code_profile - ] + k = int(erasure_profile['k']) + m = int(erasure_profile['m']) + pgs = self.get_pgs(k + m, self.percent_data) + self.nautilus_or_later = cmp_pkgrevno('ceph-common', '14.2.0') >= 0 + # Create it + if self.nautilus_or_later: + cmd = [ + 'ceph', '--id', self.service, 'osd', 'pool', 'create', + '--pg-num-min={}'.format( + min(AUTOSCALER_DEFAULT_PGS, pgs) + ), + self.name, str(pgs), str(pgs), + 'erasure', self.erasure_code_profile + ] + else: + cmd = [ + 'ceph', '--id', self.service, 'osd', 'pool', 'create', + self.name, str(pgs), str(pgs), + 'erasure', self.erasure_code_profile + ] + check_call(cmd) - try: - check_call(cmd) - try: - set_app_name_for_pool(client=self.service, - pool=self.name, - name=self.app_name) - except CalledProcessError: - log('Could not set app name for pool {}'.format(self.name), level=WARNING) - if nautilus_or_later: - # Ensure we set the expected pool ratio - update_pool(client=self.service, - pool=self.name, - settings={'target_size_ratio': str(self.percent_data / 100.0)}) - if 'pg_autoscaler' in enabled_manager_modules(): - try: - enable_pg_autoscale(self.service, self.name) - except CalledProcessError as e: - log('Could not configure auto scaling for pool {}: {}'.format( - self.name, e), level=WARNING) - except CalledProcessError: - raise - - """Get an existing erasure code profile if it already exists. - Returns json formatted output""" + def _post_create(self): + super(ErasurePool, self)._post_create() + if self.allow_ec_overwrites: + update_pool(self.service, self.name, + {'allow_ec_overwrites': 'true'}) def enabled_manager_modules(): @@ -541,22 +779,28 @@ def enabled_manager_modules(): def enable_pg_autoscale(service, pool_name): - """ - Enable Ceph's PG autoscaler for the specified pool. + """Enable Ceph's PG autoscaler for the specified pool. - :param service: six.string_types. The Ceph user name to run the command under - :param pool_name: six.string_types. The name of the pool to enable sutoscaling on - :raise: CalledProcessError if the command fails + :param service: The Ceph user name to run the command under + :type service: str + :param pool_name: The name of the pool to enable sutoscaling on + :type pool_name: str + :raises: CalledProcessError if the command fails """ - check_call(['ceph', '--id', service, 'osd', 'pool', 'set', pool_name, 'pg_autoscale_mode', 'on']) + check_call([ + 'ceph', '--id', service, + 'osd', 'pool', 'set', pool_name, 'pg_autoscale_mode', 'on']) def get_mon_map(service): - """ - Returns the current monitor map. - :param service: six.string_types. The Ceph user name to run the command under - :return: json string. :raise: ValueError if the monmap fails to parse. - Also raises CalledProcessError if our ceph command fails + """Return the current monitor map. + + :param service: The Ceph user name to run the command under + :type service: str + :returns: Dictionary with monitor map data + :rtype: Dict[str,any] + :raises: ValueError if the monmap fails to parse, CalledProcessError if our + ceph command fails. """ try: mon_status = check_output(['ceph', '--id', service, @@ -576,17 +820,16 @@ def get_mon_map(service): def hash_monitor_names(service): - """ + """Get a sorted list of monitor hashes in ascending order. + Uses the get_mon_map() function to get information about the monitor - cluster. - Hash the name of each monitor. Return a sorted list of monitor hashes - in an ascending order. - :param service: six.string_types. The Ceph user name to run the command under - :rtype : dict. json dict of monitor name, ip address and rank - example: { - 'name': 'ip-172-31-13-165', - 'rank': 0, - 'addr': '172.31.13.165:6789/0'} + cluster. Hash the name of each monitor. + + :param service: The Ceph user name to run the command under. + :type service: str + :returns: a sorted list of monitor hashes in an ascending order. + :rtype : List[str] + :raises: CalledProcessError, ValueError """ try: hash_list = [] @@ -603,46 +846,56 @@ def hash_monitor_names(service): def monitor_key_delete(service, key): - """ - Delete a key and value pair from the monitor cluster - :param service: six.string_types. The Ceph user name to run the command under + """Delete a key and value pair from the monitor cluster. + Deletes a key value pair on the monitor cluster. - :param key: six.string_types. The key to delete. + + :param service: The Ceph user name to run the command under + :type service: str + :param key: The key to delete. + :type key: str + :raises: CalledProcessError """ try: check_output( ['ceph', '--id', service, 'config-key', 'del', str(key)]) except CalledProcessError as e: - log("Monitor config-key put failed with message: {}".format( - e.output)) + log("Monitor config-key put failed with message: {}" + .format(e.output)) raise def monitor_key_set(service, key, value): - """ - Sets a key value pair on the monitor cluster. - :param service: six.string_types. The Ceph user name to run the command under - :param key: six.string_types. The key to set. - :param value: The value to set. This will be converted to a string - before setting + """Set a key value pair on the monitor cluster. + + :param service: The Ceph user name to run the command under. + :type service str + :param key: The key to set. + :type key: str + :param value: The value to set. This will be coerced into a string. + :type value: str + :raises: CalledProcessError """ try: check_output( ['ceph', '--id', service, 'config-key', 'put', str(key), str(value)]) except CalledProcessError as e: - log("Monitor config-key put failed with message: {}".format( - e.output)) + log("Monitor config-key put failed with message: {}" + .format(e.output)) raise def monitor_key_get(service, key): - """ - Gets the value of an existing key in the monitor cluster. - :param service: six.string_types. The Ceph user name to run the command under - :param key: six.string_types. The key to search for. + """Get the value of an existing key in the monitor cluster. + + :param service: The Ceph user name to run the command under + :type service: str + :param key: The key to search for. + :type key: str :return: Returns the value of that key or None if not found. + :rtype: Optional[str] """ try: output = check_output( @@ -650,19 +903,21 @@ def monitor_key_get(service, key): 'config-key', 'get', str(key)]).decode('UTF-8') return output except CalledProcessError as e: - log("Monitor config-key get failed with message: {}".format( - e.output)) + log("Monitor config-key get failed with message: {}" + .format(e.output)) return None def monitor_key_exists(service, key): - """ - Searches for the existence of a key in the monitor cluster. - :param service: six.string_types. The Ceph user name to run the command under - :param key: six.string_types. The key to search for - :return: Returns True if the key exists, False if not and raises an - exception if an unknown error occurs. :raise: CalledProcessError if - an unknown error occurs + """Search for existence of key in the monitor cluster. + + :param service: The Ceph user name to run the command under. + :type service: str + :param key: The key to search for. + :type key: str + :return: Returns True if the key exists, False if not. + :rtype: bool + :raises: CalledProcessError if an unknown error occurs. """ try: check_call( @@ -675,16 +930,20 @@ def monitor_key_exists(service, key): if e.returncode == errno.ENOENT: return False else: - log("Unknown error from ceph config-get exists: {} {}".format( - e.returncode, e.output)) + log("Unknown error from ceph config-get exists: {} {}" + .format(e.returncode, e.output)) raise def get_erasure_profile(service, name): - """ - :param service: six.string_types. The Ceph user name to run the command under - :param name: - :return: + """Get an existing erasure code profile if it exists. + + :param service: The Ceph user name to run the command under. + :type service: str + :param name: Name of profile. + :type name: str + :returns: Dictionary with profile data. + :rtype: Optional[Dict[str]] """ try: out = check_output(['ceph', '--id', service, @@ -698,54 +957,61 @@ def get_erasure_profile(service, name): def pool_set(service, pool_name, key, value): + """Sets a value for a RADOS pool in ceph. + + :param service: The Ceph user name to run the command under. + :type service: str + :param pool_name: Name of pool to set property on. + :type pool_name: str + :param key: Property key. + :type key: str + :param value: Value, will be coerced into str and shifted to lowercase. + :type value: str + :raises: CalledProcessError """ - Sets a value for a RADOS pool in ceph. - :param service: six.string_types. The Ceph user name to run the command under - :param pool_name: six.string_types - :param key: six.string_types - :param value: - :return: None. Can raise CalledProcessError - """ - cmd = ['ceph', '--id', service, 'osd', 'pool', 'set', pool_name, key, - str(value).lower()] - try: - check_call(cmd) - except CalledProcessError: - raise + cmd = [ + 'ceph', '--id', service, + 'osd', 'pool', 'set', pool_name, key, str(value).lower()] + check_call(cmd) def snapshot_pool(service, pool_name, snapshot_name): + """Snapshots a RADOS pool in Ceph. + + :param service: The Ceph user name to run the command under. + :type service: str + :param pool_name: Name of pool to snapshot. + :type pool_name: str + :param snapshot_name: Name of snapshot to create. + :type snapshot_name: str + :raises: CalledProcessError """ - Snapshots a RADOS pool in ceph. - :param service: six.string_types. The Ceph user name to run the command under - :param pool_name: six.string_types - :param snapshot_name: six.string_types - :return: None. Can raise CalledProcessError - """ - cmd = ['ceph', '--id', service, 'osd', 'pool', 'mksnap', pool_name, snapshot_name] - try: - check_call(cmd) - except CalledProcessError: - raise + cmd = [ + 'ceph', '--id', service, + 'osd', 'pool', 'mksnap', pool_name, snapshot_name] + check_call(cmd) def remove_pool_snapshot(service, pool_name, snapshot_name): + """Remove a snapshot from a RADOS pool in Ceph. + + :param service: The Ceph user name to run the command under. + :type service: str + :param pool_name: Name of pool to remove snapshot from. + :type pool_name: str + :param snapshot_name: Name of snapshot to remove. + :type snapshot_name: str + :raises: CalledProcessError """ - Remove a snapshot from a RADOS pool in ceph. - :param service: six.string_types. The Ceph user name to run the command under - :param pool_name: six.string_types - :param snapshot_name: six.string_types - :return: None. Can raise CalledProcessError - """ - cmd = ['ceph', '--id', service, 'osd', 'pool', 'rmsnap', pool_name, snapshot_name] - try: - check_call(cmd) - except CalledProcessError: - raise + cmd = [ + 'ceph', '--id', service, + 'osd', 'pool', 'rmsnap', pool_name, snapshot_name] + check_call(cmd) def set_pool_quota(service, pool_name, max_bytes=None, max_objects=None): - """ + """Set byte quota on a RADOS pool in Ceph. + :param service: The Ceph user name to run the command under :type service: str :param pool_name: Name of pool @@ -756,7 +1022,9 @@ def set_pool_quota(service, pool_name, max_bytes=None, max_objects=None): :type max_objects: int :raises: subprocess.CalledProcessError """ - cmd = ['ceph', '--id', service, 'osd', 'pool', 'set-quota', pool_name] + cmd = [ + 'ceph', '--id', service, + 'osd', 'pool', 'set-quota', pool_name] if max_bytes: cmd = cmd + ['max_bytes', str(max_bytes)] if max_objects: @@ -765,119 +1033,216 @@ def set_pool_quota(service, pool_name, max_bytes=None, max_objects=None): def remove_pool_quota(service, pool_name): + """Remove byte quota on a RADOS pool in Ceph. + + :param service: The Ceph user name to run the command under. + :type service: str + :param pool_name: Name of pool to remove quota from. + :type pool_name: str + :raises: CalledProcessError """ - Set a byte quota on a RADOS pool in ceph. - :param service: six.string_types. The Ceph user name to run the command under - :param pool_name: six.string_types - :return: None. Can raise CalledProcessError - """ - cmd = ['ceph', '--id', service, 'osd', 'pool', 'set-quota', pool_name, 'max_bytes', '0'] - try: - check_call(cmd) - except CalledProcessError: - raise + cmd = [ + 'ceph', '--id', service, + 'osd', 'pool', 'set-quota', pool_name, 'max_bytes', '0'] + check_call(cmd) def remove_erasure_profile(service, profile_name): + """Remove erasure code profile. + + :param service: The Ceph user name to run the command under + :type service: str + :param profile_name: Name of profile to remove. + :type profile_name: str + :raises: CalledProcessError """ - Create a new erasure code profile if one does not already exist for it. Updates - the profile if it exists. Please see http://docs.ceph.com/docs/master/rados/operations/erasure-code-profile/ - for more details - :param service: six.string_types. The Ceph user name to run the command under - :param profile_name: six.string_types - :return: None. Can raise CalledProcessError - """ - cmd = ['ceph', '--id', service, 'osd', 'erasure-code-profile', 'rm', - profile_name] - try: - check_call(cmd) - except CalledProcessError: - raise + cmd = [ + 'ceph', '--id', service, + 'osd', 'erasure-code-profile', 'rm', profile_name] + check_call(cmd) -def create_erasure_profile(service, profile_name, erasure_plugin_name='jerasure', - failure_domain='host', +def create_erasure_profile(service, profile_name, + erasure_plugin_name='jerasure', + failure_domain=None, data_chunks=2, coding_chunks=1, locality=None, durability_estimator=None, - device_class=None): - """ - Create a new erasure code profile if one does not already exist for it. Updates - the profile if it exists. Please see http://docs.ceph.com/docs/master/rados/operations/erasure-code-profile/ - for more details - :param service: six.string_types. The Ceph user name to run the command under - :param profile_name: six.string_types - :param erasure_plugin_name: six.string_types - :param failure_domain: six.string_types. One of ['chassis', 'datacenter', 'host', 'osd', 'pdu', 'pod', 'rack', 'region', - 'room', 'root', 'row']) - :param data_chunks: int - :param coding_chunks: int - :param locality: int - :param durability_estimator: int - :param device_class: six.string_types - :return: None. Can raise CalledProcessError - """ - # Ensure this failure_domain is allowed by Ceph - validator(failure_domain, six.string_types, - ['chassis', 'datacenter', 'host', 'osd', 'pdu', 'pod', 'rack', 'region', 'room', 'root', 'row']) + helper_chunks=None, + scalar_mds=None, + crush_locality=None, + device_class=None, + erasure_plugin_technique=None): + """Create a new erasure code profile if one does not already exist for it. - cmd = ['ceph', '--id', service, 'osd', 'erasure-code-profile', 'set', profile_name, - 'plugin=' + erasure_plugin_name, 'k=' + str(data_chunks), 'm=' + str(coding_chunks) - ] - if locality is not None and durability_estimator is not None: - raise ValueError("create_erasure_profile should be called with k, m and one of l or c but not both.") + Updates the profile if it exists. Please refer to [0] for more details. + + 0: http://docs.ceph.com/docs/master/rados/operations/erasure-code-profile/ + + :param service: The Ceph user name to run the command under. + :type service: str + :param profile_name: Name of profile. + :type profile_name: str + :param erasure_plugin_name: Erasure code plugin. + :type erasure_plugin_name: str + :param failure_domain: Failure domain, one of: + ('chassis', 'datacenter', 'host', 'osd', 'pdu', + 'pod', 'rack', 'region', 'room', 'root', 'row'). + :type failure_domain: str + :param data_chunks: Number of data chunks. + :type data_chunks: int + :param coding_chunks: Number of coding chunks. + :type coding_chunks: int + :param locality: Locality. + :type locality: int + :param durability_estimator: Durability estimator. + :type durability_estimator: int + :param helper_chunks: int + :type helper_chunks: int + :param device_class: Restrict placement to devices of specific class. + :type device_class: str + :param scalar_mds: one of ['isa', 'jerasure', 'shec'] + :type scalar_mds: str + :param crush_locality: LRC locality faulure domain, one of: + ('chassis', 'datacenter', 'host', 'osd', 'pdu', 'pod', + 'rack', 'region', 'room', 'root', 'row') or unset. + :type crush_locaity: str + :param erasure_plugin_technique: Coding technique for EC plugin + :type erasure_plugin_technique: str + :return: None. Can raise CalledProcessError, ValueError or AssertionError + """ + plugin_techniques = { + 'jerasure': [ + 'reed_sol_van', + 'reed_sol_r6_op', + 'cauchy_orig', + 'cauchy_good', + 'liberation', + 'blaum_roth', + 'liber8tion' + ], + 'lrc': [], + 'isa': [ + 'reed_sol_van', + 'cauchy', + ], + 'shec': [ + 'single', + 'multiple' + ], + 'clay': [], + } + failure_domains = [ + 'chassis', 'datacenter', + 'host', 'osd', + 'pdu', 'pod', + 'rack', 'region', + 'room', 'root', + 'row', + ] + device_classes = [ + 'ssd', + 'hdd', + 'nvme' + ] + + validator(erasure_plugin_name, six.string_types, + list(plugin_techniques.keys())) + + cmd = [ + 'ceph', '--id', service, + 'osd', 'erasure-code-profile', 'set', profile_name, + 'plugin={}'.format(erasure_plugin_name), + 'k={}'.format(str(data_chunks)), + 'm={}'.format(str(coding_chunks)), + ] + + if erasure_plugin_technique: + validator(erasure_plugin_technique, six.string_types, + plugin_techniques[erasure_plugin_name]) + cmd.append('technique={}'.format(erasure_plugin_technique)) luminous_or_later = cmp_pkgrevno('ceph-common', '12.0.0') >= 0 - # failure_domain changed in luminous - if luminous_or_later: - cmd.append('crush-failure-domain=' + failure_domain) - else: - cmd.append('ruleset-failure-domain=' + failure_domain) + + # Set failure domain from options if not provided in args + if not failure_domain and config('customize-failure-domain'): + # Defaults to 'host' so just need to deal with + # setting 'rack' if feature is enabled + failure_domain = 'rack' + + if failure_domain: + validator(failure_domain, six.string_types, failure_domains) + # failure_domain changed in luminous + if luminous_or_later: + cmd.append('crush-failure-domain={}'.format(failure_domain)) + else: + cmd.append('ruleset-failure-domain={}'.format(failure_domain)) # device class new in luminous if luminous_or_later and device_class: + validator(device_class, six.string_types, device_classes) cmd.append('crush-device-class={}'.format(device_class)) else: log('Skipping device class configuration (ceph < 12.0.0)', level=DEBUG) # Add plugin specific information - if locality is not None: - # For local erasure codes - cmd.append('l=' + str(locality)) - if durability_estimator is not None: - # For Shec erasure codes - cmd.append('c=' + str(durability_estimator)) + if erasure_plugin_name == 'lrc': + # LRC mandatory configuration + if locality: + cmd.append('l={}'.format(str(locality))) + else: + raise ValueError("locality must be provided for lrc plugin") + # LRC optional configuration + if crush_locality: + validator(crush_locality, six.string_types, failure_domains) + cmd.append('crush-locality={}'.format(crush_locality)) + + if erasure_plugin_name == 'shec': + # SHEC optional configuration + if durability_estimator: + cmd.append('c={}'.format((durability_estimator))) + + if erasure_plugin_name == 'clay': + # CLAY optional configuration + if helper_chunks: + cmd.append('d={}'.format(str(helper_chunks))) + if scalar_mds: + cmd.append('scalar-mds={}'.format(scalar_mds)) if erasure_profile_exists(service, profile_name): cmd.append('--force') - try: - check_call(cmd) - except CalledProcessError: - raise + check_call(cmd) def rename_pool(service, old_name, new_name): - """ - Rename a Ceph pool from old_name to new_name - :param service: six.string_types. The Ceph user name to run the command under - :param old_name: six.string_types - :param new_name: six.string_types - :return: None + """Rename a Ceph pool from old_name to new_name. + + :param service: The Ceph user name to run the command under. + :type service: str + :param old_name: Name of pool subject to rename. + :type old_name: str + :param new_name: Name to rename pool to. + :type new_name: str """ validator(value=old_name, valid_type=six.string_types) validator(value=new_name, valid_type=six.string_types) - cmd = ['ceph', '--id', service, 'osd', 'pool', 'rename', old_name, new_name] + cmd = [ + 'ceph', '--id', service, + 'osd', 'pool', 'rename', old_name, new_name] check_call(cmd) def erasure_profile_exists(service, name): - """ - Check to see if an Erasure code profile already exists. - :param service: six.string_types. The Ceph user name to run the command under - :param name: six.string_types - :return: int or None + """Check to see if an Erasure code profile already exists. + + :param service: The Ceph user name to run the command under + :type service: str + :param name: Name of profile to look for. + :type name: str + :returns: True if it exists, False otherwise. + :rtype: bool """ validator(value=name, valid_type=six.string_types) try: @@ -890,11 +1255,14 @@ def erasure_profile_exists(service, name): def get_cache_mode(service, pool_name): - """ - Find the current caching mode of the pool_name given. - :param service: six.string_types. The Ceph user name to run the command under - :param pool_name: six.string_types - :return: int or None + """Find the current caching mode of the pool_name given. + + :param service: The Ceph user name to run the command under + :type service: str + :param pool_name: Name of pool. + :type pool_name: str + :returns: Current cache mode. + :rtype: Optional[int] """ validator(value=service, valid_type=six.string_types) validator(value=pool_name, valid_type=six.string_types) @@ -976,17 +1344,23 @@ def create_rbd_image(service, pool, image, sizemb): def update_pool(client, pool, settings): + """Update pool properties. + + :param client: Client/User-name to authenticate with. + :type client: str + :param pool: Name of pool to operate on + :type pool: str + :param settings: Dictionary with key/value pairs to set. + :type settings: Dict[str, str] + :raises: CalledProcessError + """ cmd = ['ceph', '--id', client, 'osd', 'pool', 'set', pool] for k, v in six.iteritems(settings): - cmd.append(k) - cmd.append(v) - - check_call(cmd) + check_call(cmd + [k, v]) def set_app_name_for_pool(client, pool, name): - """ - Calls `osd pool application enable` for the specified pool name + """Calls `osd pool application enable` for the specified pool name :param client: Name of the ceph client to use :type client: str @@ -1043,8 +1417,7 @@ def _keyring_path(service): def add_key(service, key): - """ - Add a key to a keyring. + """Add a key to a keyring. Creates the keyring if it doesn't already exist. @@ -1288,13 +1661,33 @@ class CephBrokerRq(object): The API is versioned and defaults to version 1. """ - def __init__(self, api_version=1, request_id=None): - self.api_version = api_version - if request_id: - self.request_id = request_id + def __init__(self, api_version=1, request_id=None, raw_request_data=None): + """Initialize CephBrokerRq object. + + Builds a new empty request or rebuilds a request from on-wire JSON + data. + + :param api_version: API version for request (default: 1). + :type api_version: Optional[int] + :param request_id: Unique identifier for request. + (default: string representation of generated UUID) + :type request_id: Optional[str] + :param raw_request_data: JSON-encoded string to build request from. + :type raw_request_data: Optional[str] + :raises: KeyError + """ + if raw_request_data: + request_data = json.loads(raw_request_data) + self.api_version = request_data['api-version'] + self.request_id = request_data['request-id'] + self.set_ops(request_data['ops']) else: - self.request_id = str(uuid.uuid1()) - self.ops = [] + self.api_version = api_version + if request_id: + self.request_id = request_id + else: + self.request_id = str(uuid.uuid1()) + self.ops = [] def add_op(self, op): """Add an op if it is not already in the list. @@ -1336,12 +1729,119 @@ class CephBrokerRq(object): group=group, namespace=namespace, app_name=app_name, max_bytes=max_bytes, max_objects=max_objects) + # Use function parameters and docstring to define types in a compatible + # manner. + # + # NOTE: Our caller should always use a kwarg Dict when calling us so + # no need to maintain fixed order/position for parameters. Please keep them + # sorted by name when adding new ones. + def _partial_build_common_op_create(self, + app_name=None, + compression_algorithm=None, + compression_mode=None, + compression_required_ratio=None, + compression_min_blob_size=None, + compression_min_blob_size_hdd=None, + compression_min_blob_size_ssd=None, + compression_max_blob_size=None, + compression_max_blob_size_hdd=None, + compression_max_blob_size_ssd=None, + group=None, + max_bytes=None, + max_objects=None, + namespace=None, + weight=None): + """Build common part of a create pool operation. + + :param app_name: Tag pool with application name. Note that there is + certain protocols emerging upstream with regard to + meaningful application names to use. + Examples are 'rbd' and 'rgw'. + :type app_name: Optional[str] + :param compression_algorithm: Compressor to use, one of: + ('lz4', 'snappy', 'zlib', 'zstd') + :type compression_algorithm: Optional[str] + :param compression_mode: When to compress data, one of: + ('none', 'passive', 'aggressive', 'force') + :type compression_mode: Optional[str] + :param compression_required_ratio: Minimum compression ratio for data + chunk, if the requested ratio is not + achieved the compressed version will + be thrown away and the original + stored. + :type compression_required_ratio: Optional[float] + :param compression_min_blob_size: Chunks smaller than this are never + compressed (unit: bytes). + :type compression_min_blob_size: Optional[int] + :param compression_min_blob_size_hdd: Chunks smaller than this are not + compressed when destined to + rotational media (unit: bytes). + :type compression_min_blob_size_hdd: Optional[int] + :param compression_min_blob_size_ssd: Chunks smaller than this are not + compressed when destined to flash + media (unit: bytes). + :type compression_min_blob_size_ssd: Optional[int] + :param compression_max_blob_size: Chunks larger than this are broken + into N * compression_max_blob_size + chunks before being compressed + (unit: bytes). + :type compression_max_blob_size: Optional[int] + :param compression_max_blob_size_hdd: Chunks larger than this are + broken into + N * compression_max_blob_size_hdd + chunks before being compressed + when destined for rotational + media (unit: bytes) + :type compression_max_blob_size_hdd: Optional[int] + :param compression_max_blob_size_ssd: Chunks larger than this are + broken into + N * compression_max_blob_size_ssd + chunks before being compressed + when destined for flash media + (unit: bytes). + :type compression_max_blob_size_ssd: Optional[int] + :param group: Group to add pool to + :type group: Optional[str] + :param max_bytes: Maximum bytes quota to apply + :type max_bytes: Optional[int] + :param max_objects: Maximum objects quota to apply + :type max_objects: Optional[int] + :param namespace: Group namespace + :type namespace: Optional[str] + :param weight: The percentage of data that is expected to be contained + in the pool from the total available space on the OSDs. + Used to calculate number of Placement Groups to create + for pool. + :type weight: Optional[float] + :returns: Dictionary with kwarg name as key. + :rtype: Dict[str,any] + :raises: AssertionError + """ + return { + 'app-name': app_name, + 'compression-algorithm': compression_algorithm, + 'compression-mode': compression_mode, + 'compression-required-ratio': compression_required_ratio, + 'compression-min-blob-size': compression_min_blob_size, + 'compression-min-blob-size-hdd': compression_min_blob_size_hdd, + 'compression-min-blob-size-ssd': compression_min_blob_size_ssd, + 'compression-max-blob-size': compression_max_blob_size, + 'compression-max-blob-size-hdd': compression_max_blob_size_hdd, + 'compression-max-blob-size-ssd': compression_max_blob_size_ssd, + 'group': group, + 'max-bytes': max_bytes, + 'max-objects': max_objects, + 'group-namespace': namespace, + 'weight': weight, + } + def add_op_create_replicated_pool(self, name, replica_count=3, pg_num=None, - weight=None, group=None, namespace=None, - app_name=None, max_bytes=None, - max_objects=None): + **kwargs): """Adds an operation to create a replicated pool. + Refer to docstring for ``_partial_build_common_op_create`` for + documentation of keyword arguments. + :param name: Name of pool to create :type name: str :param replica_count: Number of copies Ceph should keep of your data. @@ -1349,66 +1849,114 @@ class CephBrokerRq(object): :param pg_num: Request specific number of Placement Groups to create for pool. :type pg_num: int - :param weight: The percentage of data that is expected to be contained - in the pool from the total available space on the OSDs. - Used to calculate number of Placement Groups to create - for pool. - :type weight: float - :param group: Group to add pool to - :type group: str - :param namespace: Group namespace - :type namespace: str - :param app_name: (Optional) Tag pool with application name. Note that - there is certain protocols emerging upstream with - regard to meaningful application names to use. - Examples are ``rbd`` and ``rgw``. - :type app_name: str - :param max_bytes: Maximum bytes quota to apply - :type max_bytes: int - :param max_objects: Maximum objects quota to apply - :type max_objects: int + :raises: AssertionError if provided data is of invalid type/range """ - if pg_num and weight: + if pg_num and kwargs.get('weight'): raise ValueError('pg_num and weight are mutually exclusive') - self.add_op({'op': 'create-pool', 'name': name, - 'replicas': replica_count, 'pg_num': pg_num, - 'weight': weight, 'group': group, - 'group-namespace': namespace, 'app-name': app_name, - 'max-bytes': max_bytes, 'max-objects': max_objects}) + op = { + 'op': 'create-pool', + 'name': name, + 'replicas': replica_count, + 'pg_num': pg_num, + } + op.update(self._partial_build_common_op_create(**kwargs)) + + # Initialize Pool-object to validate type and range of ops. + pool = ReplicatedPool('dummy-service', op=op) + pool.validate() + + self.add_op(op) def add_op_create_erasure_pool(self, name, erasure_profile=None, - weight=None, group=None, app_name=None, - max_bytes=None, max_objects=None): + allow_ec_overwrites=False, **kwargs): """Adds an operation to create a erasure coded pool. + Refer to docstring for ``_partial_build_common_op_create`` for + documentation of keyword arguments. + :param name: Name of pool to create :type name: str :param erasure_profile: Name of erasure code profile to use. If not set the ceph-mon unit handling the broker request will set its default value. :type erasure_profile: str - :param weight: The percentage of data that is expected to be contained - in the pool from the total available space on the OSDs. - :type weight: float - :param group: Group to add pool to - :type group: str - :param app_name: (Optional) Tag pool with application name. Note that - there is certain protocols emerging upstream with - regard to meaningful application names to use. - Examples are ``rbd`` and ``rgw``. - :type app_name: str - :param max_bytes: Maximum bytes quota to apply - :type max_bytes: int - :param max_objects: Maximum objects quota to apply - :type max_objects: int + :param allow_ec_overwrites: allow EC pools to be overriden + :type allow_ec_overwrites: bool + :raises: AssertionError if provided data is of invalid type/range """ - self.add_op({'op': 'create-pool', 'name': name, - 'pool-type': 'erasure', - 'erasure-profile': erasure_profile, - 'weight': weight, - 'group': group, 'app-name': app_name, - 'max-bytes': max_bytes, 'max-objects': max_objects}) + op = { + 'op': 'create-pool', + 'name': name, + 'pool-type': 'erasure', + 'erasure-profile': erasure_profile, + 'allow-ec-overwrites': allow_ec_overwrites, + } + op.update(self._partial_build_common_op_create(**kwargs)) + + # Initialize Pool-object to validate type and range of ops. + pool = ErasurePool('dummy-service', op) + pool.validate() + + self.add_op(op) + + def add_op_create_erasure_profile(self, name, + erasure_type='jerasure', + erasure_technique=None, + k=None, m=None, + failure_domain=None, + lrc_locality=None, + shec_durability_estimator=None, + clay_helper_chunks=None, + device_class=None, + clay_scalar_mds=None, + lrc_crush_locality=None): + """Adds an operation to create a erasure coding profile. + + :param name: Name of profile to create + :type name: str + :param erasure_type: Which of the erasure coding plugins should be used + :type erasure_type: string + :param erasure_technique: EC plugin technique to use + :type erasure_technique: string + :param k: Number of data chunks + :type k: int + :param m: Number of coding chunks + :type m: int + :param lrc_locality: Group the coding and data chunks into sets of size locality + (lrc plugin) + :type lrc_locality: int + :param durability_estimator: The number of parity chuncks each of which includes + a data chunk in its calculation range (shec plugin) + :type durability_estimator: int + :param helper_chunks: The number of helper chunks to use for recovery operations + (clay plugin) + :type: helper_chunks: int + :param failure_domain: Type of failure domain from Ceph bucket types + to be used + :type failure_domain: string + :param device_class: Device class to use for profile (ssd, hdd) + :type device_class: string + :param clay_scalar_mds: Plugin to use for CLAY layered construction + (jerasure|isa|shec) + :type clay_scaler_mds: string + :param lrc_crush_locality: Type of crush bucket in which set of chunks + defined by lrc_locality will be stored. + :type lrc_crush_locality: string + """ + self.add_op({'op': 'create-erasure-profile', + 'name': name, + 'k': k, + 'm': m, + 'l': lrc_locality, + 'c': shec_durability_estimator, + 'd': clay_helper_chunks, + 'erasure-type': erasure_type, + 'erasure-technique': erasure_technique, + 'failure-domain': failure_domain, + 'device-class': device_class, + 'scalar-mds': clay_scalar_mds, + 'crush-locality': lrc_crush_locality}) def set_ops(self, ops): """Set request ops to provided value. @@ -1424,12 +1972,14 @@ class CephBrokerRq(object): 'request-id': self.request_id}) def _ops_equal(self, other): + keys_to_compare = [ + 'replicas', 'name', 'op', 'pg_num', 'group-permission', + 'object-prefix-permissions', + ] + keys_to_compare += list(self._partial_build_common_op_create().keys()) if len(self.ops) == len(other.ops): for req_no in range(0, len(self.ops)): - for key in [ - 'replicas', 'name', 'op', 'pg_num', 'weight', - 'group', 'group-namespace', 'group-permission', - 'object-prefix-permissions']: + for key in keys_to_compare: if self.ops[req_no].get(key) != other.ops[req_no].get(key): return False else: @@ -1522,18 +2072,15 @@ class CephBrokerRsp(object): def get_previous_request(rid): """Return the last ceph broker request sent on a given relation - @param rid: Relation id to query for request + :param rid: Relation id to query for request + :type rid: str + :returns: CephBrokerRq object or None if relation data not found. + :rtype: Optional[CephBrokerRq] """ - request = None broker_req = relation_get(attribute='broker_req', rid=rid, unit=local_unit()) if broker_req: - request_data = json.loads(broker_req) - request = CephBrokerRq(api_version=request_data['api-version'], - request_id=request_data['request-id']) - request.set_ops(request_data['ops']) - - return request + return CephBrokerRq(raw_request_data=broker_req) def get_request_states(request, relation='ceph'): diff --git a/hooks/ceph_hooks.py b/hooks/ceph_hooks.py index 456c10b..0c16d39 100755 --- a/hooks/ceph_hooks.py +++ b/hooks/ceph_hooks.py @@ -16,6 +16,7 @@ import sys _path = os.path.dirname(os.path.realpath(__file__)) _root = os.path.abspath(os.path.join(_path, '..')) +_lib = os.path.abspath(os.path.join(_path, '../lib')) def _add_path(path): @@ -24,6 +25,7 @@ def _add_path(path): _add_path(_root) +_add_path(_lib) import ceph from charmhelpers.core.hookenv import ( @@ -63,7 +65,7 @@ from charmhelpers.contrib.openstack.utils import ( from charmhelpers.core.templating import render -from ceph_broker import ( +from charms_ceph.broker import ( process_requests ) diff --git a/hooks/install b/hooks/install index eb05824..869ee20 100755 --- a/hooks/install +++ b/hooks/install @@ -2,7 +2,7 @@ # Wrapper to deal with newer Ubuntu versions that don't have py2 installed # by default. -declare -a DEPS=('apt' 'netaddr' 'netifaces' 'pip' 'yaml' 'dnspython') +declare -a DEPS=('apt' 'netaddr' 'netifaces' 'pip' 'yaml') check_and_install() { pkg="${1}-${2}" @@ -17,4 +17,5 @@ for dep in ${DEPS[@]}; do check_and_install ${PYTHON} ${dep} done +./hooks/install_deps exec ./hooks/install.real diff --git a/hooks/install_deps b/hooks/install_deps new file mode 100755 index 0000000..c480f29 --- /dev/null +++ b/hooks/install_deps @@ -0,0 +1,18 @@ +#!/bin/bash -e +# Wrapper to ensure that python dependencies are installed before we get into +# the python part of the hook execution + +declare -a DEPS=('dnspython' 'pyudev') + +check_and_install() { + pkg="${1}-${2}" + if ! dpkg -s ${pkg} 2>&1 > /dev/null; then + apt-get -y install ${pkg} + fi +} + +PYTHON="python3" + +for dep in ${DEPS[@]}; do + check_and_install ${PYTHON} ${dep} +done diff --git a/hooks/upgrade-charm b/hooks/upgrade-charm new file mode 100755 index 0000000..c32fb38 --- /dev/null +++ b/hooks/upgrade-charm @@ -0,0 +1,6 @@ +#!/bin/bash -e +# Wrapper to ensure that old python bytecode isn't hanging around +# after we upgrade the charm with newer libraries +rm -rf **/*.pyc + +./hooks/install_deps diff --git a/lib/.keep b/lib/.keep deleted file mode 100644 index f49b91a..0000000 --- a/lib/.keep +++ /dev/null @@ -1,3 +0,0 @@ - This file was created by release-tools to ensure that this empty - directory is preserved in vcs re: lint check definitions in global - tox.ini files. This file can be removed if/when this dir is actually in use. diff --git a/lib/charms_ceph/__init__.py b/lib/charms_ceph/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/hooks/ceph_broker.py b/lib/charms_ceph/broker.py similarity index 53% rename from hooks/ceph_broker.py rename to lib/charms_ceph/broker.py index ec55a67..8f040a5 100644 --- a/hooks/ceph_broker.py +++ b/lib/charms_ceph/broker.py @@ -1,12 +1,29 @@ -#!/usr/bin/python +# Copyright 2016 Canonical Ltd # -# Copyright 2015 Canonical Ltd. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import collections import json -import six -from subprocess import check_call, CalledProcessError +import os +from subprocess import check_call, check_output, CalledProcessError +from tempfile import NamedTemporaryFile + +from charms_ceph.utils import ( + get_cephfs, + get_osd_weight +) +from charms_ceph.crush_utils import Crushmap from charmhelpers.core.hookenv import ( log, @@ -25,18 +42,17 @@ from charmhelpers.contrib.storage.linux.ceph import ( pool_set, remove_pool_snapshot, rename_pool, - set_pool_quota, snapshot_pool, validator, ErasurePool, - Pool, + BasePool, ReplicatedPool, ) - # This comes from http://docs.ceph.com/docs/master/rados/operations/pools/ # This should do a decent job of preventing people from passing in bad values. # It will give a useful error message + POOL_KEYS = { # "Ceph Key Name": [Python type, [Valid Range]] "size": [int], @@ -51,8 +67,8 @@ POOL_KEYS = { "write_fadvise_dontneed": [bool], "noscrub": [bool], "nodeep-scrub": [bool], - "hit_set_type": [six.string_types, ["bloom", "explicit_hash", - "explicit_object"]], + "hit_set_type": [str, ["bloom", "explicit_hash", + "explicit_object"]], "hit_set_count": [int, [1, 1]], "hit_set_period": [int], "hit_set_fpp": [float, [0.0, 1.0]], @@ -64,6 +80,11 @@ POOL_KEYS = { "cache_min_flush_age": [int], "cache_min_evict_age": [int], "fast_read": [bool], + "allow_ec_overwrites": [bool], + "compression_mode": [str, ["none", "passive", "aggressive", "force"]], + "compression_algorithm": [str, ["lz4", "snappy", "zlib", "zstd"]], + "compression_required_ratio": [float, [0.0, 1.0]], + "crush_rule": [str], } CEPH_BUCKET_TYPES = [ @@ -96,6 +117,9 @@ def process_requests(reqs): This is a versioned api. API version must be supplied by the client making the request. + + :param reqs: dict of request parameters. + :returns: dict. exit-code and reason if not 0 """ request_id = reqs.get('request-id') try: @@ -115,7 +139,7 @@ def process_requests(reqs): log(msg, level=ERROR) return {'exit-code': 1, 'stderr': msg} - msg = ("Missing or invalid api version (%s)" % version) + msg = ("Missing or invalid api version ({})".format(version)) resp = {'exit-code': 1, 'stderr': msg} if request_id: resp['request-id'] = request_id @@ -124,200 +148,53 @@ def process_requests(reqs): def handle_create_erasure_profile(request, service): - # "local" | "shec" or it defaults to "jerasure" + """Create an erasure profile. + + :param request: dict of request operations and params + :param service: The ceph client to run the command under. + :returns: dict. exit-code and reason if not 0 + """ + # "isa" | "lrc" | "shec" | "clay" or it defaults to "jerasure" erasure_type = request.get('erasure-type') - # "host" | "rack" or it defaults to "host" # Any valid Ceph bucket + # dependent on erasure coding type + erasure_technique = request.get('erasure-technique') + # "host" | "rack" | ... failure_domain = request.get('failure-domain') name = request.get('name') - k = request.get('k') - m = request.get('m') - l = request.get('l') + # Binary Distribution Matrix (BDM) parameters + bdm_k = request.get('k') + bdm_m = request.get('m') + # LRC parameters + bdm_l = request.get('l') + crush_locality = request.get('crush-locality') + # SHEC parameters + bdm_c = request.get('c') + # CLAY parameters + bdm_d = request.get('d') + scalar_mds = request.get('scalar-mds') + # Device Class + device_class = request.get('device-class') - if failure_domain not in CEPH_BUCKET_TYPES: + if failure_domain and failure_domain not in CEPH_BUCKET_TYPES: msg = "failure-domain must be one of {}".format(CEPH_BUCKET_TYPES) log(msg, level=ERROR) return {'exit-code': 1, 'stderr': msg} - create_erasure_profile(service=service, erasure_plugin_name=erasure_type, - profile_name=name, failure_domain=failure_domain, - data_chunks=k, coding_chunks=m, locality=l) + create_erasure_profile(service=service, + erasure_plugin_name=erasure_type, + profile_name=name, + failure_domain=failure_domain, + data_chunks=bdm_k, + coding_chunks=bdm_m, + locality=bdm_l, + durability_estimator=bdm_d, + helper_chunks=bdm_c, + scalar_mds=scalar_mds, + crush_locality=crush_locality, + device_class=device_class, + erasure_plugin_technique=erasure_technique) - -def handle_erasure_pool(request, service): - """Create a new erasure coded pool. - - :param request: dict of request operations and params. - :param service: The ceph client to run the command under. - :returns: dict. exit-code and reason if not 0. - """ - pool_name = request.get('name') - erasure_profile = request.get('erasure-profile') - max_bytes = request.get('max-bytes') - max_objects = request.get('max-objects') - weight = request.get('weight') - group_name = request.get('group') - - if erasure_profile is None: - erasure_profile = "default-canonical" - - app_name = request.get('app-name') - - # Check for missing params - if pool_name is None: - msg = "Missing parameter. name is required for the pool" - log(msg, level=ERROR) - return {'exit-code': 1, 'stderr': msg} - - if group_name: - group_namespace = request.get('group-namespace') - # Add the pool to the group named "group_name" - add_pool_to_group(pool=pool_name, - group=group_name, - namespace=group_namespace) - - # TODO: Default to 3/2 erasure coding. I believe this requires min 5 osds - if not erasure_profile_exists(service=service, name=erasure_profile): - # TODO: Fail and tell them to create the profile or default - msg = ("erasure-profile {} does not exist. Please create it with: " - "create-erasure-profile".format(erasure_profile)) - log(msg, level=ERROR) - return {'exit-code': 1, 'stderr': msg} - - pool = ErasurePool(service=service, name=pool_name, - erasure_code_profile=erasure_profile, - percent_data=weight, app_name=app_name) - # Ok make the erasure pool - if not pool_exists(service=service, name=pool_name): - log("Creating pool '{}' (erasure_profile={})" - .format(pool.name, erasure_profile), level=INFO) - pool.create() - - # Set a quota if requested - if max_bytes or max_objects: - set_pool_quota(service=service, pool_name=pool_name, - max_bytes=max_bytes, max_objects=max_objects) - - -def handle_replicated_pool(request, service): - """Create a new replicated pool. - - :param request: dict of request operations and params. - :param service: The ceph client to run the command under. - :returns: dict. exit-code and reason if not 0. - """ - pool_name = request.get('name') - replicas = request.get('replicas') - max_bytes = request.get('max-bytes') - max_objects = request.get('max-objects') - weight = request.get('weight') - group_name = request.get('group') - - # Optional params - pg_num = request.get('pg_num') - if pg_num: - # Cap pg_num to max allowed just in case. - osds = get_osds(service) - if osds: - pg_num = min(pg_num, (len(osds) * 100 // replicas)) - - app_name = request.get('app-name') - # Check for missing params - if pool_name is None or replicas is None: - msg = "Missing parameter. name and replicas are required" - log(msg, level=ERROR) - return {'exit-code': 1, 'stderr': msg} - - if group_name: - group_namespace = request.get('group-namespace') - # Add the pool to the group named "group_name" - add_pool_to_group(pool=pool_name, - group=group_name, - namespace=group_namespace) - - kwargs = {} - if pg_num: - kwargs['pg_num'] = pg_num - if weight: - kwargs['percent_data'] = weight - if replicas: - kwargs['replicas'] = replicas - if app_name: - kwargs['app_name'] = app_name - - pool = ReplicatedPool(service=service, - name=pool_name, **kwargs) - if not pool_exists(service=service, name=pool_name): - log("Creating pool '{}' (replicas={})".format(pool.name, replicas), - level=INFO) - pool.create() - else: - log("Pool '{}' already exists - skipping create".format(pool.name), - level=DEBUG) - - # Set a quota if requested - if max_bytes or max_objects: - set_pool_quota(service=service, pool_name=pool_name, - max_bytes=max_bytes, max_objects=max_objects) - - -def handle_create_cache_tier(request, service): - # mode = "writeback" | "readonly" - storage_pool = request.get('cold-pool') - cache_pool = request.get('hot-pool') - cache_mode = request.get('mode') - - if cache_mode is None: - cache_mode = "writeback" - - # cache and storage pool must exist first - if not pool_exists(service=service, name=storage_pool) or not pool_exists( - service=service, name=cache_pool): - msg = ("cold-pool: {} and hot-pool: {} must exist. Please create " - "them first".format(storage_pool, cache_pool)) - log(msg, level=ERROR) - return {'exit-code': 1, 'stderr': msg} - - p = Pool(service=service, name=storage_pool) - p.add_cache_tier(cache_pool=cache_pool, mode=cache_mode) - - -def handle_remove_cache_tier(request, service): - storage_pool = request.get('cold-pool') - cache_pool = request.get('hot-pool') - # cache and storage pool must exist first - if not pool_exists(service=service, name=storage_pool) or not pool_exists( - service=service, name=cache_pool): - msg = ("cold-pool: {} or hot-pool: {} doesn't exist. Not " - "deleting cache tier".format(storage_pool, cache_pool)) - log(msg, level=ERROR) - return {'exit-code': 1, 'stderr': msg} - - pool = Pool(name=storage_pool, service=service) - pool.remove_cache_tier(cache_pool=cache_pool) - - -def handle_set_pool_value(request, service): - # Set arbitrary pool values - params = {'pool': request.get('name'), - 'key': request.get('key'), - 'value': request.get('value')} - if params['key'] not in POOL_KEYS: - msg = "Invalid key '%s'" % params['key'] - log(msg, level=ERROR) - return {'exit-code': 1, 'stderr': msg} - - # Get the validation method - validator_params = POOL_KEYS[params['key']] - if len(validator_params) is 1: - # Validate that what the user passed is actually legal per Ceph's rules - validator(params['value'], validator_params[0]) - else: - # Validate that what the user passed is actually legal per Ceph's rules - validator(params['value'], validator_params[0], validator_params[1]) - - # Set the value - pool_set(service=service, pool_name=params['pool'], key=params['key'], - value=params['value']) + return {'exit-code': 0} def handle_add_permissions_to_key(request, service): @@ -358,6 +235,30 @@ def handle_add_permissions_to_key(request, service): return resp +def handle_set_key_permissions(request, service): + """Ensure the key has the requested permissions.""" + permissions = request.get('permissions') + client = request.get('client') + call = ['ceph', '--id', service, 'auth', 'caps', + 'client.{}'.format(client)] + permissions + try: + check_call(call) + except CalledProcessError as e: + log("Error updating key capabilities: {}".format(e), level=ERROR) + + +def update_service_permissions(service, service_obj=None, namespace=None): + """Update the key permissions for the named client in Ceph""" + if not service_obj: + service_obj = get_service_groups(service=service, namespace=namespace) + permissions = pool_permission_list_for_service(service_obj) + call = ['ceph', 'auth', 'caps', 'client.{}'.format(service)] + permissions + try: + check_call(call) + except CalledProcessError as e: + log("Error updating key capabilities: {}".format(e)) + + def add_pool_to_group(pool, group, namespace=None): """Add a named pool to a named group""" group_name = group @@ -394,60 +295,6 @@ def pool_permission_list_for_service(service): 'osd', ', '.join(permissions)] -def update_service_permissions(service, service_obj=None, namespace=None): - """Update the key permissions for the named client in Ceph""" - if not service_obj: - service_obj = get_service_groups(service=service, namespace=namespace) - permissions = pool_permission_list_for_service(service_obj) - call = ['ceph', 'auth', 'caps', 'client.{}'.format(service)] + permissions - try: - check_call(call) - except CalledProcessError as e: - log("Error updating key capabilities: {}".format(e)) - - -def save_service(service_name, service): - """Persist a service in the monitor cluster""" - service['groups'] = {} - return monitor_key_set(service='admin', - key="cephx.services.{}".format(service_name), - value=json.dumps(service, sort_keys=True)) - - -def save_group(group, group_name): - """Persist a group in the monitor cluster""" - group_key = get_group_key(group_name=group_name) - return monitor_key_set(service='admin', - key=group_key, - value=json.dumps(group, sort_keys=True)) - - -def get_group(group_name): - """A group is a structure to hold data about a named group, structured as: - { - pools: ['glance'], - services: ['nova'] - } - """ - group_key = get_group_key(group_name=group_name) - group_json = monitor_key_get(service='admin', key=group_key) - try: - group = json.loads(group_json) - except (TypeError, ValueError): - group = None - if not group: - group = { - 'pools': [], - 'services': [] - } - return group - - -def get_group_key(group_name): - """Build group key""" - return 'cephx.groups.{}'.format(group_name) - - def get_service_groups(service, namespace=None): """Services are objects stored with some metadata, they look like (for a service named "nova"): @@ -506,6 +353,478 @@ def _build_service_groups(service, namespace=None): return all_groups +def get_group(group_name): + """A group is a structure to hold data about a named group, structured as: + { + pools: ['glance'], + services: ['nova'] + } + """ + group_key = get_group_key(group_name=group_name) + group_json = monitor_key_get(service='admin', key=group_key) + try: + group = json.loads(group_json) + except (TypeError, ValueError): + group = None + if not group: + group = { + 'pools': [], + 'services': [] + } + return group + + +def save_service(service_name, service): + """Persist a service in the monitor cluster""" + service['groups'] = {} + return monitor_key_set(service='admin', + key="cephx.services.{}".format(service_name), + value=json.dumps(service, sort_keys=True)) + + +def save_group(group, group_name): + """Persist a group in the monitor cluster""" + group_key = get_group_key(group_name=group_name) + return monitor_key_set(service='admin', + key=group_key, + value=json.dumps(group, sort_keys=True)) + + +def get_group_key(group_name): + """Build group key""" + return 'cephx.groups.{}'.format(group_name) + + +def handle_erasure_pool(request, service): + """Create a new erasure coded pool. + + :param request: dict of request operations and params. + :param service: The ceph client to run the command under. + :returns: dict. exit-code and reason if not 0. + """ + pool_name = request.get('name') + erasure_profile = request.get('erasure-profile') + group_name = request.get('group') + + if erasure_profile is None: + erasure_profile = "default-canonical" + + if group_name: + group_namespace = request.get('group-namespace') + # Add the pool to the group named "group_name" + add_pool_to_group(pool=pool_name, + group=group_name, + namespace=group_namespace) + + # TODO: Default to 3/2 erasure coding. I believe this requires min 5 osds + if not erasure_profile_exists(service=service, name=erasure_profile): + # TODO: Fail and tell them to create the profile or default + msg = ("erasure-profile {} does not exist. Please create it with: " + "create-erasure-profile".format(erasure_profile)) + log(msg, level=ERROR) + return {'exit-code': 1, 'stderr': msg} + + try: + pool = ErasurePool(service=service, + op=request) + except KeyError: + msg = "Missing parameter." + log(msg, level=ERROR) + return {'exit-code': 1, 'stderr': msg} + + # Ok make the erasure pool + if not pool_exists(service=service, name=pool_name): + log("Creating pool '{}' (erasure_profile={})" + .format(pool.name, erasure_profile), level=INFO) + pool.create() + + # Set/update properties that are allowed to change after pool creation. + pool.update() + + +def handle_replicated_pool(request, service): + """Create a new replicated pool. + + :param request: dict of request operations and params. + :param service: The ceph client to run the command under. + :returns: dict. exit-code and reason if not 0. + """ + pool_name = request.get('name') + group_name = request.get('group') + + # Optional params + # NOTE: Check this against the handling in the Pool classes, reconcile and + # remove. + pg_num = request.get('pg_num') + replicas = request.get('replicas') + if pg_num: + # Cap pg_num to max allowed just in case. + osds = get_osds(service) + if osds: + pg_num = min(pg_num, (len(osds) * 100 // replicas)) + request.update({'pg_num': pg_num}) + + if group_name: + group_namespace = request.get('group-namespace') + # Add the pool to the group named "group_name" + add_pool_to_group(pool=pool_name, + group=group_name, + namespace=group_namespace) + + try: + pool = ReplicatedPool(service=service, + op=request) + except KeyError: + msg = "Missing parameter." + log(msg, level=ERROR) + return {'exit-code': 1, 'stderr': msg} + + if not pool_exists(service=service, name=pool_name): + log("Creating pool '{}' (replicas={})".format(pool.name, replicas), + level=INFO) + pool.create() + else: + log("Pool '{}' already exists - skipping create".format(pool.name), + level=DEBUG) + + # Set/update properties that are allowed to change after pool creation. + pool.update() + + +def handle_create_cache_tier(request, service): + """Create a cache tier on a cold pool. Modes supported are + "writeback" and "readonly". + + :param request: dict of request operations and params + :param service: The ceph client to run the command under. + :returns: dict. exit-code and reason if not 0 + """ + # mode = "writeback" | "readonly" + storage_pool = request.get('cold-pool') + cache_pool = request.get('hot-pool') + cache_mode = request.get('mode') + + if cache_mode is None: + cache_mode = "writeback" + + # cache and storage pool must exist first + if not pool_exists(service=service, name=storage_pool) or not pool_exists( + service=service, name=cache_pool): + msg = ("cold-pool: {} and hot-pool: {} must exist. Please create " + "them first".format(storage_pool, cache_pool)) + log(msg, level=ERROR) + return {'exit-code': 1, 'stderr': msg} + + p = BasePool(service=service, name=storage_pool) + p.add_cache_tier(cache_pool=cache_pool, mode=cache_mode) + + +def handle_remove_cache_tier(request, service): + """Remove a cache tier from the cold pool. + + :param request: dict of request operations and params + :param service: The ceph client to run the command under. + :returns: dict. exit-code and reason if not 0 + """ + storage_pool = request.get('cold-pool') + cache_pool = request.get('hot-pool') + # cache and storage pool must exist first + if not pool_exists(service=service, name=storage_pool) or not pool_exists( + service=service, name=cache_pool): + msg = ("cold-pool: {} or hot-pool: {} doesn't exist. Not " + "deleting cache tier".format(storage_pool, cache_pool)) + log(msg, level=ERROR) + return {'exit-code': 1, 'stderr': msg} + + pool = BasePool(name=storage_pool, service=service) + pool.remove_cache_tier(cache_pool=cache_pool) + + +def handle_set_pool_value(request, service, coerce=False): + """Sets an arbitrary pool value. + + :param request: dict of request operations and params + :param service: The ceph client to run the command under. + :param coerce: Try to parse/coerce the value into the correct type. + Used by the action code that only gets Str from Juju + :returns: dict. exit-code and reason if not 0 + """ + # Set arbitrary pool values + params = {'pool': request.get('name'), + 'key': request.get('key'), + 'value': request.get('value')} + if params['key'] not in POOL_KEYS: + msg = "Invalid key '{}'".format(params['key']) + log(msg, level=ERROR) + return {'exit-code': 1, 'stderr': msg} + + # Get the validation method + validator_params = POOL_KEYS[params['key']] + # BUG: #1838650 - the function needs to try to coerce the value param to + # the type required for the validator to pass. Note, if this blows, then + # the param isn't parsable to the correct type. + if coerce: + try: + params['value'] = validator_params[0](params['value']) + except ValueError: + raise RuntimeError("Value {} isn't of type {}" + .format(params['value'], validator_params[0])) + # end of BUG: #1838650 + if len(validator_params) == 1: + # Validate that what the user passed is actually legal per Ceph's rules + validator(params['value'], validator_params[0]) + else: + # Validate that what the user passed is actually legal per Ceph's rules + validator(params['value'], validator_params[0], validator_params[1]) + + # Set the value + pool_set(service=service, pool_name=params['pool'], key=params['key'], + value=params['value']) + + +def handle_rgw_regionmap_update(request, service): + """Change the radosgw region map. + + :param request: dict of request operations and params + :param service: The ceph client to run the command under. + :returns: dict. exit-code and reason if not 0 + """ + name = request.get('client-name') + if not name: + msg = "Missing rgw-region or client-name params" + log(msg, level=ERROR) + return {'exit-code': 1, 'stderr': msg} + try: + check_output(['radosgw-admin', + '--id', service, + 'regionmap', 'update', '--name', name]) + except CalledProcessError as err: + log(err.output, level=ERROR) + return {'exit-code': 1, 'stderr': err.output} + + +def handle_rgw_regionmap_default(request, service): + """Create a radosgw region map. + + :param request: dict of request operations and params + :param service: The ceph client to run the command under. + :returns: dict. exit-code and reason if not 0 + """ + region = request.get('rgw-region') + name = request.get('client-name') + if not region or not name: + msg = "Missing rgw-region or client-name params" + log(msg, level=ERROR) + return {'exit-code': 1, 'stderr': msg} + try: + check_output( + [ + 'radosgw-admin', + '--id', service, + 'regionmap', + 'default', + '--rgw-region', region, + '--name', name]) + except CalledProcessError as err: + log(err.output, level=ERROR) + return {'exit-code': 1, 'stderr': err.output} + + +def handle_rgw_zone_set(request, service): + """Create a radosgw zone. + + :param request: dict of request operations and params + :param service: The ceph client to run the command under. + :returns: dict. exit-code and reason if not 0 + """ + json_file = request.get('zone-json') + name = request.get('client-name') + region_name = request.get('region-name') + zone_name = request.get('zone-name') + if not json_file or not name or not region_name or not zone_name: + msg = "Missing json-file or client-name params" + log(msg, level=ERROR) + return {'exit-code': 1, 'stderr': msg} + infile = NamedTemporaryFile(delete=False) + with open(infile.name, 'w') as infile_handle: + infile_handle.write(json_file) + try: + check_output( + [ + 'radosgw-admin', + '--id', service, + 'zone', + 'set', + '--rgw-zone', zone_name, + '--infile', infile.name, + '--name', name, + ] + ) + except CalledProcessError as err: + log(err.output, level=ERROR) + return {'exit-code': 1, 'stderr': err.output} + os.unlink(infile.name) + + +def handle_put_osd_in_bucket(request, service): + """Move an osd into a specified crush bucket. + + :param request: dict of request operations and params + :param service: The ceph client to run the command under. + :returns: dict. exit-code and reason if not 0 + """ + osd_id = request.get('osd') + target_bucket = request.get('bucket') + if not osd_id or not target_bucket: + msg = "Missing OSD ID or Bucket" + log(msg, level=ERROR) + return {'exit-code': 1, 'stderr': msg} + crushmap = Crushmap() + try: + crushmap.ensure_bucket_is_present(target_bucket) + check_output( + [ + 'ceph', + '--id', service, + 'osd', + 'crush', + 'set', + str(osd_id), + str(get_osd_weight(osd_id)), + "root={}".format(target_bucket) + ] + ) + + except Exception as exc: + msg = "Failed to move OSD " \ + "{} into Bucket {} :: {}".format(osd_id, target_bucket, exc) + log(msg, level=ERROR) + return {'exit-code': 1, 'stderr': msg} + + +def handle_rgw_create_user(request, service): + """Create a new rados gateway user. + + :param request: dict of request operations and params + :param service: The ceph client to run the command under. + :returns: dict. exit-code and reason if not 0 + """ + user_id = request.get('rgw-uid') + display_name = request.get('display-name') + name = request.get('client-name') + if not name or not display_name or not user_id: + msg = "Missing client-name, display-name or rgw-uid" + log(msg, level=ERROR) + return {'exit-code': 1, 'stderr': msg} + try: + create_output = check_output( + [ + 'radosgw-admin', + '--id', service, + 'user', + 'create', + '--uid', user_id, + '--display-name', display_name, + '--name', name, + '--system' + ] + ) + try: + user_json = json.loads(str(create_output.decode('UTF-8'))) + return {'exit-code': 0, 'user': user_json} + except ValueError as err: + log(err, level=ERROR) + return {'exit-code': 1, 'stderr': err} + + except CalledProcessError as err: + log(err.output, level=ERROR) + return {'exit-code': 1, 'stderr': err.output} + + +def handle_create_cephfs(request, service): + """Create a new cephfs. + + :param request: The broker request + :param service: The ceph client to run the command under. + :returns: dict. exit-code and reason if not 0 + """ + cephfs_name = request.get('mds_name') + data_pool = request.get('data_pool') + metadata_pool = request.get('metadata_pool') + # Check if the user params were provided + if not cephfs_name or not data_pool or not metadata_pool: + msg = "Missing mds_name, data_pool or metadata_pool params" + log(msg, level=ERROR) + return {'exit-code': 1, 'stderr': msg} + + # Sanity check that the required pools exist + if not pool_exists(service=service, name=data_pool): + msg = "CephFS data pool does not exist. Cannot create CephFS" + log(msg, level=ERROR) + return {'exit-code': 1, 'stderr': msg} + if not pool_exists(service=service, name=metadata_pool): + msg = "CephFS metadata pool does not exist. Cannot create CephFS" + log(msg, level=ERROR) + return {'exit-code': 1, 'stderr': msg} + + if get_cephfs(service=service): + # CephFS new has already been called + log("CephFS already created") + return + + # Finally create CephFS + try: + check_output(["ceph", + '--id', service, + "fs", "new", cephfs_name, + metadata_pool, + data_pool]) + except CalledProcessError as err: + if err.returncode == 22: + log("CephFS already created") + return + else: + log(err.output, level=ERROR) + return {'exit-code': 1, 'stderr': err.output} + + +def handle_rgw_region_set(request, service): + # radosgw-admin region set --infile us.json --name client.radosgw.us-east-1 + """Set the rados gateway region. + + :param request: dict. The broker request. + :param service: The ceph client to run the command under. + :returns: dict. exit-code and reason if not 0 + """ + json_file = request.get('region-json') + name = request.get('client-name') + region_name = request.get('region-name') + zone_name = request.get('zone-name') + if not json_file or not name or not region_name or not zone_name: + msg = "Missing json-file or client-name params" + log(msg, level=ERROR) + return {'exit-code': 1, 'stderr': msg} + infile = NamedTemporaryFile(delete=False) + with open(infile.name, 'w') as infile_handle: + infile_handle.write(json_file) + try: + check_output( + [ + 'radosgw-admin', + '--id', service, + 'region', + 'set', + '--rgw-zone', zone_name, + '--infile', infile.name, + '--name', name, + ] + ) + except CalledProcessError as err: + log(err.output, level=ERROR) + return {'exit-code': 1, 'stderr': err.output} + os.unlink(infile.name) + + def process_requests_v1(reqs): """Process v1 requests. @@ -516,10 +835,10 @@ def process_requests_v1(reqs): operation failed along with an explanation). """ ret = None - log("Processing %s ceph broker requests" % (len(reqs)), level=INFO) + log("Processing {} ceph broker requests".format(len(reqs)), level=INFO) for req in reqs: op = req.get('op') - log("Processing op='%s'" % op, level=DEBUG) + log("Processing op='{}'".format(op), level=DEBUG) # Use admin client since we do not have other client key locations # setup to use them for these operations. svc = 'admin' @@ -531,7 +850,8 @@ def process_requests_v1(reqs): ret = handle_erasure_pool(request=req, service=svc) else: ret = handle_replicated_pool(request=req, service=svc) - + elif op == "create-cephfs": + ret = handle_create_cephfs(request=req, service=svc) elif op == "create-cache-tier": ret = handle_create_cache_tier(request=req, service=svc) elif op == "remove-cache-tier": @@ -558,10 +878,24 @@ def process_requests_v1(reqs): snapshot_name=snapshot_name) elif op == "set-pool-value": ret = handle_set_pool_value(request=req, service=svc) + elif op == "rgw-region-set": + ret = handle_rgw_region_set(request=req, service=svc) + elif op == "rgw-zone-set": + ret = handle_rgw_zone_set(request=req, service=svc) + elif op == "rgw-regionmap-update": + ret = handle_rgw_regionmap_update(request=req, service=svc) + elif op == "rgw-regionmap-default": + ret = handle_rgw_regionmap_default(request=req, service=svc) + elif op == "rgw-create-user": + ret = handle_rgw_create_user(request=req, service=svc) + elif op == "move-osd-to-bucket": + ret = handle_put_osd_in_bucket(request=req, service=svc) elif op == "add-permissions-to-key": ret = handle_add_permissions_to_key(request=req, service=svc) + elif op == 'set-key-permissions': + ret = handle_set_key_permissions(request=req, service=svc) else: - msg = "Unknown operation '%s'" % op + msg = "Unknown operation '{}'".format(op) log(msg, level=ERROR) return {'exit-code': 1, 'stderr': msg} diff --git a/lib/charms_ceph/crush_utils.py b/lib/charms_ceph/crush_utils.py new file mode 100644 index 0000000..8fe09fa --- /dev/null +++ b/lib/charms_ceph/crush_utils.py @@ -0,0 +1,154 @@ +# Copyright 2014 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re + +from subprocess import check_output, CalledProcessError + +from charmhelpers.core.hookenv import ( + log, + ERROR, +) + +CRUSH_BUCKET = """root {name} {{ + id {id} # do not change unnecessarily + # weight 0.000 + alg straw2 + hash 0 # rjenkins1 +}} + +rule {name} {{ + ruleset 0 + type replicated + min_size 1 + max_size 10 + step take {name} + step chooseleaf firstn 0 type host + step emit +}}""" + +# This regular expression looks for a string like: +# root NAME { +# id NUMBER +# so that we can extract NAME and ID from the crushmap +CRUSHMAP_BUCKETS_RE = re.compile(r"root\s+(.+)\s+\{\s*id\s+(-?\d+)") + +# This regular expression looks for ID strings in the crushmap like: +# id NUMBER +# so that we can extract the IDs from a crushmap +CRUSHMAP_ID_RE = re.compile(r"id\s+(-?\d+)") + + +class Crushmap(object): + """An object oriented approach to Ceph crushmap management.""" + + def __init__(self): + self._crushmap = self.load_crushmap() + roots = re.findall(CRUSHMAP_BUCKETS_RE, self._crushmap) + buckets = [] + ids = list(map( + lambda x: int(x), + re.findall(CRUSHMAP_ID_RE, self._crushmap))) + ids = sorted(ids) + if roots != []: + for root in roots: + buckets.append(CRUSHBucket(root[0], root[1], True)) + + self._buckets = buckets + if ids != []: + self._ids = ids + else: + self._ids = [0] + + def load_crushmap(self): + try: + crush = str(check_output(['ceph', 'osd', 'getcrushmap']) + .decode('UTF-8')) + return str(check_output(['crushtool', '-d', '-'], + stdin=crush.stdout) + .decode('UTF-8')) + except CalledProcessError as e: + log("Error occured while loading and decompiling CRUSH map:" + "{}".format(e), ERROR) + raise "Failed to read CRUSH map" + + def ensure_bucket_is_present(self, bucket_name): + if bucket_name not in [bucket.name for bucket in self.buckets()]: + self.add_bucket(bucket_name) + self.save() + + def buckets(self): + """Return a list of buckets that are in the Crushmap.""" + return self._buckets + + def add_bucket(self, bucket_name): + """Add a named bucket to Ceph""" + new_id = min(self._ids) - 1 + self._ids.append(new_id) + self._buckets.append(CRUSHBucket(bucket_name, new_id)) + + def save(self): + """Persist Crushmap to Ceph""" + try: + crushmap = self.build_crushmap() + compiled = str(check_output(['crushtool', '-c', '/dev/stdin', '-o', + '/dev/stdout'], stdin=crushmap) + .decode('UTF-8')) + ceph_output = str(check_output(['ceph', 'osd', 'setcrushmap', '-i', + '/dev/stdin'], stdin=compiled) + .decode('UTF-8')) + return ceph_output + except CalledProcessError as e: + log("save error: {}".format(e)) + raise "Failed to save CRUSH map." + + def build_crushmap(self): + """Modifies the current CRUSH map to include the new buckets""" + tmp_crushmap = self._crushmap + for bucket in self._buckets: + if not bucket.default: + tmp_crushmap = "{}\n\n{}".format( + tmp_crushmap, + Crushmap.bucket_string(bucket.name, bucket.id)) + + return tmp_crushmap + + @staticmethod + def bucket_string(name, id): + return CRUSH_BUCKET.format(name=name, id=id) + + +class CRUSHBucket(object): + """CRUSH bucket description object.""" + + def __init__(self, name, id, default=False): + self.name = name + self.id = int(id) + self.default = default + + def __repr__(self): + return "Bucket {{Name: {name}, ID: {id}}}".format( + name=self.name, id=self.id) + + def __eq__(self, other): + """Override the default Equals behavior""" + if isinstance(other, self.__class__): + return self.__dict__ == other.__dict__ + return NotImplemented + + def __ne__(self, other): + """Define a non-equality test""" + if isinstance(other, self.__class__): + return not self.__eq__(other) + return NotImplemented diff --git a/lib/charms_ceph/utils.py b/lib/charms_ceph/utils.py new file mode 100644 index 0000000..72e6b92 --- /dev/null +++ b/lib/charms_ceph/utils.py @@ -0,0 +1,3349 @@ +# Copyright 2017 Canonical Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import collections +import glob +import json +import os +import pyudev +import random +import re +import socket +import subprocess +import sys +import time +import uuid + +from contextlib import contextmanager +from datetime import datetime + +from charmhelpers.core import hookenv +from charmhelpers.core import templating +from charmhelpers.core.host import ( + chownr, + cmp_pkgrevno, + lsb_release, + mkdir, + owner, + service_restart, + service_start, + service_stop, + CompareHostReleases, + write_file, +) +from charmhelpers.core.hookenv import ( + cached, + config, + log, + status_set, + DEBUG, + ERROR, + WARNING, + storage_get, + storage_list, +) +from charmhelpers.fetch import ( + apt_cache, + add_source, apt_install, apt_update +) +from charmhelpers.contrib.storage.linux.ceph import ( + get_mon_map, + monitor_key_set, + monitor_key_exists, + monitor_key_get, +) +from charmhelpers.contrib.storage.linux.utils import ( + is_block_device, + is_device_mounted, +) +from charmhelpers.contrib.openstack.utils import ( + get_os_codename_install_source, +) +from charmhelpers.contrib.storage.linux import lvm +from charmhelpers.core.unitdata import kv + +CEPH_BASE_DIR = os.path.join(os.sep, 'var', 'lib', 'ceph') +OSD_BASE_DIR = os.path.join(CEPH_BASE_DIR, 'osd') +HDPARM_FILE = os.path.join(os.sep, 'etc', 'hdparm.conf') + +LEADER = 'leader' +PEON = 'peon' +QUORUM = [LEADER, PEON] + +PACKAGES = ['ceph', 'gdisk', + 'radosgw', 'xfsprogs', + 'lvm2', 'parted', 'smartmontools'] + +CEPH_KEY_MANAGER = 'ceph' +VAULT_KEY_MANAGER = 'vault' +KEY_MANAGERS = [ + CEPH_KEY_MANAGER, + VAULT_KEY_MANAGER, +] + +LinkSpeed = { + "BASE_10": 10, + "BASE_100": 100, + "BASE_1000": 1000, + "GBASE_10": 10000, + "GBASE_40": 40000, + "GBASE_100": 100000, + "UNKNOWN": None +} + +# Mapping of adapter speed to sysctl settings +NETWORK_ADAPTER_SYSCTLS = { + # 10Gb + LinkSpeed["GBASE_10"]: { + 'net.core.rmem_default': 524287, + 'net.core.wmem_default': 524287, + 'net.core.rmem_max': 524287, + 'net.core.wmem_max': 524287, + 'net.core.optmem_max': 524287, + 'net.core.netdev_max_backlog': 300000, + 'net.ipv4.tcp_rmem': '10000000 10000000 10000000', + 'net.ipv4.tcp_wmem': '10000000 10000000 10000000', + 'net.ipv4.tcp_mem': '10000000 10000000 10000000' + }, + # Mellanox 10/40Gb + LinkSpeed["GBASE_40"]: { + 'net.ipv4.tcp_timestamps': 0, + 'net.ipv4.tcp_sack': 1, + 'net.core.netdev_max_backlog': 250000, + 'net.core.rmem_max': 4194304, + 'net.core.wmem_max': 4194304, + 'net.core.rmem_default': 4194304, + 'net.core.wmem_default': 4194304, + 'net.core.optmem_max': 4194304, + 'net.ipv4.tcp_rmem': '4096 87380 4194304', + 'net.ipv4.tcp_wmem': '4096 65536 4194304', + 'net.ipv4.tcp_low_latency': 1, + 'net.ipv4.tcp_adv_win_scale': 1 + } +} + + +class Partition(object): + def __init__(self, name, number, size, start, end, sectors, uuid): + """A block device partition. + + :param name: Name of block device + :param number: Partition number + :param size: Capacity of the device + :param start: Starting block + :param end: Ending block + :param sectors: Number of blocks + :param uuid: UUID of the partition + """ + self.name = name, + self.number = number + self.size = size + self.start = start + self.end = end + self.sectors = sectors + self.uuid = uuid + + def __str__(self): + return "number: {} start: {} end: {} sectors: {} size: {} " \ + "name: {} uuid: {}".format(self.number, self.start, + self.end, + self.sectors, self.size, + self.name, self.uuid) + + def __eq__(self, other): + if isinstance(other, self.__class__): + return self.__dict__ == other.__dict__ + return False + + def __ne__(self, other): + return not self.__eq__(other) + + +def unmounted_disks(): + """List of unmounted block devices on the current host.""" + disks = [] + context = pyudev.Context() + for device in context.list_devices(DEVTYPE='disk'): + if device['SUBSYSTEM'] == 'block': + if device.device_node is None: + continue + + matched = False + for block_type in [u'dm-', u'loop', u'ram', u'nbd']: + if block_type in device.device_node: + matched = True + if matched: + continue + + disks.append(device.device_node) + log("Found disks: {}".format(disks)) + return [disk for disk in disks if not is_device_mounted(disk)] + + +def save_sysctls(sysctl_dict, save_location): + """Persist the sysctls to the hard drive. + + :param sysctl_dict: dict + :param save_location: path to save the settings to + :raises: IOError if anything goes wrong with writing. + """ + try: + # Persist the settings for reboots + with open(save_location, "w") as fd: + for key, value in sysctl_dict.items(): + fd.write("{}={}\n".format(key, value)) + + except IOError as e: + log("Unable to persist sysctl settings to {}. Error {}".format( + save_location, e), level=ERROR) + raise + + +def tune_nic(network_interface): + """This will set optimal sysctls for the particular network adapter. + + :param network_interface: string The network adapter name. + """ + speed = get_link_speed(network_interface) + if speed in NETWORK_ADAPTER_SYSCTLS: + status_set('maintenance', 'Tuning device {}'.format( + network_interface)) + sysctl_file = os.path.join( + os.sep, + 'etc', + 'sysctl.d', + '51-ceph-osd-charm-{}.conf'.format(network_interface)) + try: + log("Saving sysctl_file: {} values: {}".format( + sysctl_file, NETWORK_ADAPTER_SYSCTLS[speed]), + level=DEBUG) + save_sysctls(sysctl_dict=NETWORK_ADAPTER_SYSCTLS[speed], + save_location=sysctl_file) + except IOError as e: + log("Write to /etc/sysctl.d/51-ceph-osd-charm-{} " + "failed. {}".format(network_interface, e), + level=ERROR) + + try: + # Apply the settings + log("Applying sysctl settings", level=DEBUG) + subprocess.check_output(["sysctl", "-p", sysctl_file]) + except subprocess.CalledProcessError as err: + log('sysctl -p {} failed with error {}'.format(sysctl_file, + err.output), + level=ERROR) + else: + log("No settings found for network adapter: {}".format( + network_interface), level=DEBUG) + + +def get_link_speed(network_interface): + """This will find the link speed for a given network device. Returns None + if an error occurs. + :param network_interface: string The network adapter interface. + :returns: LinkSpeed + """ + speed_path = os.path.join(os.sep, 'sys', 'class', 'net', + network_interface, 'speed') + # I'm not sure where else we'd check if this doesn't exist + if not os.path.exists(speed_path): + return LinkSpeed["UNKNOWN"] + + try: + with open(speed_path, 'r') as sysfs: + nic_speed = sysfs.readlines() + + # Did we actually read anything? + if not nic_speed: + return LinkSpeed["UNKNOWN"] + + # Try to find a sysctl match for this particular speed + for name, speed in LinkSpeed.items(): + if speed == int(nic_speed[0].strip()): + return speed + # Default to UNKNOWN if we can't find a match + return LinkSpeed["UNKNOWN"] + except IOError as e: + log("Unable to open {path} because of error: {error}".format( + path=speed_path, + error=e), level='error') + return LinkSpeed["UNKNOWN"] + + +def persist_settings(settings_dict): + # Write all settings to /etc/hdparm.conf + """ This will persist the hard drive settings to the /etc/hdparm.conf file + + The settings_dict should be in the form of {"uuid": {"key":"value"}} + + :param settings_dict: dict of settings to save + """ + if not settings_dict: + return + + try: + templating.render(source='hdparm.conf', target=HDPARM_FILE, + context=settings_dict) + except IOError as err: + log("Unable to open {path} because of error: {error}".format( + path=HDPARM_FILE, error=err), level=ERROR) + except Exception as e: + # The templating.render can raise a jinja2 exception if the + # template is not found. Rather than polluting the import + # space of this charm, simply catch Exception + log('Unable to render {path} due to error: {error}'.format( + path=HDPARM_FILE, error=e), level=ERROR) + + +def set_max_sectors_kb(dev_name, max_sectors_size): + """This function sets the max_sectors_kb size of a given block device. + + :param dev_name: Name of the block device to query + :param max_sectors_size: int of the max_sectors_size to save + """ + max_sectors_kb_path = os.path.join('sys', 'block', dev_name, 'queue', + 'max_sectors_kb') + try: + with open(max_sectors_kb_path, 'w') as f: + f.write(max_sectors_size) + except IOError as e: + log('Failed to write max_sectors_kb to {}. Error: {}'.format( + max_sectors_kb_path, e), level=ERROR) + + +def get_max_sectors_kb(dev_name): + """This function gets the max_sectors_kb size of a given block device. + + :param dev_name: Name of the block device to query + :returns: int which is either the max_sectors_kb or 0 on error. + """ + max_sectors_kb_path = os.path.join('sys', 'block', dev_name, 'queue', + 'max_sectors_kb') + + # Read in what Linux has set by default + if os.path.exists(max_sectors_kb_path): + try: + with open(max_sectors_kb_path, 'r') as f: + max_sectors_kb = f.read().strip() + return int(max_sectors_kb) + except IOError as e: + log('Failed to read max_sectors_kb to {}. Error: {}'.format( + max_sectors_kb_path, e), level=ERROR) + # Bail. + return 0 + return 0 + + +def get_max_hw_sectors_kb(dev_name): + """This function gets the max_hw_sectors_kb for a given block device. + + :param dev_name: Name of the block device to query + :returns: int which is either the max_hw_sectors_kb or 0 on error. + """ + max_hw_sectors_kb_path = os.path.join('sys', 'block', dev_name, 'queue', + 'max_hw_sectors_kb') + # Read in what the hardware supports + if os.path.exists(max_hw_sectors_kb_path): + try: + with open(max_hw_sectors_kb_path, 'r') as f: + max_hw_sectors_kb = f.read().strip() + return int(max_hw_sectors_kb) + except IOError as e: + log('Failed to read max_hw_sectors_kb to {}. Error: {}'.format( + max_hw_sectors_kb_path, e), level=ERROR) + return 0 + return 0 + + +def set_hdd_read_ahead(dev_name, read_ahead_sectors=256): + """This function sets the hard drive read ahead. + + :param dev_name: Name of the block device to set read ahead on. + :param read_ahead_sectors: int How many sectors to read ahead. + """ + try: + # Set the read ahead sectors to 256 + log('Setting read ahead to {} for device {}'.format( + read_ahead_sectors, + dev_name)) + subprocess.check_output(['hdparm', + '-a{}'.format(read_ahead_sectors), + dev_name]) + except subprocess.CalledProcessError as e: + log('hdparm failed with error: {}'.format(e.output), + level=ERROR) + + +def get_block_uuid(block_dev): + """This queries blkid to get the uuid for a block device. + + :param block_dev: Name of the block device to query. + :returns: The UUID of the device or None on Error. + """ + try: + block_info = str(subprocess + .check_output(['blkid', '-o', 'export', block_dev]) + .decode('UTF-8')) + for tag in block_info.split('\n'): + parts = tag.split('=') + if parts[0] == 'UUID': + return parts[1] + return None + except subprocess.CalledProcessError as err: + log('get_block_uuid failed with error: {}'.format(err.output), + level=ERROR) + return None + + +def check_max_sectors(save_settings_dict, + block_dev, + uuid): + """Tune the max_hw_sectors if needed. + + make sure that /sys/.../max_sectors_kb matches max_hw_sectors_kb or at + least 1MB for spinning disks + If the box has a RAID card with cache this could go much bigger. + + :param save_settings_dict: The dict used to persist settings + :param block_dev: A block device name: Example: /dev/sda + :param uuid: The uuid of the block device + """ + dev_name = None + path_parts = os.path.split(block_dev) + if len(path_parts) == 2: + dev_name = path_parts[1] + else: + log('Unable to determine the block device name from path: {}'.format( + block_dev)) + # Play it safe and bail + return + max_sectors_kb = get_max_sectors_kb(dev_name=dev_name) + max_hw_sectors_kb = get_max_hw_sectors_kb(dev_name=dev_name) + + if max_sectors_kb < max_hw_sectors_kb: + # OK we have a situation where the hardware supports more than Linux is + # currently requesting + config_max_sectors_kb = hookenv.config('max-sectors-kb') + if config_max_sectors_kb < max_hw_sectors_kb: + # Set the max_sectors_kb to the config.yaml value if it is less + # than the max_hw_sectors_kb + log('Setting max_sectors_kb for device {} to {}'.format( + dev_name, config_max_sectors_kb)) + save_settings_dict[ + "drive_settings"][uuid][ + "read_ahead_sect"] = config_max_sectors_kb + set_max_sectors_kb(dev_name=dev_name, + max_sectors_size=config_max_sectors_kb) + else: + # Set to the max_hw_sectors_kb + log('Setting max_sectors_kb for device {} to {}'.format( + dev_name, max_hw_sectors_kb)) + save_settings_dict[ + "drive_settings"][uuid]['read_ahead_sect'] = max_hw_sectors_kb + set_max_sectors_kb(dev_name=dev_name, + max_sectors_size=max_hw_sectors_kb) + else: + log('max_sectors_kb match max_hw_sectors_kb. No change needed for ' + 'device: {}'.format(block_dev)) + + +def tune_dev(block_dev): + """Try to make some intelligent decisions with HDD tuning. Future work will + include optimizing SSDs. + + This function will change the read ahead sectors and the max write + sectors for each block device. + + :param block_dev: A block device name: Example: /dev/sda + """ + uuid = get_block_uuid(block_dev) + if uuid is None: + log('block device {} uuid is None. Unable to save to ' + 'hdparm.conf'.format(block_dev), level=DEBUG) + return + save_settings_dict = {} + log('Tuning device {}'.format(block_dev)) + status_set('maintenance', 'Tuning device {}'.format(block_dev)) + set_hdd_read_ahead(block_dev) + save_settings_dict["drive_settings"] = {} + save_settings_dict["drive_settings"][uuid] = {} + save_settings_dict["drive_settings"][uuid]['read_ahead_sect'] = 256 + + check_max_sectors(block_dev=block_dev, + save_settings_dict=save_settings_dict, + uuid=uuid) + + persist_settings(settings_dict=save_settings_dict) + status_set('maintenance', 'Finished tuning device {}'.format(block_dev)) + + +def ceph_user(): + if get_version() > 1: + return 'ceph' + else: + return "root" + + +class CrushLocation(object): + def __init__(self, + name, + identifier, + host, + rack, + row, + datacenter, + chassis, + root): + self.name = name + self.identifier = identifier + self.host = host + self.rack = rack + self.row = row + self.datacenter = datacenter + self.chassis = chassis + self.root = root + + def __str__(self): + return "name: {} id: {} host: {} rack: {} row: {} datacenter: {} " \ + "chassis :{} root: {}".format(self.name, self.identifier, + self.host, self.rack, self.row, + self.datacenter, self.chassis, + self.root) + + def __eq__(self, other): + return not self.name < other.name and not other.name < self.name + + def __ne__(self, other): + return self.name < other.name or other.name < self.name + + def __gt__(self, other): + return self.name > other.name + + def __ge__(self, other): + return not self.name < other.name + + def __le__(self, other): + return self.name < other.name + + +def get_osd_weight(osd_id): + """Returns the weight of the specified OSD. + + :returns: Float + :raises: ValueError if the monmap fails to parse. + :raises: CalledProcessError if our ceph command fails. + """ + try: + tree = str(subprocess + .check_output(['ceph', 'osd', 'tree', '--format=json']) + .decode('UTF-8')) + try: + json_tree = json.loads(tree) + # Make sure children are present in the json + if not json_tree['nodes']: + return None + for device in json_tree['nodes']: + if device['type'] == 'osd' and device['name'] == osd_id: + return device['crush_weight'] + except ValueError as v: + log("Unable to parse ceph tree json: {}. Error: {}".format( + tree, v)) + raise + except subprocess.CalledProcessError as e: + log("ceph osd tree command failed with message: {}".format( + e)) + raise + + +def get_osd_tree(service): + """Returns the current osd map in JSON. + + :returns: List. + :raises: ValueError if the monmap fails to parse. + Also raises CalledProcessError if our ceph command fails + """ + try: + tree = str(subprocess + .check_output(['ceph', '--id', service, + 'osd', 'tree', '--format=json']) + .decode('UTF-8')) + try: + json_tree = json.loads(tree) + crush_list = [] + # Make sure children are present in the json + if not json_tree['nodes']: + return None + host_nodes = [ + node for node in json_tree['nodes'] + if node['type'] == 'host' + ] + for host in host_nodes: + crush_list.append( + CrushLocation( + name=host.get('name'), + identifier=host['id'], + host=host.get('host'), + rack=host.get('rack'), + row=host.get('row'), + datacenter=host.get('datacenter'), + chassis=host.get('chassis'), + root=host.get('root') + ) + ) + return crush_list + except ValueError as v: + log("Unable to parse ceph tree json: {}. Error: {}".format( + tree, v)) + raise + except subprocess.CalledProcessError as e: + log("ceph osd tree command failed with message: {}".format( + e)) + raise + + +def _get_child_dirs(path): + """Returns a list of directory names in the specified path. + + :param path: a full path listing of the parent directory to return child + directory names + :returns: list. A list of child directories under the parent directory + :raises: ValueError if the specified path does not exist or is not a + directory, + OSError if an error occurs reading the directory listing + """ + if not os.path.exists(path): + raise ValueError('Specfied path "%s" does not exist' % path) + if not os.path.isdir(path): + raise ValueError('Specified path "%s" is not a directory' % path) + + files_in_dir = [os.path.join(path, f) for f in os.listdir(path)] + return list(filter(os.path.isdir, files_in_dir)) + + +def _get_osd_num_from_dirname(dirname): + """Parses the dirname and returns the OSD id. + + Parses a string in the form of 'ceph-{osd#}' and returns the osd number + from the directory name. + + :param dirname: the directory name to return the OSD number from + :return int: the osd number the directory name corresponds to + :raises ValueError: if the osd number cannot be parsed from the provided + directory name. + """ + match = re.search(r'ceph-(?P\d+)', dirname) + if not match: + raise ValueError("dirname not in correct format: {}".format(dirname)) + + return match.group('osd_id') + + +def get_local_osd_ids(): + """This will list the /var/lib/ceph/osd/* directories and try + to split the ID off of the directory name and return it in + a list. + + :returns: list. A list of osd identifiers + :raises: OSError if something goes wrong with listing the directory. + """ + osd_ids = [] + osd_path = os.path.join(os.sep, 'var', 'lib', 'ceph', 'osd') + if os.path.exists(osd_path): + try: + dirs = os.listdir(osd_path) + for osd_dir in dirs: + osd_id = osd_dir.split('-')[1] + if _is_int(osd_id): + osd_ids.append(osd_id) + except OSError: + raise + return osd_ids + + +def get_local_mon_ids(): + """This will list the /var/lib/ceph/mon/* directories and try + to split the ID off of the directory name and return it in + a list. + + :returns: list. A list of monitor identifiers + :raises: OSError if something goes wrong with listing the directory. + """ + mon_ids = [] + mon_path = os.path.join(os.sep, 'var', 'lib', 'ceph', 'mon') + if os.path.exists(mon_path): + try: + dirs = os.listdir(mon_path) + for mon_dir in dirs: + # Basically this takes everything after ceph- as the monitor ID + match = re.search('ceph-(?P.*)', mon_dir) + if match: + mon_ids.append(match.group('mon_id')) + except OSError: + raise + return mon_ids + + +def _is_int(v): + """Return True if the object v can be turned into an integer.""" + try: + int(v) + return True + except ValueError: + return False + + +def get_version(): + """Derive Ceph release from an installed package.""" + import apt_pkg as apt + + cache = apt_cache() + package = "ceph" + try: + pkg = cache[package] + except KeyError: + # the package is unknown to the current apt cache. + e = 'Could not determine version of package with no installation ' \ + 'candidate: %s' % package + error_out(e) + + if not pkg.current_ver: + # package is known, but no version is currently installed. + e = 'Could not determine version of uninstalled package: %s' % package + error_out(e) + + vers = apt.upstream_version(pkg.current_ver.ver_str) + + # x.y match only for 20XX.X + # and ignore patch level for other packages + match = re.match(r'^(\d+)\.(\d+)', vers) + + if match: + vers = match.group(0) + return float(vers) + + +def error_out(msg): + log("FATAL ERROR: {}".format(msg), + level=ERROR) + sys.exit(1) + + +def is_quorum(): + asok = "/var/run/ceph/ceph-mon.{}.asok".format(socket.gethostname()) + cmd = [ + "sudo", + "-u", + ceph_user(), + "ceph", + "--admin-daemon", + asok, + "mon_status" + ] + if os.path.exists(asok): + try: + result = json.loads(str(subprocess + .check_output(cmd) + .decode('UTF-8'))) + except subprocess.CalledProcessError: + return False + except ValueError: + # Non JSON response from mon_status + return False + if result['state'] in QUORUM: + return True + else: + return False + else: + return False + + +def is_leader(): + asok = "/var/run/ceph/ceph-mon.{}.asok".format(socket.gethostname()) + cmd = [ + "sudo", + "-u", + ceph_user(), + "ceph", + "--admin-daemon", + asok, + "mon_status" + ] + if os.path.exists(asok): + try: + result = json.loads(str(subprocess + .check_output(cmd) + .decode('UTF-8'))) + except subprocess.CalledProcessError: + return False + except ValueError: + # Non JSON response from mon_status + return False + if result['state'] == LEADER: + return True + else: + return False + else: + return False + + +def manager_available(): + # if manager daemon isn't on this release, just say it is Fine + if cmp_pkgrevno('ceph', '11.0.0') < 0: + return True + cmd = ["sudo", "-u", "ceph", "ceph", "mgr", "dump", "-f", "json"] + try: + result = json.loads(subprocess.check_output(cmd).decode('UTF-8')) + return result['available'] + except subprocess.CalledProcessError as e: + log("'{}' failed: {}".format(" ".join(cmd), str(e))) + return False + except Exception: + return False + + +def wait_for_quorum(): + while not is_quorum(): + log("Waiting for quorum to be reached") + time.sleep(3) + + +def wait_for_manager(): + while not manager_available(): + log("Waiting for manager to be available") + time.sleep(5) + + +def add_bootstrap_hint(peer): + asok = "/var/run/ceph/ceph-mon.{}.asok".format(socket.gethostname()) + cmd = [ + "sudo", + "-u", + ceph_user(), + "ceph", + "--admin-daemon", + asok, + "add_bootstrap_peer_hint", + peer + ] + if os.path.exists(asok): + # Ignore any errors for this call + subprocess.call(cmd) + + +DISK_FORMATS = [ + 'xfs', + 'ext4', + 'btrfs' +] + +CEPH_PARTITIONS = [ + '89C57F98-2FE5-4DC0-89C1-5EC00CEFF2BE', # ceph encrypted disk in creation + '45B0969E-9B03-4F30-B4C6-5EC00CEFF106', # ceph encrypted journal + '4FBD7E29-9D25-41B8-AFD0-5EC00CEFF05D', # ceph encrypted osd data + '4FBD7E29-9D25-41B8-AFD0-062C0CEFF05D', # ceph osd data + '45B0969E-9B03-4F30-B4C6-B4B80CEFF106', # ceph osd journal + '89C57F98-2FE5-4DC0-89C1-F3AD0CEFF2BE', # ceph disk in creation +] + + +def get_partition_list(dev): + """Lists the partitions of a block device. + + :param dev: Path to a block device. ex: /dev/sda + :returns: Returns a list of Partition objects. + :raises: CalledProcessException if lsblk fails + """ + partitions_list = [] + try: + partitions = get_partitions(dev) + # For each line of output + for partition in partitions: + parts = partition.split() + try: + partitions_list.append( + Partition(number=parts[0], + start=parts[1], + end=parts[2], + sectors=parts[3], + size=parts[4], + name=parts[5], + uuid=parts[6]) + ) + except IndexError: + partitions_list.append( + Partition(number=parts[0], + start=parts[1], + end=parts[2], + sectors=parts[3], + size=parts[4], + name="", + uuid=parts[5]) + ) + + return partitions_list + except subprocess.CalledProcessError: + raise + + +def is_pristine_disk(dev): + """ + Read first 2048 bytes (LBA 0 - 3) of block device to determine whether it + is actually all zeros and safe for us to use. + + Existing partitioning tools does not discern between a failure to read from + block device, failure to understand a partition table and the fact that a + block device has no partition table. Since we need to be positive about + which is which we need to read the device directly and confirm ourselves. + + :param dev: Path to block device + :type dev: str + :returns: True all 2048 bytes == 0x0, False if not + :rtype: bool + """ + want_bytes = 2048 + + try: + f = open(dev, 'rb') + except OSError as e: + log(e) + return False + + data = f.read(want_bytes) + read_bytes = len(data) + if read_bytes != want_bytes: + log('{}: short read, got {} bytes expected {}.' + .format(dev, read_bytes, want_bytes), level=WARNING) + return False + + return all(byte == 0x0 for byte in data) + + +def is_osd_disk(dev): + db = kv() + osd_devices = db.get('osd-devices', []) + if dev in osd_devices: + log('Device {} already processed by charm,' + ' skipping'.format(dev)) + return True + + partitions = get_partition_list(dev) + for partition in partitions: + try: + info = str(subprocess + .check_output(['sgdisk', '-i', partition.number, dev]) + .decode('UTF-8')) + info = info.split("\n") # IGNORE:E1103 + for line in info: + for ptype in CEPH_PARTITIONS: + sig = 'Partition GUID code: {}'.format(ptype) + if line.startswith(sig): + return True + except subprocess.CalledProcessError as e: + log("sgdisk inspection of partition {} on {} failed with " + "error: {}. Skipping".format(partition.minor, dev, e), + level=ERROR) + return False + + +def start_osds(devices): + # Scan for ceph block devices + rescan_osd_devices() + if (cmp_pkgrevno('ceph', '0.56.6') >= 0 and + cmp_pkgrevno('ceph', '14.2.0') < 0): + # Use ceph-disk activate for directory based OSD's + for dev_or_path in devices: + if os.path.exists(dev_or_path) and os.path.isdir(dev_or_path): + subprocess.check_call( + ['ceph-disk', 'activate', dev_or_path]) + + +def udevadm_settle(): + cmd = ['udevadm', 'settle'] + subprocess.call(cmd) + + +def rescan_osd_devices(): + cmd = [ + 'udevadm', 'trigger', + '--subsystem-match=block', '--action=add' + ] + + subprocess.call(cmd) + + udevadm_settle() + + +_client_admin_keyring = '/etc/ceph/ceph.client.admin.keyring' + + +def is_bootstrapped(): + return os.path.exists( + '/var/lib/ceph/mon/ceph-{}/done'.format(socket.gethostname())) + + +def wait_for_bootstrap(): + while not is_bootstrapped(): + time.sleep(3) + + +def generate_monitor_secret(): + cmd = [ + 'ceph-authtool', + '/dev/stdout', + '--name=mon.', + '--gen-key' + ] + res = str(subprocess.check_output(cmd).decode('UTF-8')) + + return "{}==".format(res.split('=')[1].strip()) + + +# OSD caps taken from ceph-create-keys +_osd_bootstrap_caps = { + 'mon': [ + 'allow command osd create ...', + 'allow command osd crush set ...', + r'allow command auth add * osd allow\ * mon allow\ rwx', + 'allow command mon getmap' + ] +} + +_osd_bootstrap_caps_profile = { + 'mon': [ + 'allow profile bootstrap-osd' + ] +} + + +def parse_key(raw_key): + # get-or-create appears to have different output depending + # on whether its 'get' or 'create' + # 'create' just returns the key, 'get' is more verbose and + # needs parsing + key = None + if len(raw_key.splitlines()) == 1: + key = raw_key + else: + for element in raw_key.splitlines(): + if 'key' in element: + return element.split(' = ')[1].strip() # IGNORE:E1103 + return key + + +def get_osd_bootstrap_key(): + try: + # Attempt to get/create a key using the OSD bootstrap profile first + key = get_named_key('bootstrap-osd', + _osd_bootstrap_caps_profile) + except Exception: + # If that fails try with the older style permissions + key = get_named_key('bootstrap-osd', + _osd_bootstrap_caps) + return key + + +_radosgw_keyring = "/etc/ceph/keyring.rados.gateway" + + +def import_radosgw_key(key): + if not os.path.exists(_radosgw_keyring): + cmd = [ + "sudo", + "-u", + ceph_user(), + 'ceph-authtool', + _radosgw_keyring, + '--create-keyring', + '--name=client.radosgw.gateway', + '--add-key={}'.format(key) + ] + subprocess.check_call(cmd) + + +# OSD caps taken from ceph-create-keys +_radosgw_caps = { + 'mon': ['allow rw'], + 'osd': ['allow rwx'] +} +_upgrade_caps = { + 'mon': ['allow rwx'] +} + + +def get_radosgw_key(pool_list=None, name=None): + return get_named_key(name=name or 'radosgw.gateway', + caps=_radosgw_caps, + pool_list=pool_list) + + +def get_mds_key(name): + return create_named_keyring(entity='mds', + name=name, + caps=mds_caps) + + +_mds_bootstrap_caps_profile = { + 'mon': [ + 'allow profile bootstrap-mds' + ] +} + + +def get_mds_bootstrap_key(): + return get_named_key('bootstrap-mds', + _mds_bootstrap_caps_profile) + + +_default_caps = collections.OrderedDict([ + ('mon', ['allow r', + 'allow command "osd blacklist"']), + ('osd', ['allow rwx']), +]) + +admin_caps = collections.OrderedDict([ + ('mds', ['allow *']), + ('mgr', ['allow *']), + ('mon', ['allow *']), + ('osd', ['allow *']) +]) + +mds_caps = collections.OrderedDict([ + ('osd', ['allow *']), + ('mds', ['allow']), + ('mon', ['allow rwx']), +]) + +osd_upgrade_caps = collections.OrderedDict([ + ('mon', ['allow command "config-key"', + 'allow command "osd tree"', + 'allow command "config-key list"', + 'allow command "config-key put"', + 'allow command "config-key get"', + 'allow command "config-key exists"', + 'allow command "osd out"', + 'allow command "osd in"', + 'allow command "osd rm"', + 'allow command "auth del"', + ]) +]) + +rbd_mirror_caps = collections.OrderedDict([ + ('mon', ['profile rbd; allow r']), + ('osd', ['profile rbd']), + ('mgr', ['allow r']), +]) + + +def get_rbd_mirror_key(name): + return get_named_key(name=name, caps=rbd_mirror_caps) + + +def create_named_keyring(entity, name, caps=None): + caps = caps or _default_caps + cmd = [ + "sudo", + "-u", + ceph_user(), + 'ceph', + '--name', 'mon.', + '--keyring', + '/var/lib/ceph/mon/ceph-{}/keyring'.format( + socket.gethostname() + ), + 'auth', 'get-or-create', '{entity}.{name}'.format(entity=entity, + name=name), + ] + for subsystem, subcaps in caps.items(): + cmd.extend([subsystem, '; '.join(subcaps)]) + log("Calling check_output: {}".format(cmd), level=DEBUG) + return (parse_key(str(subprocess + .check_output(cmd) + .decode('UTF-8')) + .strip())) # IGNORE:E1103 + + +def get_upgrade_key(): + return get_named_key('upgrade-osd', _upgrade_caps) + + +def get_named_key(name, caps=None, pool_list=None): + """Retrieve a specific named cephx key. + + :param name: String Name of key to get. + :param pool_list: The list of pools to give access to + :param caps: dict of cephx capabilities + :returns: Returns a cephx key + """ + key_name = 'client.{}'.format(name) + try: + # Does the key already exist? + output = str(subprocess.check_output( + [ + 'sudo', + '-u', ceph_user(), + 'ceph', + '--name', 'mon.', + '--keyring', + '/var/lib/ceph/mon/ceph-{}/keyring'.format( + socket.gethostname() + ), + 'auth', + 'get', + key_name, + ]).decode('UTF-8')).strip() + # NOTE(jamespage); + # Apply any changes to key capabilities, dealing with + # upgrades which requires new caps for operation. + upgrade_key_caps(key_name, + caps or _default_caps, + pool_list) + return parse_key(output) + except subprocess.CalledProcessError: + # Couldn't get the key, time to create it! + log("Creating new key for {}".format(name), level=DEBUG) + caps = caps or _default_caps + cmd = [ + "sudo", + "-u", + ceph_user(), + 'ceph', + '--name', 'mon.', + '--keyring', + '/var/lib/ceph/mon/ceph-{}/keyring'.format( + socket.gethostname() + ), + 'auth', 'get-or-create', key_name, + ] + # Add capabilities + for subsystem, subcaps in caps.items(): + if subsystem == 'osd': + if pool_list: + # This will output a string similar to: + # "pool=rgw pool=rbd pool=something" + pools = " ".join(['pool={0}'.format(i) for i in pool_list]) + subcaps[0] = subcaps[0] + " " + pools + cmd.extend([subsystem, '; '.join(subcaps)]) + + log("Calling check_output: {}".format(cmd), level=DEBUG) + return parse_key(str(subprocess + .check_output(cmd) + .decode('UTF-8')) + .strip()) # IGNORE:E1103 + + +def upgrade_key_caps(key, caps, pool_list=None): + """ Upgrade key to have capabilities caps """ + if not is_leader(): + # Not the MON leader OR not clustered + return + cmd = [ + "sudo", "-u", ceph_user(), 'ceph', 'auth', 'caps', key + ] + for subsystem, subcaps in caps.items(): + if subsystem == 'osd': + if pool_list: + # This will output a string similar to: + # "pool=rgw pool=rbd pool=something" + pools = " ".join(['pool={0}'.format(i) for i in pool_list]) + subcaps[0] = subcaps[0] + " " + pools + cmd.extend([subsystem, '; '.join(subcaps)]) + subprocess.check_call(cmd) + + +@cached +def systemd(): + return CompareHostReleases(lsb_release()['DISTRIB_CODENAME']) >= 'vivid' + + +def use_bluestore(): + """Determine whether bluestore should be used for OSD's + + :returns: whether bluestore disk format should be used + :rtype: bool""" + if cmp_pkgrevno('ceph', '12.2.0') < 0: + return False + return config('bluestore') + + +def bootstrap_monitor_cluster(secret): + """Bootstrap local ceph mon into the ceph cluster + + :param secret: cephx secret to use for monitor authentication + :type secret: str + :raises: Exception if ceph mon cannot be bootstrapped + """ + hostname = socket.gethostname() + path = '/var/lib/ceph/mon/ceph-{}'.format(hostname) + done = '{}/done'.format(path) + if systemd(): + init_marker = '{}/systemd'.format(path) + else: + init_marker = '{}/upstart'.format(path) + + keyring = '/var/lib/ceph/tmp/{}.mon.keyring'.format(hostname) + + if os.path.exists(done): + log('bootstrap_monitor_cluster: mon already initialized.') + else: + # Ceph >= 0.61.3 needs this for ceph-mon fs creation + mkdir('/var/run/ceph', owner=ceph_user(), + group=ceph_user(), perms=0o755) + mkdir(path, owner=ceph_user(), group=ceph_user(), + perms=0o755) + # end changes for Ceph >= 0.61.3 + try: + _create_monitor(keyring, + secret, + hostname, + path, + done, + init_marker) + except Exception: + raise + finally: + os.unlink(keyring) + + +def _create_monitor(keyring, secret, hostname, path, done, init_marker): + """Create monitor filesystem and enable and start ceph-mon process + + :param keyring: path to temporary keyring on disk + :type keyring: str + :param secret: cephx secret to use for monitor authentication + :type: secret: str + :param hostname: hostname of the local unit + :type hostname: str + :param path: full path to ceph mon directory + :type path: str + :param done: full path to 'done' marker for ceph mon + :type done: str + :param init_marker: full path to 'init' marker for ceph mon + :type init_marker: str + """ + subprocess.check_call(['ceph-authtool', keyring, + '--create-keyring', '--name=mon.', + '--add-key={}'.format(secret), + '--cap', 'mon', 'allow *']) + subprocess.check_call(['ceph-mon', '--mkfs', + '-i', hostname, + '--keyring', keyring]) + chownr('/var/log/ceph', ceph_user(), ceph_user()) + chownr(path, ceph_user(), ceph_user()) + with open(done, 'w'): + pass + with open(init_marker, 'w'): + pass + + if systemd(): + if cmp_pkgrevno('ceph', '14.0.0') >= 0: + systemd_unit = 'ceph-mon@{}'.format(socket.gethostname()) + else: + systemd_unit = 'ceph-mon' + subprocess.check_call(['systemctl', 'enable', systemd_unit]) + service_restart(systemd_unit) + else: + service_restart('ceph-mon-all') + + +def create_keyrings(): + """Create keyrings for operation of ceph-mon units + + NOTE: The quorum should be done before to execute this function. + + :raises: Exception if keyrings cannot be created + """ + if cmp_pkgrevno('ceph', '14.0.0') >= 0: + # NOTE(jamespage): At Nautilus, keys are created by the + # monitors automatically and just need + # exporting. + output = str(subprocess.check_output( + [ + 'sudo', + '-u', ceph_user(), + 'ceph', + '--name', 'mon.', + '--keyring', + '/var/lib/ceph/mon/ceph-{}/keyring'.format( + socket.gethostname() + ), + 'auth', 'get', 'client.admin', + ]).decode('UTF-8')).strip() + if not output: + # NOTE: key not yet created, raise exception and retry + raise Exception + # NOTE: octopus wants newline at end of file LP: #1864706 + output += '\n' + write_file(_client_admin_keyring, output, + owner=ceph_user(), group=ceph_user(), + perms=0o400) + else: + # NOTE(jamespage): Later ceph releases require explicit + # call to ceph-create-keys to setup the + # admin keys for the cluster; this command + # will wait for quorum in the cluster before + # returning. + # NOTE(fnordahl): Explicitly run `ceph-create-keys` for older + # ceph releases too. This improves bootstrap + # resilience as the charm will wait for + # presence of peer units before attempting + # to bootstrap. Note that charms deploying + # ceph-mon service should disable running of + # `ceph-create-keys` service in init system. + cmd = ['ceph-create-keys', '--id', socket.gethostname()] + if cmp_pkgrevno('ceph', '12.0.0') >= 0: + # NOTE(fnordahl): The default timeout in ceph-create-keys of 600 + # seconds is not adequate. Increase timeout when + # timeout parameter available. For older releases + # we rely on retry_on_exception decorator. + # LP#1719436 + cmd.extend(['--timeout', '1800']) + subprocess.check_call(cmd) + osstat = os.stat(_client_admin_keyring) + if not osstat.st_size: + # NOTE(fnordahl): Retry will fail as long as this file exists. + # LP#1719436 + os.remove(_client_admin_keyring) + raise Exception + + +def update_monfs(): + hostname = socket.gethostname() + monfs = '/var/lib/ceph/mon/ceph-{}'.format(hostname) + if systemd(): + init_marker = '{}/systemd'.format(monfs) + else: + init_marker = '{}/upstart'.format(monfs) + if os.path.exists(monfs) and not os.path.exists(init_marker): + # Mark mon as managed by upstart so that + # it gets start correctly on reboots + with open(init_marker, 'w'): + pass + + +def get_partitions(dev): + cmd = ['partx', '--raw', '--noheadings', dev] + try: + out = str(subprocess.check_output(cmd).decode('UTF-8')).splitlines() + log("get partitions: {}".format(out), level=DEBUG) + return out + except subprocess.CalledProcessError as e: + log("Can't get info for {0}: {1}".format(dev, e.output)) + return [] + + +def get_lvs(dev): + """ + List logical volumes for the provided block device + + :param: dev: Full path to block device. + :raises subprocess.CalledProcessError: in the event that any supporting + operation failed. + :returns: list: List of logical volumes provided by the block device + """ + if not lvm.is_lvm_physical_volume(dev): + return [] + vg_name = lvm.list_lvm_volume_group(dev) + return lvm.list_logical_volumes('vg_name={}'.format(vg_name)) + + +def find_least_used_utility_device(utility_devices, lvs=False): + """ + Find a utility device which has the smallest number of partitions + among other devices in the supplied list. + + :utility_devices: A list of devices to be used for filestore journal + or bluestore wal or db. + :lvs: flag to indicate whether inspection should be based on LVM LV's + :return: string device name + """ + if lvs: + usages = map(lambda a: (len(get_lvs(a)), a), utility_devices) + else: + usages = map(lambda a: (len(get_partitions(a)), a), utility_devices) + least = min(usages, key=lambda t: t[0]) + return least[1] + + +def get_devices(name): + """ Merge config and juju storage based devices + + :name: THe name of the device type, eg: wal, osd, journal + :returns: Set(device names), which are strings + """ + if config(name): + devices = [dev.strip() for dev in config(name).split(' ')] + else: + devices = [] + storage_ids = storage_list(name) + devices.extend((storage_get('location', sid) for sid in storage_ids)) + devices = filter(os.path.exists, devices) + + return set(devices) + + +def osdize(dev, osd_format, osd_journal, ignore_errors=False, encrypt=False, + bluestore=False, key_manager=CEPH_KEY_MANAGER): + if dev.startswith('/dev'): + osdize_dev(dev, osd_format, osd_journal, + ignore_errors, encrypt, + bluestore, key_manager) + else: + if cmp_pkgrevno('ceph', '14.0.0') >= 0: + log("Directory backed OSDs can not be created on Nautilus", + level=WARNING) + return + osdize_dir(dev, encrypt, bluestore) + + +def osdize_dev(dev, osd_format, osd_journal, ignore_errors=False, + encrypt=False, bluestore=False, key_manager=CEPH_KEY_MANAGER): + """ + Prepare a block device for use as a Ceph OSD + + A block device will only be prepared once during the lifetime + of the calling charm unit; future executions will be skipped. + + :param: dev: Full path to block device to use + :param: osd_format: Format for OSD filesystem + :param: osd_journal: List of block devices to use for OSD journals + :param: ignore_errors: Don't fail in the event of any errors during + processing + :param: encrypt: Encrypt block devices using 'key_manager' + :param: bluestore: Use bluestore native ceph block device format + :param: key_manager: Key management approach for encryption keys + :raises subprocess.CalledProcessError: in the event that any supporting + subprocess operation failed + :raises ValueError: if an invalid key_manager is provided + """ + if key_manager not in KEY_MANAGERS: + raise ValueError('Unsupported key manager: {}'.format(key_manager)) + + db = kv() + osd_devices = db.get('osd-devices', []) + try: + if dev in osd_devices: + log('Device {} already processed by charm,' + ' skipping'.format(dev)) + return + + if not os.path.exists(dev): + log('Path {} does not exist - bailing'.format(dev)) + return + + if not is_block_device(dev): + log('Path {} is not a block device - bailing'.format(dev)) + return + + if is_osd_disk(dev): + log('Looks like {} is already an' + ' OSD data or journal, skipping.'.format(dev)) + if is_device_mounted(dev): + osd_devices.append(dev) + return + + if is_device_mounted(dev): + log('Looks like {} is in use, skipping.'.format(dev)) + return + + if is_active_bluestore_device(dev): + log('{} is in use as an active bluestore block device,' + ' skipping.'.format(dev)) + osd_devices.append(dev) + return + + if is_mapped_luks_device(dev): + log('{} is a mapped LUKS device,' + ' skipping.'.format(dev)) + return + + if cmp_pkgrevno('ceph', '12.2.4') >= 0: + cmd = _ceph_volume(dev, + osd_journal, + encrypt, + bluestore, + key_manager) + else: + cmd = _ceph_disk(dev, + osd_format, + osd_journal, + encrypt, + bluestore) + + try: + status_set('maintenance', 'Initializing device {}'.format(dev)) + log("osdize cmd: {}".format(cmd)) + subprocess.check_call(cmd) + except subprocess.CalledProcessError: + try: + lsblk_output = subprocess.check_output( + ['lsblk', '-P']).decode('UTF-8') + except subprocess.CalledProcessError as e: + log("Couldn't get lsblk output: {}".format(e), ERROR) + if ignore_errors: + log('Unable to initialize device: {}'.format(dev), WARNING) + if lsblk_output: + log('lsblk output: {}'.format(lsblk_output), DEBUG) + else: + log('Unable to initialize device: {}'.format(dev), ERROR) + if lsblk_output: + log('lsblk output: {}'.format(lsblk_output), WARNING) + raise + + # NOTE: Record processing of device only on success to ensure that + # the charm only tries to initialize a device of OSD usage + # once during its lifetime. + osd_devices.append(dev) + finally: + db.set('osd-devices', osd_devices) + db.flush() + + +def _ceph_disk(dev, osd_format, osd_journal, encrypt=False, bluestore=False): + """ + Prepare a device for usage as a Ceph OSD using ceph-disk + + :param: dev: Full path to use for OSD block device setup, + The function looks up realpath of the device + :param: osd_journal: List of block devices to use for OSD journals + :param: encrypt: Use block device encryption (unsupported) + :param: bluestore: Use bluestore storage for OSD + :returns: list. 'ceph-disk' command and required parameters for + execution by check_call + """ + cmd = ['ceph-disk', 'prepare'] + + if encrypt: + cmd.append('--dmcrypt') + + if osd_format and not bluestore: + cmd.append('--fs-type') + cmd.append(osd_format) + + # NOTE(jamespage): enable experimental bluestore support + if use_bluestore(): + cmd.append('--bluestore') + wal = get_devices('bluestore-wal') + if wal: + cmd.append('--block.wal') + least_used_wal = find_least_used_utility_device(wal) + cmd.append(least_used_wal) + db = get_devices('bluestore-db') + if db: + cmd.append('--block.db') + least_used_db = find_least_used_utility_device(db) + cmd.append(least_used_db) + elif cmp_pkgrevno('ceph', '12.1.0') >= 0 and not bluestore: + cmd.append('--filestore') + + cmd.append(os.path.realpath(dev)) + + if osd_journal: + least_used = find_least_used_utility_device(osd_journal) + cmd.append(least_used) + + return cmd + + +def _ceph_volume(dev, osd_journal, encrypt=False, bluestore=False, + key_manager=CEPH_KEY_MANAGER): + """ + Prepare and activate a device for usage as a Ceph OSD using ceph-volume. + + This also includes creation of all PV's, VG's and LV's required to + support the initialization of the OSD. + + :param: dev: Full path to use for OSD block device setup + :param: osd_journal: List of block devices to use for OSD journals + :param: encrypt: Use block device encryption + :param: bluestore: Use bluestore storage for OSD + :param: key_manager: dm-crypt Key Manager to use + :raises subprocess.CalledProcessError: in the event that any supporting + LVM operation failed. + :returns: list. 'ceph-volume' command and required parameters for + execution by check_call + """ + cmd = ['ceph-volume', 'lvm', 'create'] + + osd_fsid = str(uuid.uuid4()) + cmd.append('--osd-fsid') + cmd.append(osd_fsid) + + if bluestore: + cmd.append('--bluestore') + main_device_type = 'block' + else: + cmd.append('--filestore') + main_device_type = 'data' + + if encrypt and key_manager == CEPH_KEY_MANAGER: + cmd.append('--dmcrypt') + + # On-disk journal volume creation + if not osd_journal and not bluestore: + journal_lv_type = 'journal' + cmd.append('--journal') + cmd.append(_allocate_logical_volume( + dev=dev, + lv_type=journal_lv_type, + osd_fsid=osd_fsid, + size='{}M'.format(calculate_volume_size('journal')), + encrypt=encrypt, + key_manager=key_manager) + ) + + cmd.append('--data') + cmd.append(_allocate_logical_volume(dev=dev, + lv_type=main_device_type, + osd_fsid=osd_fsid, + encrypt=encrypt, + key_manager=key_manager)) + + if bluestore: + for extra_volume in ('wal', 'db'): + devices = get_devices('bluestore-{}'.format(extra_volume)) + if devices: + cmd.append('--block.{}'.format(extra_volume)) + least_used = find_least_used_utility_device(devices, + lvs=True) + cmd.append(_allocate_logical_volume( + dev=least_used, + lv_type=extra_volume, + osd_fsid=osd_fsid, + size='{}M'.format(calculate_volume_size(extra_volume)), + shared=True, + encrypt=encrypt, + key_manager=key_manager) + ) + + elif osd_journal: + cmd.append('--journal') + least_used = find_least_used_utility_device(osd_journal, + lvs=True) + cmd.append(_allocate_logical_volume( + dev=least_used, + lv_type='journal', + osd_fsid=osd_fsid, + size='{}M'.format(calculate_volume_size('journal')), + shared=True, + encrypt=encrypt, + key_manager=key_manager) + ) + + return cmd + + +def _partition_name(dev): + """ + Derive the first partition name for a block device + + :param: dev: Full path to block device. + :returns: str: Full path to first partition on block device. + """ + if dev[-1].isdigit(): + return '{}p1'.format(dev) + else: + return '{}1'.format(dev) + + +def is_active_bluestore_device(dev): + """ + Determine whether provided device is part of an active + bluestore based OSD (as its block component). + + :param: dev: Full path to block device to check for Bluestore usage. + :returns: boolean: indicating whether device is in active use. + """ + if not lvm.is_lvm_physical_volume(dev): + return False + + vg_name = lvm.list_lvm_volume_group(dev) + try: + lv_name = lvm.list_logical_volumes('vg_name={}'.format(vg_name))[0] + except IndexError: + return False + + block_symlinks = glob.glob('/var/lib/ceph/osd/ceph-*/block') + for block_candidate in block_symlinks: + if os.path.islink(block_candidate): + target = os.readlink(block_candidate) + if target.endswith(lv_name): + return True + + return False + + +def is_luks_device(dev): + """ + Determine if dev is a LUKS-formatted block device. + + :param: dev: A full path to a block device to check for LUKS header + presence + :returns: boolean: indicates whether a device is used based on LUKS header. + """ + return True if _luks_uuid(dev) else False + + +def is_mapped_luks_device(dev): + """ + Determine if dev is a mapped LUKS device + :param: dev: A full path to a block device to be checked + :returns: boolean: indicates whether a device is mapped + """ + _, dirs, _ = next(os.walk( + '/sys/class/block/{}/holders/' + .format(os.path.basename(os.path.realpath(dev)))) + ) + is_held = len(dirs) > 0 + return is_held and is_luks_device(dev) + + +def get_conf(variable): + """ + Get the value of the given configuration variable from the + cluster. + + :param variable: ceph configuration variable + :returns: str. configured value for provided variable + + """ + return subprocess.check_output([ + 'ceph-osd', + '--show-config-value={}'.format(variable), + '--no-mon-config', + ]).strip() + + +def calculate_volume_size(lv_type): + """ + Determine the configured size for Bluestore DB/WAL or + Filestore Journal devices + + :param lv_type: volume type (db, wal or journal) + :raises KeyError: if invalid lv_type is supplied + :returns: int. Configured size in megabytes for volume type + """ + # lv_type -> ceph configuration option + _config_map = { + 'db': 'bluestore_block_db_size', + 'wal': 'bluestore_block_wal_size', + 'journal': 'osd_journal_size', + } + + # default sizes in MB + _default_size = { + 'db': 1024, + 'wal': 576, + 'journal': 1024, + } + + # conversion of ceph config units to MB + _units = { + 'db': 1048576, # Bytes -> MB + 'wal': 1048576, # Bytes -> MB + 'journal': 1, # Already in MB + } + + configured_size = get_conf(_config_map[lv_type]) + + if configured_size is None or int(configured_size) == 0: + return _default_size[lv_type] + else: + return int(configured_size) / _units[lv_type] + + +def _luks_uuid(dev): + """ + Check to see if dev is a LUKS encrypted volume, returning the UUID + of volume if it is. + + :param: dev: path to block device to check. + :returns: str. UUID of LUKS device or None if not a LUKS device + """ + try: + cmd = ['cryptsetup', 'luksUUID', dev] + return subprocess.check_output(cmd).decode('UTF-8').strip() + except subprocess.CalledProcessError: + return None + + +def _initialize_disk(dev, dev_uuid, encrypt=False, + key_manager=CEPH_KEY_MANAGER): + """ + Initialize a raw block device consuming 100% of the avaliable + disk space. + + Function assumes that block device has already been wiped. + + :param: dev: path to block device to initialize + :param: dev_uuid: UUID to use for any dm-crypt operations + :param: encrypt: Encrypt OSD devices using dm-crypt + :param: key_manager: Key management approach for dm-crypt keys + :raises: subprocess.CalledProcessError: if any parted calls fail + :returns: str: Full path to new partition. + """ + use_vaultlocker = encrypt and key_manager == VAULT_KEY_MANAGER + + if use_vaultlocker: + # NOTE(jamespage): Check to see if already initialized as a LUKS + # volume, which indicates this is a shared block + # device for journal, db or wal volumes. + luks_uuid = _luks_uuid(dev) + if luks_uuid: + return '/dev/mapper/crypt-{}'.format(luks_uuid) + + dm_crypt = '/dev/mapper/crypt-{}'.format(dev_uuid) + + if use_vaultlocker and not os.path.exists(dm_crypt): + subprocess.check_call([ + 'vaultlocker', + 'encrypt', + '--uuid', dev_uuid, + dev, + ]) + subprocess.check_call([ + 'dd', + 'if=/dev/zero', + 'of={}'.format(dm_crypt), + 'bs=512', + 'count=1', + ]) + + if use_vaultlocker: + return dm_crypt + else: + return dev + + +def _allocate_logical_volume(dev, lv_type, osd_fsid, + size=None, shared=False, + encrypt=False, + key_manager=CEPH_KEY_MANAGER): + """ + Allocate a logical volume from a block device, ensuring any + required initialization and setup of PV's and VG's to support + the LV. + + :param: dev: path to block device to allocate from. + :param: lv_type: logical volume type to create + (data, block, journal, wal, db) + :param: osd_fsid: UUID of the OSD associate with the LV + :param: size: Size in LVM format for the device; + if unset 100% of VG + :param: shared: Shared volume group (journal, wal, db) + :param: encrypt: Encrypt OSD devices using dm-crypt + :param: key_manager: dm-crypt Key Manager to use + :raises subprocess.CalledProcessError: in the event that any supporting + LVM or parted operation fails. + :returns: str: String in the format 'vg_name/lv_name'. + """ + lv_name = "osd-{}-{}".format(lv_type, osd_fsid) + current_volumes = lvm.list_logical_volumes() + if shared: + dev_uuid = str(uuid.uuid4()) + else: + dev_uuid = osd_fsid + pv_dev = _initialize_disk(dev, dev_uuid, encrypt, key_manager) + + vg_name = None + if not lvm.is_lvm_physical_volume(pv_dev): + lvm.create_lvm_physical_volume(pv_dev) + if not os.path.exists(pv_dev): + # NOTE: trigger rescan to work around bug 1878752 + rescan_osd_devices() + if shared: + vg_name = 'ceph-{}-{}'.format(lv_type, + str(uuid.uuid4())) + else: + vg_name = 'ceph-{}'.format(osd_fsid) + lvm.create_lvm_volume_group(vg_name, pv_dev) + else: + vg_name = lvm.list_lvm_volume_group(pv_dev) + + if lv_name not in current_volumes: + lvm.create_logical_volume(lv_name, vg_name, size) + + return "{}/{}".format(vg_name, lv_name) + + +def osdize_dir(path, encrypt=False, bluestore=False): + """Ask ceph-disk to prepare a directory to become an osd. + + :param path: str. The directory to osdize + :param encrypt: bool. Should the OSD directory be encrypted at rest + :returns: None + """ + + db = kv() + osd_devices = db.get('osd-devices', []) + if path in osd_devices: + log('Device {} already processed by charm,' + ' skipping'.format(path)) + return + + for t in ['upstart', 'systemd']: + if os.path.exists(os.path.join(path, t)): + log('Path {} is already used as an OSD dir - bailing'.format(path)) + return + + if cmp_pkgrevno('ceph', "0.56.6") < 0: + log('Unable to use directories for OSDs with ceph < 0.56.6', + level=ERROR) + return + + mkdir(path, owner=ceph_user(), group=ceph_user(), perms=0o755) + chownr('/var/lib/ceph', ceph_user(), ceph_user()) + cmd = [ + 'sudo', '-u', ceph_user(), + 'ceph-disk', + 'prepare', + '--data-dir', + path + ] + if cmp_pkgrevno('ceph', '0.60') >= 0: + if encrypt: + cmd.append('--dmcrypt') + + # NOTE(icey): enable experimental bluestore support + if cmp_pkgrevno('ceph', '10.2.0') >= 0 and bluestore: + cmd.append('--bluestore') + elif cmp_pkgrevno('ceph', '12.1.0') >= 0 and not bluestore: + cmd.append('--filestore') + log("osdize dir cmd: {}".format(cmd)) + subprocess.check_call(cmd) + + # NOTE: Record processing of device only on success to ensure that + # the charm only tries to initialize a device of OSD usage + # once during its lifetime. + osd_devices.append(path) + db.set('osd-devices', osd_devices) + db.flush() + + +def filesystem_mounted(fs): + return subprocess.call(['grep', '-wqs', fs, '/proc/mounts']) == 0 + + +def get_running_osds(): + """Returns a list of the pids of the current running OSD daemons""" + cmd = ['pgrep', 'ceph-osd'] + try: + result = str(subprocess.check_output(cmd).decode('UTF-8')) + return result.split() + except subprocess.CalledProcessError: + return [] + + +def get_cephfs(service): + """List the Ceph Filesystems that exist. + + :param service: The service name to run the ceph command under + :returns: list. Returns a list of the ceph filesystems + """ + if get_version() < 0.86: + # This command wasn't introduced until 0.86 ceph + return [] + try: + output = str(subprocess + .check_output(["ceph", '--id', service, "fs", "ls"]) + .decode('UTF-8')) + if not output: + return [] + """ + Example subprocess output: + 'name: ip-172-31-23-165, metadata pool: ip-172-31-23-165_metadata, + data pools: [ip-172-31-23-165_data ]\n' + output: filesystems: ['ip-172-31-23-165'] + """ + filesystems = [] + for line in output.splitlines(): + parts = line.split(',') + for part in parts: + if "name" in part: + filesystems.append(part.split(' ')[1]) + except subprocess.CalledProcessError: + return [] + + +def wait_for_all_monitors_to_upgrade(new_version, upgrade_key): + """Fairly self explanatory name. This function will wait + for all monitors in the cluster to upgrade or it will + return after a timeout period has expired. + + :param new_version: str of the version to watch + :param upgrade_key: the cephx key name to use + """ + done = False + start_time = time.time() + monitor_list = [] + + mon_map = get_mon_map('admin') + if mon_map['monmap']['mons']: + for mon in mon_map['monmap']['mons']: + monitor_list.append(mon['name']) + while not done: + try: + done = all(monitor_key_exists(upgrade_key, "{}_{}_{}_done".format( + "mon", mon, new_version + )) for mon in monitor_list) + current_time = time.time() + if current_time > (start_time + 10 * 60): + raise Exception + else: + # Wait 30 seconds and test again if all monitors are upgraded + time.sleep(30) + except subprocess.CalledProcessError: + raise + + +# Edge cases: +# 1. Previous node dies on upgrade, can we retry? +def roll_monitor_cluster(new_version, upgrade_key): + """This is tricky to get right so here's what we're going to do. + + There's 2 possible cases: Either I'm first in line or not. + If I'm not first in line I'll wait a random time between 5-30 seconds + and test to see if the previous monitor is upgraded yet. + + :param new_version: str of the version to upgrade to + :param upgrade_key: the cephx key name to use when upgrading + """ + log('roll_monitor_cluster called with {}'.format(new_version)) + my_name = socket.gethostname() + monitor_list = [] + mon_map = get_mon_map('admin') + if mon_map['monmap']['mons']: + for mon in mon_map['monmap']['mons']: + monitor_list.append(mon['name']) + else: + status_set('blocked', 'Unable to get monitor cluster information') + sys.exit(1) + log('monitor_list: {}'.format(monitor_list)) + + # A sorted list of osd unit names + mon_sorted_list = sorted(monitor_list) + + try: + position = mon_sorted_list.index(my_name) + log("upgrade position: {}".format(position)) + if position == 0: + # I'm first! Roll + # First set a key to inform others I'm about to roll + lock_and_roll(upgrade_key=upgrade_key, + service='mon', + my_name=my_name, + version=new_version) + else: + # Check if the previous node has finished + status_set('waiting', + 'Waiting on {} to finish upgrading'.format( + mon_sorted_list[position - 1])) + wait_on_previous_node(upgrade_key=upgrade_key, + service='mon', + previous_node=mon_sorted_list[position - 1], + version=new_version) + lock_and_roll(upgrade_key=upgrade_key, + service='mon', + my_name=my_name, + version=new_version) + # NOTE(jamespage): + # Wait until all monitors have upgraded before bootstrapping + # the ceph-mgr daemons due to use of new mgr keyring profiles + if new_version == 'luminous': + wait_for_all_monitors_to_upgrade(new_version=new_version, + upgrade_key=upgrade_key) + bootstrap_manager() + except ValueError: + log("Failed to find {} in list {}.".format( + my_name, mon_sorted_list)) + status_set('blocked', 'failed to upgrade monitor') + + +# For E731 we can't assign a lambda, therefore, instead pass this. +def noop(): + pass + + +def upgrade_monitor(new_version, kick_function=None): + """Upgrade the current ceph monitor to the new version + + :param new_version: String version to upgrade to. + """ + if kick_function is None: + kick_function = noop + current_version = get_version() + status_set("maintenance", "Upgrading monitor") + log("Current ceph version is {}".format(current_version)) + log("Upgrading to: {}".format(new_version)) + + # Needed to determine if whether to stop/start ceph-mgr + luminous_or_later = cmp_pkgrevno('ceph-common', '12.2.0') >= 0 + + kick_function() + try: + add_source(config('source'), config('key')) + apt_update(fatal=True) + except subprocess.CalledProcessError as err: + log("Adding the ceph source failed with message: {}".format( + err)) + status_set("blocked", "Upgrade to {} failed".format(new_version)) + sys.exit(1) + kick_function() + try: + if systemd(): + service_stop('ceph-mon') + log("restarting ceph-mgr.target maybe: {}" + .format(luminous_or_later)) + if luminous_or_later: + service_stop('ceph-mgr.target') + else: + service_stop('ceph-mon-all') + apt_install(packages=determine_packages(), fatal=True) + kick_function() + + owner = ceph_user() + + # Ensure the files and directories under /var/lib/ceph is chowned + # properly as part of the move to the Jewel release, which moved the + # ceph daemons to running as ceph:ceph instead of root:root. + if new_version == 'jewel': + # Ensure the ownership of Ceph's directories is correct + chownr(path=os.path.join(os.sep, "var", "lib", "ceph"), + owner=owner, + group=owner, + follow_links=True) + + kick_function() + + # Ensure that mon directory is user writable + hostname = socket.gethostname() + path = '/var/lib/ceph/mon/ceph-{}'.format(hostname) + mkdir(path, owner=ceph_user(), group=ceph_user(), + perms=0o755) + + if systemd(): + service_restart('ceph-mon') + log("starting ceph-mgr.target maybe: {}".format(luminous_or_later)) + if luminous_or_later: + # due to BUG: #1849874 we have to force a restart to get it to + # drop the previous version of ceph-manager and start the new + # one. + service_restart('ceph-mgr.target') + else: + service_start('ceph-mon-all') + except subprocess.CalledProcessError as err: + log("Stopping ceph and upgrading packages failed " + "with message: {}".format(err)) + status_set("blocked", "Upgrade to {} failed".format(new_version)) + sys.exit(1) + + +def lock_and_roll(upgrade_key, service, my_name, version): + """Create a lock on the ceph monitor cluster and upgrade. + + :param upgrade_key: str. The cephx key to use + :param service: str. The cephx id to use + :param my_name: str. The current hostname + :param version: str. The version we are upgrading to + """ + start_timestamp = time.time() + + log('monitor_key_set {}_{}_{}_start {}'.format( + service, + my_name, + version, + start_timestamp)) + monitor_key_set(upgrade_key, "{}_{}_{}_start".format( + service, my_name, version), start_timestamp) + + # alive indication: + alive_function = ( + lambda: monitor_key_set( + upgrade_key, "{}_{}_{}_alive" + .format(service, my_name, version), time.time())) + dog = WatchDog(kick_interval=3 * 60, + kick_function=alive_function) + + log("Rolling") + + # This should be quick + if service == 'osd': + upgrade_osd(version, kick_function=dog.kick_the_dog) + elif service == 'mon': + upgrade_monitor(version, kick_function=dog.kick_the_dog) + else: + log("Unknown service {}. Unable to upgrade".format(service), + level=ERROR) + log("Done") + + stop_timestamp = time.time() + # Set a key to inform others I am finished + log('monitor_key_set {}_{}_{}_done {}'.format(service, + my_name, + version, + stop_timestamp)) + status_set('maintenance', 'Finishing upgrade') + monitor_key_set(upgrade_key, "{}_{}_{}_done".format(service, + my_name, + version), + stop_timestamp) + + +def wait_on_previous_node(upgrade_key, service, previous_node, version): + """A lock that sleeps the current thread while waiting for the previous + node to finish upgrading. + + :param upgrade_key: + :param service: str. the cephx id to use + :param previous_node: str. The name of the previous node to wait on + :param version: str. The version we are upgrading to + :returns: None + """ + log("Previous node is: {}".format(previous_node)) + + previous_node_started_f = ( + lambda: monitor_key_exists( + upgrade_key, + "{}_{}_{}_start".format(service, previous_node, version))) + previous_node_finished_f = ( + lambda: monitor_key_exists( + upgrade_key, + "{}_{}_{}_done".format(service, previous_node, version))) + previous_node_alive_time_f = ( + lambda: monitor_key_get( + upgrade_key, + "{}_{}_{}_alive".format(service, previous_node, version))) + + # wait for 30 minutes until the previous node starts. We don't proceed + # unless we get a start condition. + try: + WatchDog.wait_until(previous_node_started_f, timeout=30 * 60) + except WatchDog.WatchDogTimeoutException: + log("Waited for previous node to start for 30 minutes. " + "It didn't start, so may have a serious issue. Continuing with " + "upgrade of this node.", + level=WARNING) + return + + # keep the time it started from this nodes' perspective. + previous_node_started_at = time.time() + log("Detected that previous node {} has started. Time now: {}" + .format(previous_node, previous_node_started_at)) + + # Now wait for the node to complete. The node may optionally be kicking + # with the *_alive key, which allows this node to wait longer as it 'knows' + # the other node is proceeding. + try: + WatchDog.timed_wait(kicked_at_function=previous_node_alive_time_f, + complete_function=previous_node_finished_f, + wait_time=30 * 60, + compatibility_wait_time=10 * 60, + max_kick_interval=5 * 60) + except WatchDog.WatchDogDeadException: + # previous node was kicking, but timed out; log this condition and move + # on. + now = time.time() + waited = int((now - previous_node_started_at) / 60) + log("Previous node started, but has now not ticked for 5 minutes. " + "Waited total of {} mins on node {}. current time: {} > " + "previous node start time: {}. " + "Continuing with upgrade of this node." + .format(waited, previous_node, now, previous_node_started_at), + level=WARNING) + except WatchDog.WatchDogTimeoutException: + # previous node never kicked, or simply took too long; log this + # condition and move on. + now = time.time() + waited = int((now - previous_node_started_at) / 60) + log("Previous node is taking too long; assuming it has died." + "Waited {} mins on node {}. current time: {} > " + "previous node start time: {}. " + "Continuing with upgrade of this node." + .format(waited, previous_node, now, previous_node_started_at), + level=WARNING) + + +class WatchDog(object): + """Watch a dog; basically a kickable timer with a timeout between two async + units. + + The idea is that you have an overall timeout and then can kick that timeout + with intermediary hits, with a max time between those kicks allowed. + + Note that this watchdog doesn't rely on the clock of the other side; just + roughly when it detects when the other side started. All timings are based + on the local clock. + + The kicker will not 'kick' more often than a set interval, regardless of + how often the kick_the_dog() function is called. The kicker provides a + function (lambda: -> None) that is called when the kick interval is + reached. + + The waiter calls the static method with a check function + (lambda: -> Boolean) that indicates when the wait should be over and the + maximum interval to wait. e.g. 30 minutes with a 5 minute kick interval. + + So the waiter calls wait(f, 30, 3) and the kicker sets up a 3 minute kick + interval, or however long it is expected for the key to propagate and to + allow for other delays. + + There is a compatibility mode where if the otherside never kicks, then it + simply waits for the compatability timer. + """ + + class WatchDogDeadException(Exception): + pass + + class WatchDogTimeoutException(Exception): + pass + + def __init__(self, kick_interval=3 * 60, kick_function=None): + """Initialise a new WatchDog + + :param kick_interval: the interval when this side kicks the other in + seconds. + :type kick_interval: Int + :param kick_function: The function to call that does the kick. + :type kick_function: Callable[] + """ + self.start_time = time.time() + self.last_run_func = None + self.last_kick_at = None + self.kick_interval = kick_interval + self.kick_f = kick_function + + def kick_the_dog(self): + """Might call the kick_function if it's time. + + This function can be called as frequently as needed, but will run the + self.kick_function after kick_interval seconds have passed. + """ + now = time.time() + if (self.last_run_func is None or + (now - self.last_run_func > self.kick_interval)): + if self.kick_f is not None: + self.kick_f() + self.last_run_func = now + self.last_kick_at = now + + @staticmethod + def wait_until(wait_f, timeout=10 * 60): + """Wait for timeout seconds until the passed function return True. + + :param wait_f: The function to call that will end the wait. + :type wait_f: Callable[[], Boolean] + :param timeout: The time to wait in seconds. + :type timeout: int + """ + start_time = time.time() + while(not wait_f()): + now = time.time() + if now > start_time + timeout: + raise WatchDog.WatchDogTimeoutException() + wait_time = random.randrange(5, 30) + log('wait_until: waiting for {} seconds'.format(wait_time)) + time.sleep(wait_time) + + @staticmethod + def timed_wait(kicked_at_function, + complete_function, + wait_time=30 * 60, + compatibility_wait_time=10 * 60, + max_kick_interval=5 * 60): + """Wait a maximum time with an intermediate 'kick' time. + + This function will wait for max_kick_interval seconds unless the + kicked_at_function() call returns a time that is not older that + max_kick_interval (in seconds). i.e. the other side can signal that it + is still doing things during the max_kick_interval as long as it kicks + at least every max_kick_interval seconds. + + The maximum wait is "wait_time", but the otherside must keep kicking + during this period. + + The "compatibility_wait_time" is used if the other side never kicks + (i.e. the kicked_at_function() always returns None. In this case the + function wait up to "compatibility_wait_time". + + Note that the type of the return from the kicked_at_function is an + Optional[str], not a Float. The function will coerce this to a float + for the comparison. This represents the return value of + time.time() at the "other side". It's a string to simplify the + function obtaining the time value from the other side. + + The function raises WatchDogTimeoutException if either the + compatibility_wait_time or the wait_time are exceeded. + + The function raises WatchDogDeadException if the max_kick_interval is + exceeded. + + Note that it is possible that the first kick interval is extended to + compatibility_wait_time if the "other side" doesn't kick immediately. + The best solution is for the other side to kick early and often. + + :param kicked_at_function: The function to call to retrieve the time + that the other side 'kicked' at. None if the other side hasn't + kicked. + :type kicked_at_function: Callable[[], Optional[str]] + :param complete_function: The callable that returns True when done. + :type complete_function: Callable[[], Boolean] + :param wait_time: the maximum time to wait, even with kicks, in + seconds. + :type wait_time: int + :param compatibility_wait_time: The time to wait if no kicks are + received, in seconds. + :type compatibility_wait_time: int + :param max_kick_interval: The maximum time allowed between kicks before + the wait is over, in seconds: + :type max_kick_interval: int + :raises: WatchDog.WatchDogTimeoutException, + WatchDog.WatchDogDeadException + """ + start_time = time.time() + while True: + if complete_function(): + break + # the time when the waiting for unit last kicked. + kicked_at = kicked_at_function() + now = time.time() + if kicked_at is None: + # assume other end doesn't do alive kicks + if (now - start_time > compatibility_wait_time): + raise WatchDog.WatchDogTimeoutException() + else: + # other side is participating in kicks; must kick at least + # every 'max_kick_interval' to stay alive. + if (now - float(kicked_at) > max_kick_interval): + raise WatchDog.WatchDogDeadException() + if (now - start_time > wait_time): + raise WatchDog.WatchDogTimeoutException() + delay_time = random.randrange(5, 30) + log('waiting for {} seconds'.format(delay_time)) + time.sleep(delay_time) + + +def get_upgrade_position(osd_sorted_list, match_name): + """Return the upgrade position for the given osd. + + :param osd_sorted_list: Osds sorted + :type osd_sorted_list: [str] + :param match_name: The osd name to match + :type match_name: str + :returns: The position of the name + :rtype: int + :raises: ValueError if name is not found + """ + for index, item in enumerate(osd_sorted_list): + if item.name == match_name: + return index + raise ValueError("osd name '{}' not found in get_upgrade_position list" + .format(match_name)) + + +# Edge cases: +# 1. Previous node dies on upgrade, can we retry? +# 2. This assumes that the osd failure domain is not set to osd. +# It rolls an entire server at a time. +def roll_osd_cluster(new_version, upgrade_key): + """This is tricky to get right so here's what we're going to do. + + There's 2 possible cases: Either I'm first in line or not. + If I'm not first in line I'll wait a random time between 5-30 seconds + and test to see if the previous osd is upgraded yet. + + TODO: If you're not in the same failure domain it's safe to upgrade + 1. Examine all pools and adopt the most strict failure domain policy + Example: Pool 1: Failure domain = rack + Pool 2: Failure domain = host + Pool 3: Failure domain = row + + outcome: Failure domain = host + + :param new_version: str of the version to upgrade to + :param upgrade_key: the cephx key name to use when upgrading + """ + log('roll_osd_cluster called with {}'.format(new_version)) + my_name = socket.gethostname() + osd_tree = get_osd_tree(service=upgrade_key) + # A sorted list of osd unit names + osd_sorted_list = sorted(osd_tree) + log("osd_sorted_list: {}".format(osd_sorted_list)) + + try: + position = get_upgrade_position(osd_sorted_list, my_name) + log("upgrade position: {}".format(position)) + if position == 0: + # I'm first! Roll + # First set a key to inform others I'm about to roll + lock_and_roll(upgrade_key=upgrade_key, + service='osd', + my_name=my_name, + version=new_version) + else: + # Check if the previous node has finished + status_set('waiting', + 'Waiting on {} to finish upgrading'.format( + osd_sorted_list[position - 1].name)) + wait_on_previous_node( + upgrade_key=upgrade_key, + service='osd', + previous_node=osd_sorted_list[position - 1].name, + version=new_version) + lock_and_roll(upgrade_key=upgrade_key, + service='osd', + my_name=my_name, + version=new_version) + except ValueError: + log("Failed to find name {} in list {}".format( + my_name, osd_sorted_list)) + status_set('blocked', 'failed to upgrade osd') + + +def upgrade_osd(new_version, kick_function=None): + """Upgrades the current osd + + :param new_version: str. The new version to upgrade to + """ + if kick_function is None: + kick_function = noop + + current_version = get_version() + status_set("maintenance", "Upgrading osd") + log("Current ceph version is {}".format(current_version)) + log("Upgrading to: {}".format(new_version)) + + try: + add_source(config('source'), config('key')) + apt_update(fatal=True) + except subprocess.CalledProcessError as err: + log("Adding the ceph sources failed with message: {}".format( + err)) + status_set("blocked", "Upgrade to {} failed".format(new_version)) + sys.exit(1) + + kick_function() + + try: + # Upgrade the packages before restarting the daemons. + status_set('maintenance', 'Upgrading packages to %s' % new_version) + apt_install(packages=determine_packages(), fatal=True) + kick_function() + + # If the upgrade does not need an ownership update of any of the + # directories in the osd service directory, then simply restart + # all of the OSDs at the same time as this will be the fastest + # way to update the code on the node. + if not dirs_need_ownership_update('osd'): + log('Restarting all OSDs to load new binaries', DEBUG) + with maintain_all_osd_states(): + if systemd(): + service_restart('ceph-osd.target') + else: + service_restart('ceph-osd-all') + return + + # Need to change the ownership of all directories which are not OSD + # directories as well. + # TODO - this should probably be moved to the general upgrade function + # and done before mon/osd. + update_owner(CEPH_BASE_DIR, recurse_dirs=False) + non_osd_dirs = filter(lambda x: not x == 'osd', + os.listdir(CEPH_BASE_DIR)) + non_osd_dirs = map(lambda x: os.path.join(CEPH_BASE_DIR, x), + non_osd_dirs) + for i, path in enumerate(non_osd_dirs): + if i % 100 == 0: + kick_function() + update_owner(path) + + # Fast service restart wasn't an option because each of the OSD + # directories need the ownership updated for all the files on + # the OSD. Walk through the OSDs one-by-one upgrading the OSD. + for osd_dir in _get_child_dirs(OSD_BASE_DIR): + kick_function() + try: + osd_num = _get_osd_num_from_dirname(osd_dir) + _upgrade_single_osd(osd_num, osd_dir) + except ValueError as ex: + # Directory could not be parsed - junk directory? + log('Could not parse osd directory %s: %s' % (osd_dir, ex), + WARNING) + continue + + except (subprocess.CalledProcessError, IOError) as err: + log("Stopping ceph and upgrading packages failed " + "with message: {}".format(err)) + status_set("blocked", "Upgrade to {} failed".format(new_version)) + sys.exit(1) + + +def _upgrade_single_osd(osd_num, osd_dir): + """Upgrades the single OSD directory. + + :param osd_num: the num of the OSD + :param osd_dir: the directory of the OSD to upgrade + :raises CalledProcessError: if an error occurs in a command issued as part + of the upgrade process + :raises IOError: if an error occurs reading/writing to a file as part + of the upgrade process + """ + with maintain_osd_state(osd_num): + stop_osd(osd_num) + disable_osd(osd_num) + update_owner(osd_dir) + enable_osd(osd_num) + start_osd(osd_num) + + +def stop_osd(osd_num): + """Stops the specified OSD number. + + :param osd_num: the osd number to stop + """ + if systemd(): + service_stop('ceph-osd@{}'.format(osd_num)) + else: + service_stop('ceph-osd', id=osd_num) + + +def start_osd(osd_num): + """Starts the specified OSD number. + + :param osd_num: the osd number to start. + """ + if systemd(): + service_start('ceph-osd@{}'.format(osd_num)) + else: + service_start('ceph-osd', id=osd_num) + + +def disable_osd(osd_num): + """Disables the specified OSD number. + + Ensures that the specified osd will not be automatically started at the + next reboot of the system. Due to differences between init systems, + this method cannot make any guarantees that the specified osd cannot be + started manually. + + :param osd_num: the osd id which should be disabled. + :raises CalledProcessError: if an error occurs invoking the systemd cmd + to disable the OSD + :raises IOError, OSError: if the attempt to read/remove the ready file in + an upstart enabled system fails + """ + if systemd(): + # When running under systemd, the individual ceph-osd daemons run as + # templated units and can be directly addressed by referring to the + # templated service name ceph-osd@. Additionally, systemd + # allows one to disable a specific templated unit by running the + # 'systemctl disable ceph-osd@' command. When disabled, the + # OSD should remain disabled until re-enabled via systemd. + # Note: disabling an already disabled service in systemd returns 0, so + # no need to check whether it is enabled or not. + cmd = ['systemctl', 'disable', 'ceph-osd@{}'.format(osd_num)] + subprocess.check_call(cmd) + else: + # Neither upstart nor the ceph-osd upstart script provides for + # disabling the starting of an OSD automatically. The specific OSD + # cannot be prevented from running manually, however it can be + # prevented from running automatically on reboot by removing the + # 'ready' file in the OSD's root directory. This is due to the + # ceph-osd-all upstart script checking for the presence of this file + # before starting the OSD. + ready_file = os.path.join(OSD_BASE_DIR, 'ceph-{}'.format(osd_num), + 'ready') + if os.path.exists(ready_file): + os.unlink(ready_file) + + +def enable_osd(osd_num): + """Enables the specified OSD number. + + Ensures that the specified osd_num will be enabled and ready to start + automatically in the event of a reboot. + + :param osd_num: the osd id which should be enabled. + :raises CalledProcessError: if the call to the systemd command issued + fails when enabling the service + :raises IOError: if the attempt to write the ready file in an usptart + enabled system fails + """ + if systemd(): + cmd = ['systemctl', 'enable', 'ceph-osd@{}'.format(osd_num)] + subprocess.check_call(cmd) + else: + # When running on upstart, the OSDs are started via the ceph-osd-all + # upstart script which will only start the osd if it has a 'ready' + # file. Make sure that file exists. + ready_file = os.path.join(OSD_BASE_DIR, 'ceph-{}'.format(osd_num), + 'ready') + with open(ready_file, 'w') as f: + f.write('ready') + + # Make sure the correct user owns the file. It shouldn't be necessary + # as the upstart script should run with root privileges, but its better + # to have all the files matching ownership. + update_owner(ready_file) + + +def update_owner(path, recurse_dirs=True): + """Changes the ownership of the specified path. + + Changes the ownership of the specified path to the new ceph daemon user + using the system's native chown functionality. This may take awhile, + so this method will issue a set_status for any changes of ownership which + recurses into directory structures. + + :param path: the path to recursively change ownership for + :param recurse_dirs: boolean indicating whether to recursively change the + ownership of all the files in a path's subtree or to + simply change the ownership of the path. + :raises CalledProcessError: if an error occurs issuing the chown system + command + """ + user = ceph_user() + user_group = '{ceph_user}:{ceph_user}'.format(ceph_user=user) + cmd = ['chown', user_group, path] + if os.path.isdir(path) and recurse_dirs: + status_set('maintenance', ('Updating ownership of %s to %s' % + (path, user))) + cmd.insert(1, '-R') + + log('Changing ownership of {path} to {user}'.format( + path=path, user=user_group), DEBUG) + start = datetime.now() + subprocess.check_call(cmd) + elapsed_time = (datetime.now() - start) + + log('Took {secs} seconds to change the ownership of path: {path}'.format( + secs=elapsed_time.total_seconds(), path=path), DEBUG) + + +def get_osd_state(osd_num, osd_goal_state=None): + """Get OSD state or loop until OSD state matches OSD goal state. + + If osd_goal_state is None, just return the current OSD state. + If osd_goal_state is not None, loop until the current OSD state matches + the OSD goal state. + + :param osd_num: the osd id to get state for + :param osd_goal_state: (Optional) string indicating state to wait for + Defaults to None + :returns: Returns a str, the OSD state. + :rtype: str + """ + while True: + asok = "/var/run/ceph/ceph-osd.{}.asok".format(osd_num) + cmd = [ + 'ceph', + 'daemon', + asok, + 'status' + ] + try: + result = json.loads(str(subprocess + .check_output(cmd) + .decode('UTF-8'))) + except (subprocess.CalledProcessError, ValueError) as e: + log("{}".format(e), level=DEBUG) + continue + osd_state = result['state'] + log("OSD {} state: {}, goal state: {}".format( + osd_num, osd_state, osd_goal_state), level=DEBUG) + if not osd_goal_state: + return osd_state + if osd_state == osd_goal_state: + return osd_state + time.sleep(3) + + +def get_all_osd_states(osd_goal_states=None): + """Get all OSD states or loop until all OSD states match OSD goal states. + + If osd_goal_states is None, just return a dictionary of current OSD states. + If osd_goal_states is not None, loop until the current OSD states match + the OSD goal states. + + :param osd_goal_states: (Optional) dict indicating states to wait for + Defaults to None + :returns: Returns a dictionary of current OSD states. + :rtype: dict + """ + osd_states = {} + for osd_num in get_local_osd_ids(): + if not osd_goal_states: + osd_states[osd_num] = get_osd_state(osd_num) + else: + osd_states[osd_num] = get_osd_state( + osd_num, + osd_goal_state=osd_goal_states[osd_num]) + return osd_states + + +@contextmanager +def maintain_osd_state(osd_num): + """Ensure the state of an OSD is maintained. + + Ensures the state of an OSD is the same at the end of a block nested + in a with statement as it was at the beginning of the block. + + :param osd_num: the osd id to maintain state for + """ + osd_state = get_osd_state(osd_num) + try: + yield + finally: + get_osd_state(osd_num, osd_goal_state=osd_state) + + +@contextmanager +def maintain_all_osd_states(): + """Ensure all local OSD states are maintained. + + Ensures the states of all local OSDs are the same at the end of a + block nested in a with statement as they were at the beginning of + the block. + """ + osd_states = get_all_osd_states() + try: + yield + finally: + get_all_osd_states(osd_goal_states=osd_states) + + +def list_pools(client='admin'): + """This will list the current pools that Ceph has + + :param client: (Optional) client id for ceph key to use + Defaults to ``admin`` + :type cilent: str + :returns: Returns a list of available pools. + :rtype: list + :raises: subprocess.CalledProcessError if the subprocess fails to run. + """ + try: + pool_list = [] + pools = subprocess.check_output(['rados', '--id', client, 'lspools'], + universal_newlines=True, + stderr=subprocess.STDOUT) + for pool in pools.splitlines(): + pool_list.append(pool) + return pool_list + except subprocess.CalledProcessError as err: + log("rados lspools failed with error: {}".format(err.output)) + raise + + +def get_pool_param(pool, param, client='admin'): + """Get parameter from pool. + + :param pool: Name of pool to get variable from + :type pool: str + :param param: Name of variable to get + :type param: str + :param client: (Optional) client id for ceph key to use + Defaults to ``admin`` + :type cilent: str + :returns: Value of variable on pool or None + :rtype: str or None + :raises: subprocess.CalledProcessError + """ + try: + output = subprocess.check_output( + ['ceph', '--id', client, 'osd', 'pool', 'get', pool, param], + universal_newlines=True, stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as cp: + if cp.returncode == 2 and 'ENOENT: option' in cp.output: + return None + raise + if ':' in output: + return output.split(':')[1].lstrip().rstrip() + + +def get_pool_erasure_profile(pool, client='admin'): + """Get erasure code profile for pool. + + :param pool: Name of pool to get variable from + :type pool: str + :param client: (Optional) client id for ceph key to use + Defaults to ``admin`` + :type cilent: str + :returns: Erasure code profile of pool or None + :rtype: str or None + :raises: subprocess.CalledProcessError + """ + try: + return get_pool_param(pool, 'erasure_code_profile', client=client) + except subprocess.CalledProcessError as cp: + if cp.returncode == 13 and 'EACCES: pool' in cp.output: + # Not a Erasure coded pool + return None + raise + + +def get_pool_quota(pool, client='admin'): + """Get pool quota. + + :param pool: Name of pool to get variable from + :type pool: str + :param client: (Optional) client id for ceph key to use + Defaults to ``admin`` + :type cilent: str + :returns: Dictionary with quota variables + :rtype: dict + :raises: subprocess.CalledProcessError + """ + output = subprocess.check_output( + ['ceph', '--id', client, 'osd', 'pool', 'get-quota', pool], + universal_newlines=True, stderr=subprocess.STDOUT) + rc = re.compile(r'\s+max\s+(\S+)\s*:\s+(\d+)') + result = {} + for line in output.splitlines(): + m = rc.match(line) + if m: + result.update({'max_{}'.format(m.group(1)): m.group(2)}) + return result + + +def get_pool_applications(pool='', client='admin'): + """Get pool applications. + + :param pool: (Optional) Name of pool to get applications for + Defaults to get for all pools + :type pool: str + :param client: (Optional) client id for ceph key to use + Defaults to ``admin`` + :type cilent: str + :returns: Dictionary with pool name as key + :rtype: dict + :raises: subprocess.CalledProcessError + """ + + cmd = ['ceph', '--id', client, 'osd', 'pool', 'application', 'get'] + if pool: + cmd.append(pool) + try: + output = subprocess.check_output(cmd, + universal_newlines=True, + stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as cp: + if cp.returncode == 2 and 'ENOENT' in cp.output: + return {} + raise + return json.loads(output) + + +def list_pools_detail(): + """Get detailed information about pools. + + Structure: + {'pool_name_1': {'applications': {'application': {}}, + 'parameters': {'pg_num': '42', 'size': '42'}, + 'quota': {'max_bytes': '1000', + 'max_objects': '10'}, + }, + 'pool_name_2': ... + } + + :returns: Dictionary with detailed pool information. + :rtype: dict + :raises: subproces.CalledProcessError + """ + get_params = ['pg_num', 'size'] + result = {} + applications = get_pool_applications() + for pool in list_pools(): + result[pool] = { + 'applications': applications.get(pool, {}), + 'parameters': {}, + 'quota': get_pool_quota(pool), + } + for param in get_params: + result[pool]['parameters'].update({ + param: get_pool_param(pool, param)}) + erasure_profile = get_pool_erasure_profile(pool) + if erasure_profile: + result[pool]['parameters'].update({ + 'erasure_code_profile': erasure_profile}) + return result + + +def dirs_need_ownership_update(service): + """Determines if directories still need change of ownership. + + Examines the set of directories under the /var/lib/ceph/{service} directory + and determines if they have the correct ownership or not. This is + necessary due to the upgrade from Hammer to Jewel where the daemon user + changes from root: to ceph:. + + :param service: the name of the service folder to check (e.g. osd, mon) + :returns: boolean. True if the directories need a change of ownership, + False otherwise. + :raises IOError: if an error occurs reading the file stats from one of + the child directories. + :raises OSError: if the specified path does not exist or some other error + """ + expected_owner = expected_group = ceph_user() + path = os.path.join(CEPH_BASE_DIR, service) + for child in _get_child_dirs(path): + curr_owner, curr_group = owner(child) + + if (curr_owner == expected_owner) and (curr_group == expected_group): + continue + + # NOTE(lathiat): when config_changed runs on reboot, the OSD might not + # yet be mounted or started, and the underlying directory the OSD is + # mounted to is expected to be owned by root. So skip the check. This + # may also happen for OSD directories for OSDs that were removed. + if (service == 'osd' and + not os.path.exists(os.path.join(child, 'magic'))): + continue + + log('Directory "%s" needs its ownership updated' % child, DEBUG) + return True + + # All child directories had the expected ownership + return False + + +# A dict of valid ceph upgrade paths. Mapping is old -> new +UPGRADE_PATHS = collections.OrderedDict([ + ('firefly', 'hammer'), + ('hammer', 'jewel'), + ('jewel', 'luminous'), + ('luminous', 'mimic'), + ('mimic', 'nautilus'), + ('nautilus', 'octopus'), +]) + +# Map UCA codenames to ceph codenames +UCA_CODENAME_MAP = { + 'icehouse': 'firefly', + 'juno': 'firefly', + 'kilo': 'hammer', + 'liberty': 'hammer', + 'mitaka': 'jewel', + 'newton': 'jewel', + 'ocata': 'jewel', + 'pike': 'luminous', + 'queens': 'luminous', + 'rocky': 'mimic', + 'stein': 'mimic', + 'train': 'nautilus', + 'ussuri': 'octopus', +} + + +def pretty_print_upgrade_paths(): + """Pretty print supported upgrade paths for ceph""" + return ["{} -> {}".format(key, value) + for key, value in UPGRADE_PATHS.items()] + + +def resolve_ceph_version(source): + """Resolves a version of ceph based on source configuration + based on Ubuntu Cloud Archive pockets. + + @param: source: source configuration option of charm + :returns: ceph release codename or None if not resolvable + """ + os_release = get_os_codename_install_source(source) + return UCA_CODENAME_MAP.get(os_release) + + +def get_ceph_pg_stat(): + """Returns the result of ceph pg stat. + + :returns: dict + """ + try: + tree = str(subprocess + .check_output(['ceph', 'pg', 'stat', '--format=json']) + .decode('UTF-8')) + try: + json_tree = json.loads(tree) + if not json_tree['num_pg_by_state']: + return None + return json_tree + except ValueError as v: + log("Unable to parse ceph pg stat json: {}. Error: {}".format( + tree, v)) + raise + except subprocess.CalledProcessError as e: + log("ceph pg stat command failed with message: {}".format(e)) + raise + + +def get_ceph_health(): + """Returns the health of the cluster from a 'ceph status' + + :returns: dict tree of ceph status + :raises: CalledProcessError if our ceph command fails to get the overall + status, use get_ceph_health()['overall_status']. + """ + try: + tree = str(subprocess + .check_output(['ceph', 'status', '--format=json']) + .decode('UTF-8')) + try: + json_tree = json.loads(tree) + # Make sure children are present in the json + if not json_tree['overall_status']: + return None + + return json_tree + except ValueError as v: + log("Unable to parse ceph tree json: {}. Error: {}".format( + tree, v)) + raise + except subprocess.CalledProcessError as e: + log("ceph status command failed with message: {}".format(e)) + raise + + +def reweight_osd(osd_num, new_weight): + """Changes the crush weight of an OSD to the value specified. + + :param osd_num: the osd id which should be changed + :param new_weight: the new weight for the OSD + :returns: bool. True if output looks right, else false. + :raises CalledProcessError: if an error occurs invoking the systemd cmd + """ + try: + cmd_result = str(subprocess + .check_output(['ceph', 'osd', 'crush', + 'reweight', "osd.{}".format(osd_num), + new_weight], + stderr=subprocess.STDOUT) + .decode('UTF-8')) + expected_result = "reweighted item id {ID} name \'osd.{ID}\'".format( + ID=osd_num) + " to {}".format(new_weight) + log(cmd_result) + if expected_result in cmd_result: + return True + return False + except subprocess.CalledProcessError as e: + log("ceph osd crush reweight command failed" + " with message: {}".format(e)) + raise + + +def determine_packages(): + """Determines packages for installation. + + :returns: list of ceph packages + """ + packages = PACKAGES.copy() + if CompareHostReleases(lsb_release()['DISTRIB_CODENAME']) >= 'eoan': + btrfs_package = 'btrfs-progs' + else: + btrfs_package = 'btrfs-tools' + packages.append(btrfs_package) + return packages + + +def bootstrap_manager(): + hostname = socket.gethostname() + path = '/var/lib/ceph/mgr/ceph-{}'.format(hostname) + keyring = os.path.join(path, 'keyring') + + if os.path.exists(keyring): + log('bootstrap_manager: mgr already initialized.') + else: + mkdir(path, owner=ceph_user(), group=ceph_user()) + subprocess.check_call(['ceph', 'auth', 'get-or-create', + 'mgr.{}'.format(hostname), 'mon', + 'allow profile mgr', 'osd', 'allow *', + 'mds', 'allow *', '--out-file', + keyring]) + chownr(path, ceph_user(), ceph_user()) + + unit = 'ceph-mgr@{}'.format(hostname) + subprocess.check_call(['systemctl', 'enable', unit]) + service_restart(unit) + + +def osd_noout(enable): + """Sets or unsets 'noout' + + :param enable: bool. True to set noout, False to unset. + :returns: bool. True if output looks right. + :raises CalledProcessError: if an error occurs invoking the systemd cmd + """ + operation = { + True: 'set', + False: 'unset', + } + try: + subprocess.check_call(['ceph', '--id', 'admin', + 'osd', operation[enable], + 'noout']) + log('running ceph osd {} noout'.format(operation[enable])) + return True + except subprocess.CalledProcessError as e: + log(e) + raise + + +class OSDConfigSetError(Exception): + """Error occured applying OSD settings.""" + pass + + +def apply_osd_settings(settings): + """Applies the provided osd settings + + Apply the provided settings to all local OSD unless settings are already + present. Settings stop being applied on encountering an error. + + :param settings: dict. Dictionary of settings to apply. + :returns: bool. True if commands ran succesfully. + :raises: OSDConfigSetError + """ + current_settings = {} + base_cmd = 'ceph daemon osd.{osd_id} config --format=json' + get_cmd = base_cmd + ' get {key}' + set_cmd = base_cmd + ' set {key} {value}' + + def _get_cli_key(key): + return(key.replace(' ', '_')) + # Retrieve the current values to check keys are correct and to make this a + # noop if setting are already applied. + for osd_id in get_local_osd_ids(): + for key, value in sorted(settings.items()): + cli_key = _get_cli_key(key) + cmd = get_cmd.format(osd_id=osd_id, key=cli_key) + out = json.loads( + subprocess.check_output(cmd.split()).decode('UTF-8')) + if 'error' in out: + log("Error retrieving osd setting: {}".format(out['error']), + level=ERROR) + return False + current_settings[key] = out[cli_key] + settings_diff = { + k: v + for k, v in settings.items() + if str(v) != str(current_settings[k])} + for key, value in sorted(settings_diff.items()): + log("Setting {} to {}".format(key, value), level=DEBUG) + cmd = set_cmd.format( + osd_id=osd_id, + key=_get_cli_key(key), + value=value) + out = json.loads( + subprocess.check_output(cmd.split()).decode('UTF-8')) + if 'error' in out: + log("Error applying osd setting: {}".format(out['error']), + level=ERROR) + raise OSDConfigSetError + return True diff --git a/unit_tests/test_ceph_broker.py b/unit_tests/test_ceph_broker.py deleted file mode 100644 index c1be649..0000000 --- a/unit_tests/test_ceph_broker.py +++ /dev/null @@ -1,136 +0,0 @@ -import json -import unittest - -import mock - -import ceph_broker - - -class CephBrokerTestCase(unittest.TestCase): - def setUp(self): - super(CephBrokerTestCase, self).setUp() - - @mock.patch('ceph_broker.log') - def test_process_requests_noop(self, mock_log): - req = json.dumps({'api-version': 1, 'ops': []}) - rc = ceph_broker.process_requests(req) - self.assertEqual(json.loads(rc), {'exit-code': 0}) - - @mock.patch('ceph_broker.log') - def test_process_requests_missing_api_version(self, mock_log): - req = json.dumps({'ops': []}) - rc = ceph_broker.process_requests(req) - self.assertEqual(json.loads(rc), { - 'exit-code': 1, - 'stderr': 'Missing or invalid api version (None)'}) - - @mock.patch('ceph_broker.log') - def test_process_requests_invalid_api_version(self, mock_log): - req = json.dumps({'api-version': 2, 'ops': []}) - rc = ceph_broker.process_requests(req) - print("Return: {}".format(rc)) - self.assertEqual(json.loads(rc), - {'exit-code': 1, - 'stderr': 'Missing or invalid api version (2)'}) - - @mock.patch('ceph_broker.log') - def test_process_requests_invalid(self, mock_log): - reqs = json.dumps({'api-version': 1, 'ops': [{'op': 'invalid_op'}]}) - rc = ceph_broker.process_requests(reqs) - self.assertEqual(json.loads(rc), - {'exit-code': 1, - 'stderr': "Unknown operation 'invalid_op'"}) - - @mock.patch('ceph_broker.get_osds') - @mock.patch('ceph_broker.ReplicatedPool') - @mock.patch('ceph_broker.pool_exists') - @mock.patch('ceph_broker.log') - def test_process_requests_create_pool_w_pg_num(self, mock_log, - mock_pool_exists, - mock_replicated_pool, - mock_get_osds): - mock_get_osds.return_value = [0, 1, 2] - mock_pool_exists.return_value = False - reqs = json.dumps({'api-version': 1, - 'ops': [{ - 'op': 'create-pool', - 'name': 'foo', - 'replicas': 3, - 'pg_num': 100}]}) - rc = ceph_broker.process_requests(reqs) - mock_pool_exists.assert_called_with(service='admin', name='foo') - mock_replicated_pool.assert_called_with(service='admin', name='foo', - replicas=3, pg_num=100) - self.assertEqual(json.loads(rc), {'exit-code': 0}) - - @mock.patch('ceph_broker.get_osds') - @mock.patch('ceph_broker.ReplicatedPool') - @mock.patch('ceph_broker.pool_exists') - @mock.patch('ceph_broker.log') - def test_process_requests_create_pool_w_pg_num_capped(self, mock_log, - mock_pool_exists, - mock_replicated_pool, - mock_get_osds): - mock_get_osds.return_value = [0, 1, 2] - mock_pool_exists.return_value = False - reqs = json.dumps({'api-version': 1, - 'ops': [{ - 'op': 'create-pool', - 'name': 'foo', - 'replicas': 3, - 'pg_num': 300}]}) - rc = ceph_broker.process_requests(reqs) - mock_pool_exists.assert_called_with(service='admin', - name='foo') - mock_replicated_pool.assert_called_with(service='admin', name='foo', - replicas=3, pg_num=100) - self.assertEqual(json.loads(rc), {'exit-code': 0}) - self.assertEqual(json.loads(rc), {'exit-code': 0}) - - @mock.patch('ceph_broker.ReplicatedPool') - @mock.patch('ceph_broker.pool_exists') - @mock.patch('ceph_broker.log') - def test_process_requests_create_pool_exists(self, mock_log, - mock_pool_exists, - mock_replicated_pool): - mock_pool_exists.return_value = True - reqs = json.dumps({'api-version': 1, - 'ops': [{'op': 'create-pool', - 'name': 'foo', - 'replicas': 3}]}) - rc = ceph_broker.process_requests(reqs) - mock_pool_exists.assert_called_with(service='admin', - name='foo') - self.assertFalse(mock_replicated_pool.create.called) - self.assertEqual(json.loads(rc), {'exit-code': 0}) - - @mock.patch('ceph_broker.ReplicatedPool') - @mock.patch('ceph_broker.pool_exists') - @mock.patch('ceph_broker.log') - def test_process_requests_create_pool_rid(self, mock_log, - mock_pool_exists, - mock_replicated_pool): - mock_pool_exists.return_value = False - reqs = json.dumps({'api-version': 1, - 'request-id': '1ef5aede', - 'ops': [{ - 'op': 'create-pool', - 'name': 'foo', - 'replicas': 3}]}) - rc = ceph_broker.process_requests(reqs) - mock_pool_exists.assert_called_with(service='admin', name='foo') - mock_replicated_pool.assert_called_with(service='admin', - name='foo', - replicas=3) - self.assertEqual(json.loads(rc)['exit-code'], 0) - self.assertEqual(json.loads(rc)['request-id'], '1ef5aede') - - @mock.patch('ceph_broker.log') - def test_process_requests_invalid_api_rid(self, mock_log): - reqs = json.dumps({'api-version': 0, 'request-id': '1ef5aede', - 'ops': [{'op': 'create-pool'}]}) - rc = ceph_broker.process_requests(reqs) - self.assertEqual(json.loads(rc)['exit-code'], 1) - self.assertEqual(json.loads(rc)['stderr'], - "Missing or invalid api version (0)") - self.assertEqual(json.loads(rc)['request-id'], '1ef5aede')