Merge "Add support for erasure coding"
This commit is contained in:
commit
0f51adbba0
|
@ -9,3 +9,4 @@ tags
|
|||
.unit-state.db
|
||||
func-results.json
|
||||
.stestr/
|
||||
**/__pycache__
|
||||
|
|
98
config.yaml
98
config.yaml
|
@ -121,6 +121,104 @@ options:
|
|||
that once a pool has been created, changes to this setting will be
|
||||
ignored. Setting this value to -1, enables the number of placement
|
||||
groups to be calculated based on the Ceph placement group calculator.
|
||||
pool-type:
|
||||
type: string
|
||||
default: replicated
|
||||
description: |
|
||||
Ceph pool type to use for storage - valid values include ‘replicated’
|
||||
and ‘erasure-coded’.
|
||||
ec-profile-name:
|
||||
type: string
|
||||
default:
|
||||
description: |
|
||||
Name for the EC profile to be created for the EC pools. If not defined
|
||||
a profile name will be generated based on the name of the pool used by
|
||||
the application.
|
||||
ec-rbd-metadata-pool:
|
||||
type: string
|
||||
default:
|
||||
description: |
|
||||
Name of the metadata pool to be created (for RBD use-cases). If not
|
||||
defined a metadata pool name will be generated based on the name of
|
||||
the data pool used by the application. The metadata pool is always
|
||||
replicated, not erasure coded.
|
||||
ec-profile-k:
|
||||
type: int
|
||||
default: 1
|
||||
description: |
|
||||
Number of data chunks that will be used for EC data pool. K+M factors
|
||||
should never be greater than the number of available zones (or hosts)
|
||||
for balancing.
|
||||
ec-profile-m:
|
||||
type: int
|
||||
default: 2
|
||||
description: |
|
||||
Number of coding chunks that will be used for EC data pool. K+M factors
|
||||
should never be greater than the number of available zones (or hosts)
|
||||
for balancing.
|
||||
ec-profile-locality:
|
||||
type: int
|
||||
default:
|
||||
description: |
|
||||
(lrc plugin - l) Group the coding and data chunks into sets of size l.
|
||||
For instance, for k=4 and m=2, when l=3 two groups of three are created.
|
||||
Each set can be recovered without reading chunks from another set. Note
|
||||
that using the lrc plugin does incur more raw storage usage than isa or
|
||||
jerasure in order to reduce the cost of recovery operations.
|
||||
ec-profile-crush-locality:
|
||||
type: string
|
||||
default:
|
||||
description: |
|
||||
(lrc plugin) The type of the crush bucket in which each set of chunks
|
||||
defined by l will be stored. For instance, if it is set to rack, each
|
||||
group of l chunks will be placed in a different rack. It is used to
|
||||
create a CRUSH rule step such as step choose rack. If it is not set,
|
||||
no such grouping is done.
|
||||
ec-profile-durability-estimator:
|
||||
type: int
|
||||
default:
|
||||
description: |
|
||||
(shec plugin - c) The number of parity chunks each of which includes
|
||||
each data chunk in its calculation range. The number is used as a
|
||||
durability estimator. For instance, if c=2, 2 OSDs can be down
|
||||
without losing data.
|
||||
ec-profile-helper-chunks:
|
||||
type: int
|
||||
default:
|
||||
description: |
|
||||
(clay plugin - d) Number of OSDs requested to send data during
|
||||
recovery of a single chunk. d needs to be chosen such that
|
||||
k+1 <= d <= k+m-1. Larger the d, the better the savings.
|
||||
ec-profile-scalar-mds:
|
||||
type: string
|
||||
default:
|
||||
description: |
|
||||
(clay plugin) specifies the plugin that is used as a building
|
||||
block in the layered construction. It can be one of jerasure,
|
||||
isa, shec (defaults to jerasure).
|
||||
ec-profile-plugin:
|
||||
type: string
|
||||
default: jerasure
|
||||
description: |
|
||||
EC plugin to use for this applications pool. The following list of
|
||||
plugins acceptable - jerasure, lrc, isa, shec, clay.
|
||||
ec-profile-technique:
|
||||
type: string
|
||||
default:
|
||||
description: |
|
||||
EC profile technique used for this applications pool - will be
|
||||
validated based on the plugin configured via ec-profile-plugin.
|
||||
Supported techniques are ‘reed_sol_van’, ‘reed_sol_r6_op’,
|
||||
‘cauchy_orig’, ‘cauchy_good’, ‘liber8tion’ for jerasure,
|
||||
‘reed_sol_van’, ‘cauchy’ for isa and ‘single’, ‘multiple’
|
||||
for shec.
|
||||
ec-profile-device-class:
|
||||
type: string
|
||||
default:
|
||||
description: |
|
||||
Device class from CRUSH map to use for placement groups for
|
||||
erasure profile - valid values: ssd, hdd or nvme (or leave
|
||||
unset to not use a device class).
|
||||
# Keystone integration
|
||||
operator-roles:
|
||||
type: string
|
||||
|
|
|
@ -18,6 +18,7 @@ import subprocess
|
|||
|
||||
from charmhelpers.core.hookenv import (
|
||||
config,
|
||||
service_name,
|
||||
)
|
||||
|
||||
from charmhelpers.core.host import (
|
||||
|
@ -111,18 +112,61 @@ def get_create_rgw_pools_rq(prefix=None):
|
|||
replicas = config('ceph-osd-replication-count')
|
||||
|
||||
prefix = prefix or 'default'
|
||||
|
||||
# Buckets likely to contain the most data and therefore
|
||||
# requiring the most PGs
|
||||
heavy = [
|
||||
'.rgw.buckets.data'
|
||||
]
|
||||
bucket_weight = config('rgw-buckets-pool-weight')
|
||||
for pool in heavy:
|
||||
pool = "{prefix}{pool}".format(prefix=prefix, pool=pool)
|
||||
rq.add_op_create_pool(name=pool, replica_count=replicas,
|
||||
weight=bucket_weight, group='objects',
|
||||
app_name=CEPH_POOL_APP_NAME)
|
||||
|
||||
if config('pool-type') == 'erasure-coded':
|
||||
# General EC plugin config
|
||||
plugin = config('ec-profile-plugin')
|
||||
technique = config('ec-profile-technique')
|
||||
device_class = config('ec-profile-device-class')
|
||||
bdm_k = config('ec-profile-k')
|
||||
bdm_m = config('ec-profile-m')
|
||||
# LRC plugin config
|
||||
bdm_l = config('ec-profile-locality')
|
||||
crush_locality = config('ec-profile-crush-locality')
|
||||
# SHEC plugin config
|
||||
bdm_c = config('ec-profile-durability-estimator')
|
||||
# CLAY plugin config
|
||||
bdm_d = config('ec-profile-helper-chunks')
|
||||
scalar_mds = config('ec-profile-scalar-mds')
|
||||
# Profile name
|
||||
service = service_name()
|
||||
profile_name = (
|
||||
config('ec-profile-name') or "{}-profile".format(service)
|
||||
)
|
||||
rq.add_op_create_erasure_profile(
|
||||
name=profile_name,
|
||||
k=bdm_k, m=bdm_m,
|
||||
lrc_locality=bdm_l,
|
||||
lrc_crush_locality=crush_locality,
|
||||
shec_durability_estimator=bdm_c,
|
||||
clay_helper_chunks=bdm_d,
|
||||
clay_scalar_mds=scalar_mds,
|
||||
device_class=device_class,
|
||||
erasure_type=plugin,
|
||||
erasure_technique=technique
|
||||
)
|
||||
|
||||
for pool in heavy:
|
||||
pool = "{prefix}{pool}".format(prefix=prefix, pool=pool)
|
||||
rq.add_op_create_erasure_pool(
|
||||
name=pool,
|
||||
erasure_profile=profile_name,
|
||||
weight=bucket_weight,
|
||||
group="objects",
|
||||
app_name=CEPH_POOL_APP_NAME
|
||||
)
|
||||
else:
|
||||
for pool in heavy:
|
||||
pool = "{prefix}{pool}".format(prefix=prefix, pool=pool)
|
||||
rq.add_op_create_pool(name=pool, replica_count=replicas,
|
||||
weight=bucket_weight, group='objects',
|
||||
app_name=CEPH_POOL_APP_NAME)
|
||||
|
||||
# NOTE: we want these pools to have a smaller pg_num/pgp_num than the
|
||||
# others since they are not expected to contain as much data
|
||||
|
|
|
@ -29,6 +29,8 @@ from subprocess import check_call, CalledProcessError
|
|||
|
||||
import six
|
||||
|
||||
import charmhelpers.contrib.storage.linux.ceph as ch_ceph
|
||||
|
||||
from charmhelpers.contrib.openstack.audits.openstack_security_guide import (
|
||||
_config_ini as config_ini
|
||||
)
|
||||
|
@ -56,6 +58,7 @@ from charmhelpers.core.hookenv import (
|
|||
status_set,
|
||||
network_get_primary_address,
|
||||
WARNING,
|
||||
service_name,
|
||||
)
|
||||
|
||||
from charmhelpers.core.sysctl import create as sysctl_create
|
||||
|
@ -808,6 +811,12 @@ class CephContext(OSContextGenerator):
|
|||
|
||||
ctxt['mon_hosts'] = ' '.join(sorted(mon_hosts))
|
||||
|
||||
if config('pool-type') and config('pool-type') == 'erasure-coded':
|
||||
base_pool_name = config('rbd-pool') or config('rbd-pool-name')
|
||||
if not base_pool_name:
|
||||
base_pool_name = service_name()
|
||||
ctxt['rbd_default_data_pool'] = base_pool_name
|
||||
|
||||
if not os.path.isdir('/etc/ceph'):
|
||||
os.mkdir('/etc/ceph')
|
||||
|
||||
|
@ -3175,3 +3184,78 @@ class SRIOVContext(OSContextGenerator):
|
|||
:rtype: Dict[str,int]
|
||||
"""
|
||||
return self._map
|
||||
|
||||
|
||||
class CephBlueStoreCompressionContext(OSContextGenerator):
|
||||
"""Ceph BlueStore compression options."""
|
||||
|
||||
# Tuple with Tuples that map configuration option name to CephBrokerRq op
|
||||
# property name
|
||||
options = (
|
||||
('bluestore-compression-algorithm',
|
||||
'compression-algorithm'),
|
||||
('bluestore-compression-mode',
|
||||
'compression-mode'),
|
||||
('bluestore-compression-required-ratio',
|
||||
'compression-required-ratio'),
|
||||
('bluestore-compression-min-blob-size',
|
||||
'compression-min-blob-size'),
|
||||
('bluestore-compression-min-blob-size-hdd',
|
||||
'compression-min-blob-size-hdd'),
|
||||
('bluestore-compression-min-blob-size-ssd',
|
||||
'compression-min-blob-size-ssd'),
|
||||
('bluestore-compression-max-blob-size',
|
||||
'compression-max-blob-size'),
|
||||
('bluestore-compression-max-blob-size-hdd',
|
||||
'compression-max-blob-size-hdd'),
|
||||
('bluestore-compression-max-blob-size-ssd',
|
||||
'compression-max-blob-size-ssd'),
|
||||
)
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize context by loading values from charm config.
|
||||
|
||||
We keep two maps, one suitable for use with CephBrokerRq's and one
|
||||
suitable for template generation.
|
||||
"""
|
||||
charm_config = config()
|
||||
|
||||
# CephBrokerRq op map
|
||||
self.op = {}
|
||||
# Context exposed for template generation
|
||||
self.ctxt = {}
|
||||
for config_key, op_key in self.options:
|
||||
value = charm_config.get(config_key)
|
||||
self.ctxt.update({config_key.replace('-', '_'): value})
|
||||
self.op.update({op_key: value})
|
||||
|
||||
def __call__(self):
|
||||
"""Get context.
|
||||
|
||||
:returns: Context
|
||||
:rtype: Dict[str,any]
|
||||
"""
|
||||
return self.ctxt
|
||||
|
||||
def get_op(self):
|
||||
"""Get values for use in CephBrokerRq op.
|
||||
|
||||
:returns: Context values with CephBrokerRq op property name as key.
|
||||
:rtype: Dict[str,any]
|
||||
"""
|
||||
return self.op
|
||||
|
||||
def validate(self):
|
||||
"""Validate options.
|
||||
|
||||
:raises: AssertionError
|
||||
"""
|
||||
# We slip in a dummy name on class instantiation to allow validation of
|
||||
# the other options. It will not affect further use.
|
||||
#
|
||||
# NOTE: once we retire Python 3.5 we can fold this into a in-line
|
||||
# dictionary comprehension in the call to the initializer.
|
||||
dummy_op = {'name': 'dummy-name'}
|
||||
dummy_op.update(self.op)
|
||||
pool = ch_ceph.BasePool('dummy-service', op=dummy_op)
|
||||
pool.validate()
|
||||
|
|
|
@ -22,3 +22,7 @@ rbd default features = {{ rbd_features }}
|
|||
{{ key }} = {{ value }}
|
||||
{% endfor -%}
|
||||
{%- endif %}
|
||||
|
||||
{% if rbd_default_data_pool -%}
|
||||
rbd default data pool = {{ rbd_default_data_pool }}
|
||||
{% endif %}
|
||||
|
|
|
@ -0,0 +1,28 @@
|
|||
{# section header omitted as options can belong to multiple sections #}
|
||||
{% if bluestore_compression_algorithm -%}
|
||||
bluestore compression algorithm = {{ bluestore_compression_algorithm }}
|
||||
{% endif -%}
|
||||
{% if bluestore_compression_mode -%}
|
||||
bluestore compression mode = {{ bluestore_compression_mode }}
|
||||
{% endif -%}
|
||||
{% if bluestore_compression_required_ratio -%}
|
||||
bluestore compression required ratio = {{ bluestore_compression_required_ratio }}
|
||||
{% endif -%}
|
||||
{% if bluestore_compression_min_blob_size -%}
|
||||
bluestore compression min blob size = {{ bluestore_compression_min_blob_size }}
|
||||
{% endif -%}
|
||||
{% if bluestore_compression_min_blob_size_hdd -%}
|
||||
bluestore compression min blob size hdd = {{ bluestore_compression_min_blob_size_hdd }}
|
||||
{% endif -%}
|
||||
{% if bluestore_compression_min_blob_size_ssd -%}
|
||||
bluestore compression min blob size ssd = {{ bluestore_compression_min_blob_size_ssd }}
|
||||
{% endif -%}
|
||||
{% if bluestore_compression_max_blob_size -%}
|
||||
bluestore compression max blob size = {{ bluestore_compression_max_blob_size }}
|
||||
{% endif -%}
|
||||
{% if bluestore_compression_max_blob_size_hdd -%}
|
||||
bluestore compression max blob size hdd = {{ bluestore_compression_max_blob_size_hdd }}
|
||||
{% endif -%}
|
||||
{% if bluestore_compression_max_blob_size_ssd -%}
|
||||
bluestore compression max blob size ssd = {{ bluestore_compression_max_blob_size_ssd }}
|
||||
{% endif -%}
|
File diff suppressed because it is too large
Load Diff
|
@ -155,25 +155,47 @@ def handle_create_erasure_profile(request, service):
|
|||
:param service: The ceph client to run the command under.
|
||||
:returns: dict. exit-code and reason if not 0
|
||||
"""
|
||||
# "local" | "shec" or it defaults to "jerasure"
|
||||
# "isa" | "lrc" | "shec" | "clay" or it defaults to "jerasure"
|
||||
erasure_type = request.get('erasure-type')
|
||||
# "host" | "rack" or it defaults to "host" # Any valid Ceph bucket
|
||||
# dependent on erasure coding type
|
||||
erasure_technique = request.get('erasure-technique')
|
||||
# "host" | "rack" | ...
|
||||
failure_domain = request.get('failure-domain')
|
||||
name = request.get('name')
|
||||
# Binary Distribution Matrix (BDM) parameters
|
||||
bdm_k = request.get('k')
|
||||
bdm_m = request.get('m')
|
||||
# LRC parameters
|
||||
bdm_l = request.get('l')
|
||||
crush_locality = request.get('crush-locality')
|
||||
# SHEC parameters
|
||||
bdm_c = request.get('c')
|
||||
# CLAY parameters
|
||||
bdm_d = request.get('d')
|
||||
scalar_mds = request.get('scalar-mds')
|
||||
# Device Class
|
||||
device_class = request.get('device-class')
|
||||
|
||||
if failure_domain not in CEPH_BUCKET_TYPES:
|
||||
if failure_domain and failure_domain not in CEPH_BUCKET_TYPES:
|
||||
msg = "failure-domain must be one of {}".format(CEPH_BUCKET_TYPES)
|
||||
log(msg, level=ERROR)
|
||||
return {'exit-code': 1, 'stderr': msg}
|
||||
|
||||
create_erasure_profile(service=service, erasure_plugin_name=erasure_type,
|
||||
profile_name=name, failure_domain=failure_domain,
|
||||
data_chunks=bdm_k, coding_chunks=bdm_m,
|
||||
locality=bdm_l)
|
||||
create_erasure_profile(service=service,
|
||||
erasure_plugin_name=erasure_type,
|
||||
profile_name=name,
|
||||
failure_domain=failure_domain,
|
||||
data_chunks=bdm_k,
|
||||
coding_chunks=bdm_m,
|
||||
locality=bdm_l,
|
||||
durability_estimator=bdm_d,
|
||||
helper_chunks=bdm_c,
|
||||
scalar_mds=scalar_mds,
|
||||
crush_locality=crush_locality,
|
||||
device_class=device_class,
|
||||
erasure_plugin_technique=erasure_technique)
|
||||
|
||||
return {'exit-code': 0}
|
||||
|
||||
|
||||
def handle_add_permissions_to_key(request, service):
|
||||
|
@ -387,6 +409,7 @@ def handle_erasure_pool(request, service):
|
|||
max_objects = request.get('max-objects')
|
||||
weight = request.get('weight')
|
||||
group_name = request.get('group')
|
||||
allow_ec_overwrites = request.get('allow-ec-overwrites')
|
||||
|
||||
if erasure_profile is None:
|
||||
erasure_profile = "default-canonical"
|
||||
|
@ -416,7 +439,9 @@ def handle_erasure_pool(request, service):
|
|||
|
||||
pool = ErasurePool(service=service, name=pool_name,
|
||||
erasure_code_profile=erasure_profile,
|
||||
percent_data=weight, app_name=app_name)
|
||||
percent_data=weight,
|
||||
app_name=app_name,
|
||||
allow_ec_overwrites=allow_ec_overwrites)
|
||||
# Ok make the erasure pool
|
||||
if not pool_exists(service=service, name=pool_name):
|
||||
log("Creating pool '{}' (erasure_profile={})"
|
||||
|
|
|
@ -24,6 +24,7 @@ TO_PATCH = [
|
|||
'os',
|
||||
'subprocess',
|
||||
'mkdir',
|
||||
'service_name',
|
||||
]
|
||||
|
||||
|
||||
|
@ -31,6 +32,7 @@ class CephRadosGWCephTests(CharmTestCase):
|
|||
def setUp(self):
|
||||
super(CephRadosGWCephTests, self).setUp(ceph, TO_PATCH)
|
||||
self.config.side_effect = self.test_config.get
|
||||
self.service_name.return_value = 'ceph-radosgw'
|
||||
|
||||
def test_import_radosgw_key(self):
|
||||
self.os.path.exists.return_value = False
|
||||
|
@ -138,6 +140,82 @@ class CephRadosGWCephTests(CharmTestCase):
|
|||
name='objects',
|
||||
permission='rwx')
|
||||
|
||||
@patch('charmhelpers.contrib.storage.linux.ceph.CephBrokerRq'
|
||||
'.add_op_create_erasure_profile')
|
||||
@patch('charmhelpers.contrib.storage.linux.ceph.CephBrokerRq'
|
||||
'.add_op_create_erasure_pool')
|
||||
@patch('charmhelpers.contrib.storage.linux.ceph.CephBrokerRq'
|
||||
'.add_op_request_access_to_group')
|
||||
@patch('charmhelpers.contrib.storage.linux.ceph.CephBrokerRq'
|
||||
'.add_op_create_pool')
|
||||
def test_create_rgw_pools_rq_no_prefix_ec(self, mock_broker,
|
||||
mock_request_access,
|
||||
mock_request_create_ec_pool,
|
||||
mock_request_create_ec_profile):
|
||||
self.test_config.set('rgw-lightweight-pool-pg-num', -1)
|
||||
self.test_config.set('ceph-osd-replication-count', 3)
|
||||
self.test_config.set('rgw-buckets-pool-weight', 19)
|
||||
self.test_config.set('restrict-ceph-pools', True)
|
||||
self.test_config.set('pool-type', 'erasure-coded')
|
||||
self.test_config.set('ec-profile-k', 3)
|
||||
self.test_config.set('ec-profile-m', 9)
|
||||
self.test_config.set('ec-profile-technique', 'cauchy_good')
|
||||
ceph.get_create_rgw_pools_rq(prefix=None)
|
||||
mock_request_create_ec_profile.assert_called_once_with(
|
||||
name='ceph-radosgw-profile',
|
||||
k=3, m=9,
|
||||
lrc_locality=None,
|
||||
lrc_crush_locality=None,
|
||||
shec_durability_estimator=None,
|
||||
clay_helper_chunks=None,
|
||||
clay_scalar_mds=None,
|
||||
device_class=None,
|
||||
erasure_type='jerasure',
|
||||
erasure_technique='cauchy_good'
|
||||
)
|
||||
mock_request_create_ec_pool.assert_has_calls([
|
||||
call(name='default.rgw.buckets.data',
|
||||
erasure_profile='ceph-radosgw-profile',
|
||||
weight=19,
|
||||
group="objects",
|
||||
app_name='rgw')
|
||||
])
|
||||
mock_broker.assert_has_calls([
|
||||
call(weight=0.10, replica_count=3, name='default.rgw.control',
|
||||
group='objects', app_name='rgw'),
|
||||
call(weight=0.10, replica_count=3, name='default.rgw.data.root',
|
||||
group='objects', app_name='rgw'),
|
||||
call(weight=0.10, replica_count=3, name='default.rgw.gc',
|
||||
group='objects', app_name='rgw'),
|
||||
call(weight=0.10, replica_count=3, name='default.rgw.log',
|
||||
group='objects', app_name='rgw'),
|
||||
call(weight=0.10, replica_count=3, name='default.rgw.intent-log',
|
||||
group='objects', app_name='rgw'),
|
||||
call(weight=0.10, replica_count=3, name='default.rgw.meta',
|
||||
group='objects', app_name='rgw'),
|
||||
call(weight=0.10, replica_count=3, name='default.rgw.usage',
|
||||
group='objects', app_name='rgw'),
|
||||
call(weight=0.10, replica_count=3, name='default.rgw.users.keys',
|
||||
group='objects', app_name='rgw'),
|
||||
call(weight=0.10, replica_count=3, name='default.rgw.users.email',
|
||||
group='objects', app_name='rgw'),
|
||||
call(weight=0.10, replica_count=3, name='default.rgw.users.swift',
|
||||
group='objects', app_name='rgw'),
|
||||
call(weight=0.10, replica_count=3, name='default.rgw.users.uid',
|
||||
group='objects', app_name='rgw'),
|
||||
call(weight=1.00, replica_count=3,
|
||||
name='default.rgw.buckets.extra',
|
||||
group='objects', app_name='rgw'),
|
||||
call(weight=3.00, replica_count=3,
|
||||
name='default.rgw.buckets.index',
|
||||
group='objects', app_name='rgw'),
|
||||
call(weight=0.10, replica_count=3, name='.rgw.root',
|
||||
group='objects', app_name='rgw')],
|
||||
)
|
||||
mock_request_access.assert_called_with(key_name='radosgw.gateway',
|
||||
name='objects',
|
||||
permission='rwx')
|
||||
|
||||
@patch.object(utils.apt_pkg, 'version_compare', lambda *args: -1)
|
||||
@patch.object(utils, 'lsb_release',
|
||||
lambda: {'DISTRIB_CODENAME': 'trusty'})
|
||||
|
|
Loading…
Reference in New Issue