openstack
/
trove
Code Issues Proposed changes

Add extended properties support for mongo cluster. 

Currently when create a mongodb cluster, mongos and configsvr
use the volume_size of replica-set node. But mongos and configvr
are not data node, they don't need volume space as large as data
node. This patch attend to help user specify the number, the volume
size and the volume type of mongos/configserver with
extended_properties[1] argument when creating mongodb. Currently,
the supported parameters are, num_configsvr, num_mongos,
configsvr_volume_size, configsvr_volume_type, mongos_volume_size
and mongos_volume_type.

[1] https://review.openstack.org/#/c/206931/

Closes-Bug: #1734907

Signed-off-by: zhanggang <zhanggang@cmss.chinamobile.com>
Change-Id: Ie48f3961b21f926f983c6713a76b0492952cf4c7
This commit is contained in:
zhanggang 2018-07-16 20:33:29 +08:00
parent 87fd1c4b0e
commit f06f65dcba
8 changed files with 130 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
releasenotes/notes/mongo-cluster-create-use-extended-perperties-ced87fde31c6c110.yaml Normal file
View File

@ -0,0 +1,8 @@
---
features:
- |
User can specify the number and volume of mongos/configserver with
extended_properties argument when creating mongodb cluster. Currently,
the supported parameters are, num_configsvr, num_mongos,
configsvr_volume_size, configsvr_volume_type, mongos_volume_size
and mongos_volume_type.

1
trove/cluster/service.py
View File

@ -170,7 +170,6 @@ class ClusterController(wsgi.Controller):
datastore, datastore_version = (
datastore_models.get_datastore_version(**datastore_args))
# TODO(saurabhs): add extended_properties to apischema
extended_properties = body['cluster'].get('extended_properties', {})
try:

23
trove/common/apischema.py
View File

@ -77,6 +77,15 @@ volume_size = {
configuration_positive_integer]
}
number_of_nodes = {
"oneOf": [
{
"type": "integer",
"minimum": 1
},
configuration_positive_integer]
}
host_string = {
"type": "string",
"minLength": 1,
@ -254,7 +263,19 @@ cluster = {
}
}
},
"locality": non_empty_string
"locality": non_empty_string,
"extended_properties": {
"type": "object",
"additionalProperties": True,
"properties": {
"num_configsvr": number_of_nodes,
"num_mongos": number_of_nodes,
"configsvr_volume_size": volume_size,
"configsvr_volume_type": non_empty_string,
"mongos_volume_size": volume_size,
"mongos_volume_type": non_empty_string
}
}
}
}
}

4
trove/common/cfg.py
View File

@ -1080,6 +1080,10 @@ mongodb_opts = [
cfg.IntOpt('num_query_routers_per_cluster', default=1,
help='The number of query routers (mongos) to create '
'per cluster.'),
cfg.IntOpt('query_routers_volume_size', default=10,
help='Default volume_size (in GB) for query routers (mongos).'),
cfg.IntOpt('config_servers_volume_size', default=10,
help='Default volume_size (in GB) for config_servers.'),
cfg.BoolOpt('cluster_support', default=True,
help='Enable clusters to be created and managed.'),
cfg.BoolOpt('cluster_secure', default=True,

48
trove/common/strategies/cluster/experimental/mongodb/api.py
View File

@ -72,8 +72,12 @@ class MongoDbCluster(models.Cluster):
raise exception.ClusterNumInstancesNotSupported(num_instances=3)
mongo_conf = CONF.get(datastore_version.manager)
num_configsvr = mongo_conf.num_config_servers_per_cluster
num_mongos = mongo_conf.num_query_routers_per_cluster
num_configsvr = int(extended_properties.get(
'num_configsvr', mongo_conf.num_config_servers_per_cluster))
num_mongos = int(extended_properties.get(
'num_mongos', mongo_conf.num_query_routers_per_cluster))
delta_instances = num_instances + num_configsvr + num_mongos
models.validate_instance_flavors(
@ -81,19 +85,33 @@ class MongoDbCluster(models.Cluster):
mongo_conf.device_path)
models.assert_homogeneous_cluster(instances)
req_volume_size = models.get_required_volume_size(
instances, mongo_conf.volume_support)
deltas = {'instances': delta_instances, 'volumes': req_volume_size}
check_quotas(context.tenant, deltas)
# Checking networks are same for the cluster
models.validate_instance_nics(context, instances)
flavor_id = instances[0]['flavor_id']
volume_size = instances[0].get('volume_size', None)
volume_type = instances[0].get('volume_type', None)
configsvr_vsize = int(extended_properties.get(
'configsvr_volume_size', mongo_conf.config_servers_volume_size))
configsvr_vtype = extended_properties.get('configsvr_volume_type',
volume_type)
mongos_vsize = int(extended_properties.get(
'mongos_volume_size', mongo_conf.query_routers_volume_size))
mongos_vtype = extended_properties.get('mongos_volume_type',
volume_type)
all_instances = (instances
+ [{'volume_size': configsvr_vsize}] * num_configsvr
+ [{'volume_size': mongos_vsize}] * num_mongos)
req_volume_size = models.get_required_volume_size(
all_instances, mongo_conf.volume_support)
deltas = {'instances': delta_instances, 'volumes': req_volume_size}
check_quotas(context.tenant, deltas)
# Checking networks are same for the cluster
models.validate_instance_nics(context, instances)
nics = instances[0].get('nics', None)
azs = [instance.get('availability_zone', None)
@ -150,12 +168,12 @@ class MongoDbCluster(models.Cluster):
datastore_version.image_id,
[], [], datastore,
datastore_version,
volume_size, None,
configsvr_vsize, None,
availability_zone=None,
nics=nics,
configuration_id=None,
cluster_config=configsvr_config,
volume_type=volume_type,
volume_type=configsvr_vtype,
locality=locality,
region_name=regions[i % num_instances]
)
@ -167,12 +185,12 @@ class MongoDbCluster(models.Cluster):
datastore_version.image_id,
[], [], datastore,
datastore_version,
volume_size, None,
mongos_vsize, None,
availability_zone=None,
nics=nics,
configuration_id=None,
cluster_config=mongos_config,
volume_type=volume_type,
volume_type=mongos_vtype,
locality=locality,
region_name=regions[i % num_instances]
)

8
trove/tests/unittests/cluster/test_cluster.py
View File

@ -81,7 +81,7 @@ class ClusterTest(trove_testtools.TestCase):
self.datastore,
self.datastore_version,
[],
None, None, None)
{}, None, None)
@patch.object(remote, 'create_nova_client')
def test_create_unequal_flavors(self, mock_client):
@ -94,7 +94,7 @@ class ClusterTest(trove_testtools.TestCase):
self.datastore,
self.datastore_version,
instances,
None, None, None)
{}, None, None)
@patch.object(remote, 'create_nova_client')
def test_create_unequal_volumes(self,
@ -110,7 +110,7 @@ class ClusterTest(trove_testtools.TestCase):
self.datastore,
self.datastore_version,
instances,
None, None, None)
{}, None, None)
@patch.object(remote, 'create_nova_client')
def test_create_storage_not_specified(self,
@ -139,7 +139,7 @@ class ClusterTest(trove_testtools.TestCase):
self.datastore,
self.datastore_version,
instances,
None, None, None)
{}, None, None)
@patch('trove.cluster.models.LOG')
def test_delete_bad_task_status(self, mock_logging):

50
trove/tests/unittests/cluster/test_mongodb_cluster.py
View File

@ -37,10 +37,14 @@ CONF = cfg.CONF
class FakeOptGroup(object):
def __init__(self, num_config_servers_per_cluster=3,
num_query_routers_per_cluster=1,
config_servers_volume_size=10,
query_routers_volume_size=10,
cluster_secure=True, volume_support=True,
device_path='/dev/vdb'):
self.num_config_servers_per_cluster = num_config_servers_per_cluster
self.num_query_routers_per_cluster = num_query_routers_per_cluster
self.config_servers_volume_size = config_servers_volume_size
self.query_routers_volume_size = query_routers_volume_size
self.cluster_secure = cluster_secure
self.volume_support = volume_support
self.device_path = device_path
@ -190,6 +194,25 @@ class MongoDBClusterTest(trove_testtools.TestCase):
self.datastore_version,
self.instances, {}, None, None)
@mock.patch.object(task_api, 'load')
@mock.patch.object(inst_models.Instance, 'create')
@mock.patch.object(models.DBCluster, 'create')
@mock.patch.object(remote, 'create_neutron_client')
@mock.patch.object(remote, 'create_nova_client')
@mock.patch.object(api, 'check_quotas')
def test_create_validate_volumes_deltas(self, mock_check_quotas, *args):
extended_properties = {
"configsvr_volume_size": 5,
"mongos_volume_size": 7}
self.cluster.create(mock.Mock(),
self.cluster_name,
self.datastore,
self.datastore_version,
self.instances,
extended_properties, None, None)
deltas = {'instances': 7, 'volumes': 25} # volumes=1*3+5*3+7*1
mock_check_quotas.assert_called_with(mock.ANY, deltas)
@mock.patch.object(task_api, 'load')
@mock.patch.object(inst_models.Instance, 'create')
@mock.patch.object(models.DBCluster, 'create')
@ -230,6 +253,33 @@ class MongoDBClusterTest(trove_testtools.TestCase):
mock_ins_create.call_args_list].count(nics)
self.assertEqual(7, nics_count)
@mock.patch.object(task_api, 'load')
@mock.patch.object(models.DBCluster, 'create')
@mock.patch.object(models, 'validate_instance_nics')
@mock.patch.object(QUOTAS, 'check_quotas')
@mock.patch.object(models, 'validate_instance_flavors')
@mock.patch.object(inst_models.Instance, 'create')
def test_create_with_extended_properties(self, mock_ins_create, *args):
extended_properties = {
"num_configsvr": 5,
"num_mongos": 7,
"configsvr_volume_size": 8,
"configsvr_volume_type": "foo_type",
"mongos_volume_size": 9,
"mongos_volume_type": "bar_type"}
self.cluster.create(mock.Mock(),
self.cluster_name,
self.datastore,
self.datastore_version,
self.instances,
extended_properties, None, None)
volume_args_list = [
(arg[8], kw['volume_type']) for arg, kw in
mock_ins_create.call_args_list
]
self.assertEqual(5, volume_args_list.count((8, "foo_type")))
self.assertEqual(7, volume_args_list.count((9, "bar_type")))
@mock.patch.object(task_api, 'load')
@mock.patch.object(inst_models.Instance, 'create')
@mock.patch.object(models.DBCluster, 'create')

18
trove/tests/unittests/cluster/test_vertica_cluster.py
View File

@ -80,7 +80,7 @@ class ClusterTest(trove_testtools.TestCase):
self.cluster_name,
self.datastore,
self.datastore_version,
[], None, None, None)
[], {}, None, None)
@patch.object(DBCluster, 'create')
@patch.object(inst_models.DBInstance, 'find_all')
@ -95,7 +95,7 @@ class ClusterTest(trove_testtools.TestCase):
self.datastore,
self.datastore_version,
instances,
None, None, None)
{}, None, None)
@patch.object(DBCluster, 'create')
@patch.object(inst_models.DBInstance, 'find_all')
@ -117,7 +117,7 @@ class ClusterTest(trove_testtools.TestCase):
self.datastore,
self.datastore_version,
instances,
None, None, None)
{}, None, None)
@patch.object(DBCluster, 'create')
@patch.object(inst_models.DBInstance, 'find_all')
@ -135,7 +135,7 @@ class ClusterTest(trove_testtools.TestCase):
self.datastore,
self.datastore_version,
instances,
None, None, None)
{}, None, None)
@patch.object(DBCluster, 'create')
@patch.object(inst_models.DBInstance, 'find_all')
@ -159,7 +159,7 @@ class ClusterTest(trove_testtools.TestCase):
self.datastore,
self.datastore_version,
instances,
None, None, None)
{}, None, None)
@patch.object(DBCluster, 'create')
@patch.object(inst_models.DBInstance, 'find_all')
@ -195,7 +195,7 @@ class ClusterTest(trove_testtools.TestCase):
self.datastore,
self.datastore_version,
instances,
None, None, None)
{}, None, None)
@patch.object(DBCluster, 'create')
@patch.object(inst_models.DBInstance, 'find_all')
@ -213,7 +213,7 @@ class ClusterTest(trove_testtools.TestCase):
self.datastore,
self.datastore_version,
instances,
None, None, None)
{}, None, None)
@patch.object(inst_models.DBInstance, 'find_all')
@patch.object(inst_models.Instance, 'create')
@ -231,7 +231,7 @@ class ClusterTest(trove_testtools.TestCase):
self.datastore,
self.datastore_version,
instances,
None, None, None)
{}, None, None)
mock_task_api.return_value.create_cluster.assert_called_with(
mock_db_create.return_value.id)
self.assertEqual(3, mock_ins_create.call_count)
@ -270,7 +270,7 @@ class ClusterTest(trove_testtools.TestCase):
self.datastore,
self.datastore_version,
instances,
None, None, None)
{}, None, None)
mock_task_api.return_value.create_cluster.assert_called_with(
mock_db_create.return_value.id)
self.assertEqual(3, mock_ins_create.call_count)