From f06f65dcba13894183bde048577815fcb3bfc57a Mon Sep 17 00:00:00 2001 From: zhanggang Date: Mon, 16 Jul 2018 20:33:29 +0800 Subject: [PATCH] Add extended properties support for mongo cluster. Currently when create a mongodb cluster, mongos and configsvr use the volume_size of replica-set node. But mongos and configvr are not data node, they don't need volume space as large as data node. This patch attend to help user specify the number, the volume size and the volume type of mongos/configserver with extended_properties[1] argument when creating mongodb. Currently, the supported parameters are, num_configsvr, num_mongos, configsvr_volume_size, configsvr_volume_type, mongos_volume_size and mongos_volume_type. [1] https://review.openstack.org/#/c/206931/ Closes-Bug: #1734907 Signed-off-by: zhanggang Change-Id: Ie48f3961b21f926f983c6713a76b0492952cf4c7 --- ...-extended-perperties-ced87fde31c6c110.yaml | 8 +++ trove/cluster/service.py | 1 - trove/common/apischema.py | 23 ++++++++- trove/common/cfg.py | 4 ++ .../cluster/experimental/mongodb/api.py | 48 ++++++++++++------ trove/tests/unittests/cluster/test_cluster.py | 8 +-- .../unittests/cluster/test_mongodb_cluster.py | 50 +++++++++++++++++++ .../unittests/cluster/test_vertica_cluster.py | 18 +++---- 8 files changed, 130 insertions(+), 30 deletions(-) create mode 100644 releasenotes/notes/mongo-cluster-create-use-extended-perperties-ced87fde31c6c110.yaml diff --git a/releasenotes/notes/mongo-cluster-create-use-extended-perperties-ced87fde31c6c110.yaml b/releasenotes/notes/mongo-cluster-create-use-extended-perperties-ced87fde31c6c110.yaml new file mode 100644 index 0000000000..0c734e82c7 --- /dev/null +++ b/releasenotes/notes/mongo-cluster-create-use-extended-perperties-ced87fde31c6c110.yaml @@ -0,0 +1,8 @@ +--- +features: + - | + User can specify the number and volume of mongos/configserver with + extended_properties argument when creating mongodb cluster. Currently, + the supported parameters are, num_configsvr, num_mongos, + configsvr_volume_size, configsvr_volume_type, mongos_volume_size + and mongos_volume_type. diff --git a/trove/cluster/service.py b/trove/cluster/service.py index 63bf12d659..8e7ec3577f 100644 --- a/trove/cluster/service.py +++ b/trove/cluster/service.py @@ -170,7 +170,6 @@ class ClusterController(wsgi.Controller): datastore, datastore_version = ( datastore_models.get_datastore_version(**datastore_args)) - # TODO(saurabhs): add extended_properties to apischema extended_properties = body['cluster'].get('extended_properties', {}) try: diff --git a/trove/common/apischema.py b/trove/common/apischema.py index cbb7d632f6..fed3bf231e 100644 --- a/trove/common/apischema.py +++ b/trove/common/apischema.py @@ -77,6 +77,15 @@ volume_size = { configuration_positive_integer] } +number_of_nodes = { + "oneOf": [ + { + "type": "integer", + "minimum": 1 + }, + configuration_positive_integer] +} + host_string = { "type": "string", "minLength": 1, @@ -254,7 +263,19 @@ cluster = { } } }, - "locality": non_empty_string + "locality": non_empty_string, + "extended_properties": { + "type": "object", + "additionalProperties": True, + "properties": { + "num_configsvr": number_of_nodes, + "num_mongos": number_of_nodes, + "configsvr_volume_size": volume_size, + "configsvr_volume_type": non_empty_string, + "mongos_volume_size": volume_size, + "mongos_volume_type": non_empty_string + } + } } } } diff --git a/trove/common/cfg.py b/trove/common/cfg.py index c1027eaa15..7d8034ab4f 100644 --- a/trove/common/cfg.py +++ b/trove/common/cfg.py @@ -1080,6 +1080,10 @@ mongodb_opts = [ cfg.IntOpt('num_query_routers_per_cluster', default=1, help='The number of query routers (mongos) to create ' 'per cluster.'), + cfg.IntOpt('query_routers_volume_size', default=10, + help='Default volume_size (in GB) for query routers (mongos).'), + cfg.IntOpt('config_servers_volume_size', default=10, + help='Default volume_size (in GB) for config_servers.'), cfg.BoolOpt('cluster_support', default=True, help='Enable clusters to be created and managed.'), cfg.BoolOpt('cluster_secure', default=True, diff --git a/trove/common/strategies/cluster/experimental/mongodb/api.py b/trove/common/strategies/cluster/experimental/mongodb/api.py index 52683c2a8b..84ff04d0aa 100644 --- a/trove/common/strategies/cluster/experimental/mongodb/api.py +++ b/trove/common/strategies/cluster/experimental/mongodb/api.py @@ -72,8 +72,12 @@ class MongoDbCluster(models.Cluster): raise exception.ClusterNumInstancesNotSupported(num_instances=3) mongo_conf = CONF.get(datastore_version.manager) - num_configsvr = mongo_conf.num_config_servers_per_cluster - num_mongos = mongo_conf.num_query_routers_per_cluster + + num_configsvr = int(extended_properties.get( + 'num_configsvr', mongo_conf.num_config_servers_per_cluster)) + num_mongos = int(extended_properties.get( + 'num_mongos', mongo_conf.num_query_routers_per_cluster)) + delta_instances = num_instances + num_configsvr + num_mongos models.validate_instance_flavors( @@ -81,19 +85,33 @@ class MongoDbCluster(models.Cluster): mongo_conf.device_path) models.assert_homogeneous_cluster(instances) - req_volume_size = models.get_required_volume_size( - instances, mongo_conf.volume_support) - - deltas = {'instances': delta_instances, 'volumes': req_volume_size} - - check_quotas(context.tenant, deltas) - # Checking networks are same for the cluster - models.validate_instance_nics(context, instances) - flavor_id = instances[0]['flavor_id'] + volume_size = instances[0].get('volume_size', None) volume_type = instances[0].get('volume_type', None) + configsvr_vsize = int(extended_properties.get( + 'configsvr_volume_size', mongo_conf.config_servers_volume_size)) + configsvr_vtype = extended_properties.get('configsvr_volume_type', + volume_type) + + mongos_vsize = int(extended_properties.get( + 'mongos_volume_size', mongo_conf.query_routers_volume_size)) + mongos_vtype = extended_properties.get('mongos_volume_type', + volume_type) + + all_instances = (instances + + [{'volume_size': configsvr_vsize}] * num_configsvr + + [{'volume_size': mongos_vsize}] * num_mongos) + req_volume_size = models.get_required_volume_size( + all_instances, mongo_conf.volume_support) + + deltas = {'instances': delta_instances, 'volumes': req_volume_size} + check_quotas(context.tenant, deltas) + + # Checking networks are same for the cluster + models.validate_instance_nics(context, instances) + nics = instances[0].get('nics', None) azs = [instance.get('availability_zone', None) @@ -150,12 +168,12 @@ class MongoDbCluster(models.Cluster): datastore_version.image_id, [], [], datastore, datastore_version, - volume_size, None, + configsvr_vsize, None, availability_zone=None, nics=nics, configuration_id=None, cluster_config=configsvr_config, - volume_type=volume_type, + volume_type=configsvr_vtype, locality=locality, region_name=regions[i % num_instances] ) @@ -167,12 +185,12 @@ class MongoDbCluster(models.Cluster): datastore_version.image_id, [], [], datastore, datastore_version, - volume_size, None, + mongos_vsize, None, availability_zone=None, nics=nics, configuration_id=None, cluster_config=mongos_config, - volume_type=volume_type, + volume_type=mongos_vtype, locality=locality, region_name=regions[i % num_instances] ) diff --git a/trove/tests/unittests/cluster/test_cluster.py b/trove/tests/unittests/cluster/test_cluster.py index 9f391af514..6403c24797 100644 --- a/trove/tests/unittests/cluster/test_cluster.py +++ b/trove/tests/unittests/cluster/test_cluster.py @@ -81,7 +81,7 @@ class ClusterTest(trove_testtools.TestCase): self.datastore, self.datastore_version, [], - None, None, None) + {}, None, None) @patch.object(remote, 'create_nova_client') def test_create_unequal_flavors(self, mock_client): @@ -94,7 +94,7 @@ class ClusterTest(trove_testtools.TestCase): self.datastore, self.datastore_version, instances, - None, None, None) + {}, None, None) @patch.object(remote, 'create_nova_client') def test_create_unequal_volumes(self, @@ -110,7 +110,7 @@ class ClusterTest(trove_testtools.TestCase): self.datastore, self.datastore_version, instances, - None, None, None) + {}, None, None) @patch.object(remote, 'create_nova_client') def test_create_storage_not_specified(self, @@ -139,7 +139,7 @@ class ClusterTest(trove_testtools.TestCase): self.datastore, self.datastore_version, instances, - None, None, None) + {}, None, None) @patch('trove.cluster.models.LOG') def test_delete_bad_task_status(self, mock_logging): diff --git a/trove/tests/unittests/cluster/test_mongodb_cluster.py b/trove/tests/unittests/cluster/test_mongodb_cluster.py index b6a372aafb..a177a0f809 100644 --- a/trove/tests/unittests/cluster/test_mongodb_cluster.py +++ b/trove/tests/unittests/cluster/test_mongodb_cluster.py @@ -37,10 +37,14 @@ CONF = cfg.CONF class FakeOptGroup(object): def __init__(self, num_config_servers_per_cluster=3, num_query_routers_per_cluster=1, + config_servers_volume_size=10, + query_routers_volume_size=10, cluster_secure=True, volume_support=True, device_path='/dev/vdb'): self.num_config_servers_per_cluster = num_config_servers_per_cluster self.num_query_routers_per_cluster = num_query_routers_per_cluster + self.config_servers_volume_size = config_servers_volume_size + self.query_routers_volume_size = query_routers_volume_size self.cluster_secure = cluster_secure self.volume_support = volume_support self.device_path = device_path @@ -190,6 +194,25 @@ class MongoDBClusterTest(trove_testtools.TestCase): self.datastore_version, self.instances, {}, None, None) + @mock.patch.object(task_api, 'load') + @mock.patch.object(inst_models.Instance, 'create') + @mock.patch.object(models.DBCluster, 'create') + @mock.patch.object(remote, 'create_neutron_client') + @mock.patch.object(remote, 'create_nova_client') + @mock.patch.object(api, 'check_quotas') + def test_create_validate_volumes_deltas(self, mock_check_quotas, *args): + extended_properties = { + "configsvr_volume_size": 5, + "mongos_volume_size": 7} + self.cluster.create(mock.Mock(), + self.cluster_name, + self.datastore, + self.datastore_version, + self.instances, + extended_properties, None, None) + deltas = {'instances': 7, 'volumes': 25} # volumes=1*3+5*3+7*1 + mock_check_quotas.assert_called_with(mock.ANY, deltas) + @mock.patch.object(task_api, 'load') @mock.patch.object(inst_models.Instance, 'create') @mock.patch.object(models.DBCluster, 'create') @@ -230,6 +253,33 @@ class MongoDBClusterTest(trove_testtools.TestCase): mock_ins_create.call_args_list].count(nics) self.assertEqual(7, nics_count) + @mock.patch.object(task_api, 'load') + @mock.patch.object(models.DBCluster, 'create') + @mock.patch.object(models, 'validate_instance_nics') + @mock.patch.object(QUOTAS, 'check_quotas') + @mock.patch.object(models, 'validate_instance_flavors') + @mock.patch.object(inst_models.Instance, 'create') + def test_create_with_extended_properties(self, mock_ins_create, *args): + extended_properties = { + "num_configsvr": 5, + "num_mongos": 7, + "configsvr_volume_size": 8, + "configsvr_volume_type": "foo_type", + "mongos_volume_size": 9, + "mongos_volume_type": "bar_type"} + self.cluster.create(mock.Mock(), + self.cluster_name, + self.datastore, + self.datastore_version, + self.instances, + extended_properties, None, None) + volume_args_list = [ + (arg[8], kw['volume_type']) for arg, kw in + mock_ins_create.call_args_list + ] + self.assertEqual(5, volume_args_list.count((8, "foo_type"))) + self.assertEqual(7, volume_args_list.count((9, "bar_type"))) + @mock.patch.object(task_api, 'load') @mock.patch.object(inst_models.Instance, 'create') @mock.patch.object(models.DBCluster, 'create') diff --git a/trove/tests/unittests/cluster/test_vertica_cluster.py b/trove/tests/unittests/cluster/test_vertica_cluster.py index 48abafa45a..8f2794cfb8 100644 --- a/trove/tests/unittests/cluster/test_vertica_cluster.py +++ b/trove/tests/unittests/cluster/test_vertica_cluster.py @@ -80,7 +80,7 @@ class ClusterTest(trove_testtools.TestCase): self.cluster_name, self.datastore, self.datastore_version, - [], None, None, None) + [], {}, None, None) @patch.object(DBCluster, 'create') @patch.object(inst_models.DBInstance, 'find_all') @@ -95,7 +95,7 @@ class ClusterTest(trove_testtools.TestCase): self.datastore, self.datastore_version, instances, - None, None, None) + {}, None, None) @patch.object(DBCluster, 'create') @patch.object(inst_models.DBInstance, 'find_all') @@ -117,7 +117,7 @@ class ClusterTest(trove_testtools.TestCase): self.datastore, self.datastore_version, instances, - None, None, None) + {}, None, None) @patch.object(DBCluster, 'create') @patch.object(inst_models.DBInstance, 'find_all') @@ -135,7 +135,7 @@ class ClusterTest(trove_testtools.TestCase): self.datastore, self.datastore_version, instances, - None, None, None) + {}, None, None) @patch.object(DBCluster, 'create') @patch.object(inst_models.DBInstance, 'find_all') @@ -159,7 +159,7 @@ class ClusterTest(trove_testtools.TestCase): self.datastore, self.datastore_version, instances, - None, None, None) + {}, None, None) @patch.object(DBCluster, 'create') @patch.object(inst_models.DBInstance, 'find_all') @@ -195,7 +195,7 @@ class ClusterTest(trove_testtools.TestCase): self.datastore, self.datastore_version, instances, - None, None, None) + {}, None, None) @patch.object(DBCluster, 'create') @patch.object(inst_models.DBInstance, 'find_all') @@ -213,7 +213,7 @@ class ClusterTest(trove_testtools.TestCase): self.datastore, self.datastore_version, instances, - None, None, None) + {}, None, None) @patch.object(inst_models.DBInstance, 'find_all') @patch.object(inst_models.Instance, 'create') @@ -231,7 +231,7 @@ class ClusterTest(trove_testtools.TestCase): self.datastore, self.datastore_version, instances, - None, None, None) + {}, None, None) mock_task_api.return_value.create_cluster.assert_called_with( mock_db_create.return_value.id) self.assertEqual(3, mock_ins_create.call_count) @@ -270,7 +270,7 @@ class ClusterTest(trove_testtools.TestCase): self.datastore, self.datastore_version, instances, - None, None, None) + {}, None, None) mock_task_api.return_value.create_cluster.assert_called_with( mock_db_create.return_value.id) self.assertEqual(3, mock_ins_create.call_count)