From 14fff1e5a8133908f99d3988e71f2c4f15560075 Mon Sep 17 00:00:00 2001 From: Ken Chen Date: Thu, 23 Jul 2015 10:54:19 +0800 Subject: [PATCH] Remove extra merge methods in plugins Use the general method merge_configs in utils.configs. Change-Id: I5742541c5240a2dc941d1ae7c3534c4fedb36484 --- sahara/plugins/cdh/v5/cloudera_utils.py | 28 ++++------------- sahara/plugins/cdh/v5_3_0/cloudera_utils.py | 30 +++++-------------- sahara/plugins/cdh/v5_4_0/cloudera_utils.py | 30 +++++-------------- sahara/plugins/vanilla/hadoop2/config.py | 24 ++++----------- .../plugins/vanilla/hadoop2/test_configs.py | 30 ------------------- 5 files changed, 25 insertions(+), 117 deletions(-) diff --git a/sahara/plugins/cdh/v5/cloudera_utils.py b/sahara/plugins/cdh/v5/cloudera_utils.py index f05ec0b6..34bfc113 100644 --- a/sahara/plugins/cdh/v5/cloudera_utils.py +++ b/sahara/plugins/cdh/v5/cloudera_utils.py @@ -13,8 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import six - from sahara.i18n import _ from sahara.plugins.cdh import cloudera_utils as cu from sahara.plugins.cdh.v5 import config_helper as c_helper @@ -22,6 +20,7 @@ from sahara.plugins.cdh.v5 import plugin_utils as pu from sahara.plugins.cdh.v5 import validation as v from sahara.swift import swift_helper from sahara.utils import cluster_progress_ops as cpo +from sahara.utils import configs as s_cfg from sahara.utils import xmlutils @@ -35,21 +34,6 @@ ZOOKEEPER_SERVICE_TYPE = 'ZOOKEEPER' HBASE_SERVICE_TYPE = 'HBASE' -def _merge_dicts(a, b): - res = {} - - def update(cfg): - for service, configs in six.iteritems(cfg): - if not res.get(service): - res[service] = {} - - res[service].update(configs) - - update(a) - update(b) - return res - - class ClouderaUtilsV5(cu.ClouderaUtils): def __init__(self): cu.ClouderaUtils.__init__(self) @@ -226,9 +210,9 @@ class ClouderaUtilsV5(cu.ClouderaUtils): } } - all_confs = _merge_dicts(all_confs, hue_confs) - all_confs = _merge_dicts(all_confs, hive_confs) - all_confs = _merge_dicts(all_confs, cluster.cluster_configs) + all_confs = s_cfg.merge_configs(all_confs, hue_confs) + all_confs = s_cfg.merge_configs(all_confs, hive_confs) + all_confs = s_cfg.merge_configs(all_confs, cluster.cluster_configs) if node_group: paths = node_group.storage_paths() @@ -257,7 +241,7 @@ class ClouderaUtilsV5(cu.ClouderaUtils): ng_user_confs = self.pu.convert_process_configs( node_group.node_configs) - all_confs = _merge_dicts(all_confs, ng_user_confs) - all_confs = _merge_dicts(all_confs, ng_default_confs) + all_confs = s_cfg.merge_configs(all_confs, ng_user_confs) + all_confs = s_cfg.merge_configs(all_confs, ng_default_confs) return all_confs.get(service, {}) diff --git a/sahara/plugins/cdh/v5_3_0/cloudera_utils.py b/sahara/plugins/cdh/v5_3_0/cloudera_utils.py index 54fb3e5d..509b87ca 100644 --- a/sahara/plugins/cdh/v5_3_0/cloudera_utils.py +++ b/sahara/plugins/cdh/v5_3_0/cloudera_utils.py @@ -13,8 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import six - from sahara.i18n import _ from sahara.plugins.cdh import cloudera_utils as cu from sahara.plugins.cdh.v5_3_0 import config_helper as c_helper @@ -22,6 +20,7 @@ from sahara.plugins.cdh.v5_3_0 import plugin_utils as pu from sahara.plugins.cdh.v5_3_0 import validation as v from sahara.swift import swift_helper from sahara.utils import cluster_progress_ops as cpo +from sahara.utils import configs as s_cfg from sahara.utils import xmlutils @@ -41,21 +40,6 @@ KS_INDEXER_SERVICE_TYPE = 'KS_INDEXER' IMPALA_SERVICE_TYPE = 'IMPALA' -def _merge_dicts(a, b): - res = {} - - def update(cfg): - for service, configs in six.iteritems(cfg): - if not res.get(service): - res[service] = {} - - res[service].update(configs) - - update(a) - update(b) - return res - - class ClouderaUtilsV530(cu.ClouderaUtils): FLUME_SERVICE_NAME = 'flume01' SOLR_SERVICE_NAME = 'solr01' @@ -355,10 +339,10 @@ class ClouderaUtilsV530(cu.ClouderaUtils): } } - all_confs = _merge_dicts(all_confs, hue_confs) - all_confs = _merge_dicts(all_confs, hive_confs) - all_confs = _merge_dicts(all_confs, sentry_confs) - all_confs = _merge_dicts(all_confs, cluster.cluster_configs) + all_confs = s_cfg.merge_configs(all_confs, hue_confs) + all_confs = s_cfg.merge_configs(all_confs, hive_confs) + all_confs = s_cfg.merge_configs(all_confs, sentry_confs) + all_confs = s_cfg.merge_configs(all_confs, cluster.cluster_configs) if node_group: paths = node_group.storage_paths() @@ -399,7 +383,7 @@ class ClouderaUtilsV530(cu.ClouderaUtils): ng_user_confs = self.pu.convert_process_configs( node_group.node_configs) - all_confs = _merge_dicts(all_confs, ng_user_confs) - all_confs = _merge_dicts(all_confs, ng_default_confs) + all_confs = s_cfg.merge_configs(all_confs, ng_user_confs) + all_confs = s_cfg.merge_configs(all_confs, ng_default_confs) return all_confs.get(service, {}) diff --git a/sahara/plugins/cdh/v5_4_0/cloudera_utils.py b/sahara/plugins/cdh/v5_4_0/cloudera_utils.py index 38984100..9290d2dc 100644 --- a/sahara/plugins/cdh/v5_4_0/cloudera_utils.py +++ b/sahara/plugins/cdh/v5_4_0/cloudera_utils.py @@ -13,8 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import six - from sahara.i18n import _ from sahara.plugins.cdh import cloudera_utils as cu from sahara.plugins.cdh.v5_4_0 import config_helper as c_helper @@ -22,6 +20,7 @@ from sahara.plugins.cdh.v5_4_0 import plugin_utils as pu from sahara.plugins.cdh.v5_4_0 import validation as v from sahara.swift import swift_helper from sahara.utils import cluster_progress_ops as cpo +from sahara.utils import configs as s_cfg from sahara.utils import xmlutils @@ -42,21 +41,6 @@ IMPALA_SERVICE_TYPE = 'IMPALA' KMS_SERVICE_TYPE = 'KMS' -def _merge_dicts(a, b): - res = {} - - def update(cfg): - for service, configs in six.iteritems(cfg): - if not res.get(service): - res[service] = {} - - res[service].update(configs) - - update(a) - update(b) - return res - - class ClouderaUtilsV540(cu.ClouderaUtils): FLUME_SERVICE_NAME = 'flume01' SOLR_SERVICE_NAME = 'solr01' @@ -369,10 +353,10 @@ class ClouderaUtilsV540(cu.ClouderaUtils): } } - all_confs = _merge_dicts(all_confs, hue_confs) - all_confs = _merge_dicts(all_confs, hive_confs) - all_confs = _merge_dicts(all_confs, sentry_confs) - all_confs = _merge_dicts(all_confs, cluster.cluster_configs) + all_confs = s_cfg.merge_configs(all_confs, hue_confs) + all_confs = s_cfg.merge_configs(all_confs, hive_confs) + all_confs = s_cfg.merge_configs(all_confs, sentry_confs) + all_confs = s_cfg.merge_configs(all_confs, cluster.cluster_configs) if node_group: paths = node_group.storage_paths() @@ -413,7 +397,7 @@ class ClouderaUtilsV540(cu.ClouderaUtils): ng_user_confs = self.pu.convert_process_configs( node_group.node_configs) - all_confs = _merge_dicts(all_confs, ng_user_confs) - all_confs = _merge_dicts(all_confs, ng_default_confs) + all_confs = s_cfg.merge_configs(all_confs, ng_user_confs) + all_confs = s_cfg.merge_configs(all_confs, ng_default_confs) return all_confs.get(service, {}) diff --git a/sahara/plugins/vanilla/hadoop2/config.py b/sahara/plugins/vanilla/hadoop2/config.py index 22ef5ff1..91929059 100644 --- a/sahara/plugins/vanilla/hadoop2/config.py +++ b/sahara/plugins/vanilla/hadoop2/config.py @@ -26,6 +26,7 @@ from sahara.plugins.vanilla import utils as vu from sahara.swift import swift_helper as swift from sahara.topology import topology_helper as th from sahara.utils import cluster_progress_ops as cpo +from sahara.utils import configs as s_cfg from sahara.utils import files as f from sahara.utils import proxy from sahara.utils import xmlutils as x @@ -83,8 +84,8 @@ def _provisioning_configs(pctx, instance): def _generate_configs(pctx, node_group): hadoop_xml_confs = _get_hadoop_configs(pctx, node_group) user_xml_confs, user_env_confs = _get_user_configs(pctx, node_group) - xml_confs = _merge_configs(user_xml_confs, hadoop_xml_confs) - env_confs = _merge_configs(pctx['env_confs'], user_env_confs) + xml_confs = s_cfg.merge_configs(user_xml_confs, hadoop_xml_confs) + env_confs = s_cfg.merge_configs(pctx['env_confs'], user_env_confs) return xml_confs, env_confs @@ -190,8 +191,8 @@ def _get_user_configs(pctx, node_group): cl_xml_confs, cl_env_confs = _separate_configs( node_group.cluster.cluster_configs, pctx['env_confs']) - xml_confs = _merge_configs(cl_xml_confs, ng_xml_confs) - env_confs = _merge_configs(cl_env_confs, ng_env_confs) + xml_confs = s_cfg.merge_configs(cl_xml_confs, ng_xml_confs) + env_confs = s_cfg.merge_configs(cl_env_confs, ng_env_confs) return xml_confs, env_confs @@ -336,21 +337,6 @@ def _make_hadoop_paths(paths, hadoop_dir): return [path + hadoop_dir for path in paths] -def _merge_configs(a, b): - res = {} - - def update(cfg): - for service, configs in six.iteritems(cfg): - if not res.get(service): - res[service] = {} - - res[service].update(configs) - - update(a) - update(b) - return res - - @cpo.event_wrapper( True, step=_("Configure topology data"), param=('cluster', 1)) def configure_topology_data(pctx, cluster): diff --git a/sahara/tests/unit/plugins/vanilla/hadoop2/test_configs.py b/sahara/tests/unit/plugins/vanilla/hadoop2/test_configs.py index ed42b6fa..cbfa6cd5 100644 --- a/sahara/tests/unit/plugins/vanilla/hadoop2/test_configs.py +++ b/sahara/tests/unit/plugins/vanilla/hadoop2/test_configs.py @@ -32,36 +32,6 @@ class VanillaTwoConfigTestCase(base.SaharaTestCase): } self.assertEqual(expected, dirs) - def test_merge_configs(self): - a = { - 'HDFS': { - 'param1': 'value1', - 'param2': 'value2' - } - } - b = { - 'HDFS': { - 'param1': 'value3', - 'param3': 'value4' - }, - 'YARN': { - 'param5': 'value5' - } - } - - res = c._merge_configs(a, b) - expected = { - 'HDFS': { - 'param1': 'value3', - 'param2': 'value2', - 'param3': 'value4' - }, - 'YARN': { - 'param5': 'value5' - } - } - self.assertEqual(expected, res) - class FakeNG(object): def __init__(self, storage_paths=None):