Remove extra merge methods in plugins
Use the general method merge_configs in utils.configs. Change-Id: I5742541c5240a2dc941d1ae7c3534c4fedb36484
This commit is contained in:
Ken Chen
parent
f0e681426a
commit
14fff1e5a8
|
|
@ -13,8 +13,6 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import six
|
||||
|
||||
from sahara.i18n import _
|
||||
from sahara.plugins.cdh import cloudera_utils as cu
|
||||
from sahara.plugins.cdh.v5 import config_helper as c_helper
|
||||
|
|
@ -22,6 +20,7 @@ from sahara.plugins.cdh.v5 import plugin_utils as pu
|
|||
from sahara.plugins.cdh.v5 import validation as v
|
||||
from sahara.swift import swift_helper
|
||||
from sahara.utils import cluster_progress_ops as cpo
|
||||
from sahara.utils import configs as s_cfg
|
||||
from sahara.utils import xmlutils
|
||||
|
||||
|
||||
|
|
@ -35,21 +34,6 @@ ZOOKEEPER_SERVICE_TYPE = 'ZOOKEEPER'
|
|||
HBASE_SERVICE_TYPE = 'HBASE'
|
||||
|
||||
|
||||
def _merge_dicts(a, b):
|
||||
res = {}
|
||||
|
||||
def update(cfg):
|
||||
for service, configs in six.iteritems(cfg):
|
||||
if not res.get(service):
|
||||
res[service] = {}
|
||||
|
||||
res[service].update(configs)
|
||||
|
||||
update(a)
|
||||
update(b)
|
||||
return res
|
||||
|
||||
|
||||
class ClouderaUtilsV5(cu.ClouderaUtils):
|
||||
def __init__(self):
|
||||
cu.ClouderaUtils.__init__(self)
|
||||
|
|
@ -226,9 +210,9 @@ class ClouderaUtilsV5(cu.ClouderaUtils):
|
|||
}
|
||||
}
|
||||
|
||||
all_confs = _merge_dicts(all_confs, hue_confs)
|
||||
all_confs = _merge_dicts(all_confs, hive_confs)
|
||||
all_confs = _merge_dicts(all_confs, cluster.cluster_configs)
|
||||
all_confs = s_cfg.merge_configs(all_confs, hue_confs)
|
||||
all_confs = s_cfg.merge_configs(all_confs, hive_confs)
|
||||
all_confs = s_cfg.merge_configs(all_confs, cluster.cluster_configs)
|
||||
|
||||
if node_group:
|
||||
paths = node_group.storage_paths()
|
||||
|
|
@ -257,7 +241,7 @@ class ClouderaUtilsV5(cu.ClouderaUtils):
|
|||
|
||||
ng_user_confs = self.pu.convert_process_configs(
|
||||
node_group.node_configs)
|
||||
all_confs = _merge_dicts(all_confs, ng_user_confs)
|
||||
all_confs = _merge_dicts(all_confs, ng_default_confs)
|
||||
all_confs = s_cfg.merge_configs(all_confs, ng_user_confs)
|
||||
all_confs = s_cfg.merge_configs(all_confs, ng_default_confs)
|
||||
|
||||
return all_confs.get(service, {})
|
||||
|
|
|
|||
|
|
@ -13,8 +13,6 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import six
|
||||
|
||||
from sahara.i18n import _
|
||||
from sahara.plugins.cdh import cloudera_utils as cu
|
||||
from sahara.plugins.cdh.v5_3_0 import config_helper as c_helper
|
||||
|
|
@ -22,6 +20,7 @@ from sahara.plugins.cdh.v5_3_0 import plugin_utils as pu
|
|||
from sahara.plugins.cdh.v5_3_0 import validation as v
|
||||
from sahara.swift import swift_helper
|
||||
from sahara.utils import cluster_progress_ops as cpo
|
||||
from sahara.utils import configs as s_cfg
|
||||
from sahara.utils import xmlutils
|
||||
|
||||
|
||||
|
|
@ -41,21 +40,6 @@ KS_INDEXER_SERVICE_TYPE = 'KS_INDEXER'
|
|||
IMPALA_SERVICE_TYPE = 'IMPALA'
|
||||
|
||||
|
||||
def _merge_dicts(a, b):
|
||||
res = {}
|
||||
|
||||
def update(cfg):
|
||||
for service, configs in six.iteritems(cfg):
|
||||
if not res.get(service):
|
||||
res[service] = {}
|
||||
|
||||
res[service].update(configs)
|
||||
|
||||
update(a)
|
||||
update(b)
|
||||
return res
|
||||
|
||||
|
||||
class ClouderaUtilsV530(cu.ClouderaUtils):
|
||||
FLUME_SERVICE_NAME = 'flume01'
|
||||
SOLR_SERVICE_NAME = 'solr01'
|
||||
|
|
@ -355,10 +339,10 @@ class ClouderaUtilsV530(cu.ClouderaUtils):
|
|||
}
|
||||
}
|
||||
|
||||
all_confs = _merge_dicts(all_confs, hue_confs)
|
||||
all_confs = _merge_dicts(all_confs, hive_confs)
|
||||
all_confs = _merge_dicts(all_confs, sentry_confs)
|
||||
all_confs = _merge_dicts(all_confs, cluster.cluster_configs)
|
||||
all_confs = s_cfg.merge_configs(all_confs, hue_confs)
|
||||
all_confs = s_cfg.merge_configs(all_confs, hive_confs)
|
||||
all_confs = s_cfg.merge_configs(all_confs, sentry_confs)
|
||||
all_confs = s_cfg.merge_configs(all_confs, cluster.cluster_configs)
|
||||
|
||||
if node_group:
|
||||
paths = node_group.storage_paths()
|
||||
|
|
@ -399,7 +383,7 @@ class ClouderaUtilsV530(cu.ClouderaUtils):
|
|||
|
||||
ng_user_confs = self.pu.convert_process_configs(
|
||||
node_group.node_configs)
|
||||
all_confs = _merge_dicts(all_confs, ng_user_confs)
|
||||
all_confs = _merge_dicts(all_confs, ng_default_confs)
|
||||
all_confs = s_cfg.merge_configs(all_confs, ng_user_confs)
|
||||
all_confs = s_cfg.merge_configs(all_confs, ng_default_confs)
|
||||
|
||||
return all_confs.get(service, {})
|
||||
|
|
|
|||
|
|
@ -13,8 +13,6 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import six
|
||||
|
||||
from sahara.i18n import _
|
||||
from sahara.plugins.cdh import cloudera_utils as cu
|
||||
from sahara.plugins.cdh.v5_4_0 import config_helper as c_helper
|
||||
|
|
@ -22,6 +20,7 @@ from sahara.plugins.cdh.v5_4_0 import plugin_utils as pu
|
|||
from sahara.plugins.cdh.v5_4_0 import validation as v
|
||||
from sahara.swift import swift_helper
|
||||
from sahara.utils import cluster_progress_ops as cpo
|
||||
from sahara.utils import configs as s_cfg
|
||||
from sahara.utils import xmlutils
|
||||
|
||||
|
||||
|
|
@ -42,21 +41,6 @@ IMPALA_SERVICE_TYPE = 'IMPALA'
|
|||
KMS_SERVICE_TYPE = 'KMS'
|
||||
|
||||
|
||||
def _merge_dicts(a, b):
|
||||
res = {}
|
||||
|
||||
def update(cfg):
|
||||
for service, configs in six.iteritems(cfg):
|
||||
if not res.get(service):
|
||||
res[service] = {}
|
||||
|
||||
res[service].update(configs)
|
||||
|
||||
update(a)
|
||||
update(b)
|
||||
return res
|
||||
|
||||
|
||||
class ClouderaUtilsV540(cu.ClouderaUtils):
|
||||
FLUME_SERVICE_NAME = 'flume01'
|
||||
SOLR_SERVICE_NAME = 'solr01'
|
||||
|
|
@ -369,10 +353,10 @@ class ClouderaUtilsV540(cu.ClouderaUtils):
|
|||
}
|
||||
}
|
||||
|
||||
all_confs = _merge_dicts(all_confs, hue_confs)
|
||||
all_confs = _merge_dicts(all_confs, hive_confs)
|
||||
all_confs = _merge_dicts(all_confs, sentry_confs)
|
||||
all_confs = _merge_dicts(all_confs, cluster.cluster_configs)
|
||||
all_confs = s_cfg.merge_configs(all_confs, hue_confs)
|
||||
all_confs = s_cfg.merge_configs(all_confs, hive_confs)
|
||||
all_confs = s_cfg.merge_configs(all_confs, sentry_confs)
|
||||
all_confs = s_cfg.merge_configs(all_confs, cluster.cluster_configs)
|
||||
|
||||
if node_group:
|
||||
paths = node_group.storage_paths()
|
||||
|
|
@ -413,7 +397,7 @@ class ClouderaUtilsV540(cu.ClouderaUtils):
|
|||
|
||||
ng_user_confs = self.pu.convert_process_configs(
|
||||
node_group.node_configs)
|
||||
all_confs = _merge_dicts(all_confs, ng_user_confs)
|
||||
all_confs = _merge_dicts(all_confs, ng_default_confs)
|
||||
all_confs = s_cfg.merge_configs(all_confs, ng_user_confs)
|
||||
all_confs = s_cfg.merge_configs(all_confs, ng_default_confs)
|
||||
|
||||
return all_confs.get(service, {})
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ from sahara.plugins.vanilla import utils as vu
|
|||
from sahara.swift import swift_helper as swift
|
||||
from sahara.topology import topology_helper as th
|
||||
from sahara.utils import cluster_progress_ops as cpo
|
||||
from sahara.utils import configs as s_cfg
|
||||
from sahara.utils import files as f
|
||||
from sahara.utils import proxy
|
||||
from sahara.utils import xmlutils as x
|
||||
|
|
@ -83,8 +84,8 @@ def _provisioning_configs(pctx, instance):
|
|||
def _generate_configs(pctx, node_group):
|
||||
hadoop_xml_confs = _get_hadoop_configs(pctx, node_group)
|
||||
user_xml_confs, user_env_confs = _get_user_configs(pctx, node_group)
|
||||
xml_confs = _merge_configs(user_xml_confs, hadoop_xml_confs)
|
||||
env_confs = _merge_configs(pctx['env_confs'], user_env_confs)
|
||||
xml_confs = s_cfg.merge_configs(user_xml_confs, hadoop_xml_confs)
|
||||
env_confs = s_cfg.merge_configs(pctx['env_confs'], user_env_confs)
|
||||
|
||||
return xml_confs, env_confs
|
||||
|
||||
|
|
@ -190,8 +191,8 @@ def _get_user_configs(pctx, node_group):
|
|||
cl_xml_confs, cl_env_confs = _separate_configs(
|
||||
node_group.cluster.cluster_configs, pctx['env_confs'])
|
||||
|
||||
xml_confs = _merge_configs(cl_xml_confs, ng_xml_confs)
|
||||
env_confs = _merge_configs(cl_env_confs, ng_env_confs)
|
||||
xml_confs = s_cfg.merge_configs(cl_xml_confs, ng_xml_confs)
|
||||
env_confs = s_cfg.merge_configs(cl_env_confs, ng_env_confs)
|
||||
return xml_confs, env_confs
|
||||
|
||||
|
||||
|
|
@ -336,21 +337,6 @@ def _make_hadoop_paths(paths, hadoop_dir):
|
|||
return [path + hadoop_dir for path in paths]
|
||||
|
||||
|
||||
def _merge_configs(a, b):
|
||||
res = {}
|
||||
|
||||
def update(cfg):
|
||||
for service, configs in six.iteritems(cfg):
|
||||
if not res.get(service):
|
||||
res[service] = {}
|
||||
|
||||
res[service].update(configs)
|
||||
|
||||
update(a)
|
||||
update(b)
|
||||
return res
|
||||
|
||||
|
||||
@cpo.event_wrapper(
|
||||
True, step=_("Configure topology data"), param=('cluster', 1))
|
||||
def configure_topology_data(pctx, cluster):
|
||||
|
|
|
|||
|
|
@ -32,36 +32,6 @@ class VanillaTwoConfigTestCase(base.SaharaTestCase):
|
|||
}
|
||||
self.assertEqual(expected, dirs)
|
||||
|
||||
def test_merge_configs(self):
|
||||
a = {
|
||||
'HDFS': {
|
||||
'param1': 'value1',
|
||||
'param2': 'value2'
|
||||
}
|
||||
}
|
||||
b = {
|
||||
'HDFS': {
|
||||
'param1': 'value3',
|
||||
'param3': 'value4'
|
||||
},
|
||||
'YARN': {
|
||||
'param5': 'value5'
|
||||
}
|
||||
}
|
||||
|
||||
res = c._merge_configs(a, b)
|
||||
expected = {
|
||||
'HDFS': {
|
||||
'param1': 'value3',
|
||||
'param2': 'value2',
|
||||
'param3': 'value4'
|
||||
},
|
||||
'YARN': {
|
||||
'param5': 'value5'
|
||||
}
|
||||
}
|
||||
self.assertEqual(expected, res)
|
||||
|
||||
|
||||
class FakeNG(object):
|
||||
def __init__(self, storage_paths=None):
|
||||
|
|
|
|||
Loading…
Reference in New Issue