diff --git a/doc/source/userdoc/cdh_plugin.rst b/doc/source/userdoc/cdh_plugin.rst index 1c9b129d..57d33b0f 100644 --- a/doc/source/userdoc/cdh_plugin.rst +++ b/doc/source/userdoc/cdh_plugin.rst @@ -55,6 +55,7 @@ cloudera plugin versions: + Cluster must contain exactly one manager. + Cluster must contain exactly one namenode. + Cluster must contain exactly one secondarynamenode. + + Cluster must contain at least ``dfs_replication`` datanodes. + Cluster can contain at most one resourcemanager and this process is also required by nodemanager. + Cluster can contain at most one jobhistory and this process is also diff --git a/sahara/plugins/cdh/plugin_utils.py b/sahara/plugins/cdh/plugin_utils.py index 1f0565b2..17bcc985 100644 --- a/sahara/plugins/cdh/plugin_utils.py +++ b/sahara/plugins/cdh/plugin_utils.py @@ -22,6 +22,7 @@ from oslo_log import log as logging from sahara.conductor import resource as res from sahara import context +from sahara import exceptions as exc from sahara.i18n import _ from sahara.plugins.cdh import commands as cmd from sahara.plugins import utils as u @@ -323,3 +324,15 @@ class AbstractPluginUtils(object): cmd.write_centos_repository(r, cdh5_repo_content, 'cdh') cmd.write_centos_repository(r, cm5_repo_content, 'cm') cmd.update_repository(r) + + def _get_config_value(self, service, name, configs, cluster=None): + conf = cluster.cluster_configs + if cluster: + if service in conf and name in conf[service]: + return conf[service][name] + for config in configs: + if config.applicable_target == service and config.name == name: + return config.default_value + raise exc.InvalidDataException( + _("Unable to find config: {applicable_target: %(target)s, name: " + "%(name)s").format(target=service, name=name)) diff --git a/sahara/plugins/cdh/v5/plugin_utils.py b/sahara/plugins/cdh/v5/plugin_utils.py index 24304dfb..1a53417a 100644 --- a/sahara/plugins/cdh/v5/plugin_utils.py +++ b/sahara/plugins/cdh/v5/plugin_utils.py @@ -57,3 +57,7 @@ class PluginUtilsV5(pu.AbstractPluginUtils): def start_cloudera_manager(self, cluster): self._start_cloudera_manager( cluster, c_helper.AWAIT_MANAGER_STARTING_TIMEOUT) + + def get_config_value(self, service, name, cluster=None): + configs = c_helper.get_plugin_configs() + return self._get_config_value(service, name, configs, cluster) diff --git a/sahara/plugins/cdh/v5/validation.py b/sahara/plugins/cdh/v5/validation.py index be76aae0..b80d677a 100644 --- a/sahara/plugins/cdh/v5/validation.py +++ b/sahara/plugins/cdh/v5/validation.py @@ -36,6 +36,12 @@ def validate_cluster_creating(cluster): if snn_count != 1: raise ex.InvalidComponentCountException('HDFS_SECONDARYNAMENODE', 1, snn_count) + dn_count = _get_inst_count(cluster, 'HDFS_DATANODE') + replicas = PU.get_config_value('HDFS', 'dfs_replication', cluster) + if dn_count < replicas: + raise ex.InvalidComponentCountException( + 'HDFS_DATANODE', replicas, dn_count, + _('Number of datanodes must be not less than dfs_replication.')) rm_count = _get_inst_count(cluster, 'YARN_RESOURCEMANAGER') if rm_count > 1: @@ -58,7 +64,6 @@ def validate_cluster_creating(cluster): 'YARN_RESOURCEMANAGER', required_by='YARN_NODEMANAGER') oo_count = _get_inst_count(cluster, 'OOZIE_SERVER') - dn_count = _get_inst_count(cluster, 'HDFS_DATANODE') if oo_count > 1: raise ex.InvalidComponentCountException('OOZIE_SERVER', _('0 or 1'), oo_count) @@ -156,7 +161,8 @@ def validate_existing_ng_scaling(cluster, existing): dn_to_delete = 0 for ng in cluster.node_groups: if ng.id in existing: - if ng.count > existing[ng.id] and "datanode" in ng.node_processes: + if (ng.count > existing[ng.id] and + "HDFS_DATANODE" in ng.node_processes): dn_to_delete += ng.count - existing[ng.id] if not set(ng.node_processes).issubset(scalable_processes): @@ -165,6 +171,13 @@ def validate_existing_ng_scaling(cluster, existing): raise ex.NodeGroupCannotBeScaled( ng.name, msg % {'processes': ' '.join(ng.node_processes)}) + dn_count = _get_inst_count(cluster, 'HDFS_DATANODE') - dn_to_delete + replicas = PU.get_config_value('HDFS', 'dfs_replication', cluster) + if dn_count < replicas: + raise ex.ClusterCannotBeScaled( + cluster, + _('Number of datanodes must be not less than dfs_replication.')) + def _get_scalable_processes(): return ['HDFS_DATANODE', 'YARN_NODEMANAGER'] diff --git a/sahara/plugins/cdh/v5_3_0/plugin_utils.py b/sahara/plugins/cdh/v5_3_0/plugin_utils.py index 032cfdf1..03ffa98e 100644 --- a/sahara/plugins/cdh/v5_3_0/plugin_utils.py +++ b/sahara/plugins/cdh/v5_3_0/plugin_utils.py @@ -127,3 +127,7 @@ class PluginUtilsV530(pu.AbstractPluginUtils): def start_cloudera_manager(self, cluster): self._start_cloudera_manager( cluster, c_helper.AWAIT_MANAGER_STARTING_TIMEOUT) + + def get_config_value(self, service, name, cluster=None): + configs = c_helper.get_plugin_configs() + return self._get_config_value(service, name, configs, cluster) diff --git a/sahara/plugins/cdh/v5_3_0/validation.py b/sahara/plugins/cdh/v5_3_0/validation.py index aabc39a8..54d7bc55 100644 --- a/sahara/plugins/cdh/v5_3_0/validation.py +++ b/sahara/plugins/cdh/v5_3_0/validation.py @@ -37,6 +37,13 @@ def validate_cluster_creating(cluster): raise ex.InvalidComponentCountException('HDFS_SECONDARYNAMENODE', 1, snn_count) + dn_count = _get_inst_count(cluster, 'HDFS_DATANODE') + replicas = PU.get_config_value('HDFS', 'dfs_replication', cluster) + if dn_count < replicas: + raise ex.InvalidComponentCountException( + 'HDFS_DATANODE', replicas, dn_count, + _('Number of datanodes must be not less than dfs_replication.')) + rm_count = _get_inst_count(cluster, 'YARN_RESOURCEMANAGER') if rm_count > 1: raise ex.InvalidComponentCountException('YARN_RESOURCEMANAGER', @@ -58,7 +65,6 @@ def validate_cluster_creating(cluster): 'YARN_RESOURCEMANAGER', required_by='YARN_NODEMANAGER') oo_count = _get_inst_count(cluster, 'OOZIE_SERVER') - dn_count = _get_inst_count(cluster, 'HDFS_DATANODE') if oo_count > 1: raise ex.InvalidComponentCountException('OOZIE_SERVER', _('0 or 1'), oo_count) @@ -236,7 +242,8 @@ def validate_existing_ng_scaling(cluster, existing): dn_to_delete = 0 for ng in cluster.node_groups: if ng.id in existing: - if ng.count > existing[ng.id] and "datanode" in ng.node_processes: + if (ng.count > existing[ng.id] and + 'HDFS_DATANODE' in ng.node_processes): dn_to_delete += ng.count - existing[ng.id] if not set(ng.node_processes).issubset(scalable_processes): @@ -245,6 +252,13 @@ def validate_existing_ng_scaling(cluster, existing): raise ex.NodeGroupCannotBeScaled( ng.name, msg % {'processes': ' '.join(ng.node_processes)}) + dn_count = _get_inst_count(cluster, 'HDFS_DATANODE') - dn_to_delete + replicas = PU.get_config_value('HDFS', 'dfs_replication', cluster) + if dn_count < replicas: + raise ex.ClusterCannotBeScaled( + cluster, + _('Number of datanodes must be not less than dfs_replication.')) + def _get_scalable_processes(): return ['HDFS_DATANODE', 'YARN_NODEMANAGER']