Add CDH validation for attached volume size
CDH plugin has a DATANODE configuration parameter called "dfs_datanode_du_reserved". If user sets cinder volume as the storage location and sets volumes size less than the value of the parameter, HDFS would not work. The cluster deployment would be Error in "Starting" Step because of "Oozie Share Lib Upload Failed". This patch adds a validation in "Validating" Step before spawning Cluster. BTW, If this situation (volume size less than "du_reserved") happen in Vanilla, Spark plugin, the deploy can finish successfully, but the HDFS can't be used when upload file to HDFS. Closes-Bug: #1709071 Change-Id: I840559d54ca474299eeec9812dc1573ab305bc1b
This commit is contained in:
parent
5095b56909
commit
a126fb2140
@ -444,6 +444,10 @@ class AbstractPluginUtils(object):
|
||||
conf = cluster.cluster_configs
|
||||
if service in conf and name in conf[service]:
|
||||
return types.transform_to_num(conf[service][name])
|
||||
for node_group in cluster.node_groups:
|
||||
conf = node_group.node_configs
|
||||
if service in conf and name in conf[service]:
|
||||
return types.transform_to_num(conf[service][name])
|
||||
for config in configs:
|
||||
if config.applicable_target == service and config.name == name:
|
||||
return types.transform_to_num(config.default_value)
|
||||
|
@ -66,6 +66,14 @@ class Validator(object):
|
||||
_('Number of datanodes must be not'
|
||||
' less than dfs_replication.'))
|
||||
|
||||
du_reserved = cls.PU.get_config_value(
|
||||
'DATANODE', 'dfs_datanode_du_reserved', cluster)
|
||||
du_reserved = du_reserved/1073741824.
|
||||
for node_group in cluster.node_groups:
|
||||
volume_size = node_group.volumes_size
|
||||
if volume_size and volume_size < du_reserved:
|
||||
raise ex.InvalidVolumeSizeException(volume_size, du_reserved)
|
||||
|
||||
rm_count = cls.get_inst_count(cluster, 'YARN_RESOURCEMANAGER')
|
||||
if rm_count > 1:
|
||||
raise ex.InvalidComponentCountException('YARN_RESOURCEMANAGER',
|
||||
|
@ -178,3 +178,18 @@ class AllValidationsFailedError(ImageValidationError):
|
||||
message = self.sub_message % data
|
||||
|
||||
super(AllValidationsFailedError, self).__init__(message)
|
||||
|
||||
|
||||
class InvalidVolumeSizeException(e.SaharaException):
|
||||
"""Exception indicating invalid configuration of components in a cluster.
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, volume_size, reserved):
|
||||
message = _("Volume size: %(volume_size)s GB should be greater than "
|
||||
"value of \"dfs_datanode_du_reserved\": %(reserved)s GB")
|
||||
self.message = message % {"volume_size": volume_size,
|
||||
"reserved": reserved}
|
||||
self.code = "INVALID_CONFIGURATION"
|
||||
|
||||
super(InvalidVolumeSizeException, self).__init__()
|
||||
|
@ -21,6 +21,7 @@ from sahara.plugins import exceptions as ex
|
||||
from sahara.tests.unit import base
|
||||
from sahara.tests.unit import testutils as tu
|
||||
|
||||
ivse = ex.InvalidVolumeSizeException
|
||||
icce = ex.InvalidComponentCountException
|
||||
rsme = ex.RequiredServiceMissingException
|
||||
icte = ex.InvalidClusterTopology
|
||||
@ -28,9 +29,9 @@ nnce = ex.NameNodeHAConfigurationError
|
||||
rmce = ex.ResourceManagerHAConfigurationError
|
||||
|
||||
|
||||
def make_ng_dict_with_inst(counter, name, flavor,
|
||||
processes, count, instances=None,
|
||||
**kwargs):
|
||||
def make_ng_dict_with_inst(counter, name, flavor, processes, count,
|
||||
instances=None, volumes_size=None,
|
||||
node_configs=None, **kwargs):
|
||||
if not instances:
|
||||
instances = []
|
||||
for i in range(count):
|
||||
@ -39,8 +40,8 @@ def make_ng_dict_with_inst(counter, name, flavor,
|
||||
"fake_inst{0}".format(n),
|
||||
management_ip='1.2.3.{0}'.format(n))
|
||||
instances.append(instance)
|
||||
return tu.make_ng_dict(name, flavor, processes,
|
||||
count, instances, **kwargs)
|
||||
return tu.make_ng_dict(name, flavor, processes, count, instances,
|
||||
volumes_size, node_configs, **kwargs)
|
||||
|
||||
|
||||
def get_fake_cluster_with_process(processes=None,
|
||||
@ -139,6 +140,14 @@ class BaseValidationTestCase(base.SaharaTestCase):
|
||||
{'cluster_configs': {'HDFS': {'dfs_replication': 3}}}],
|
||||
[icce, {}, [],
|
||||
{'cluster_configs': {'HDFS': {'dfs_replication': 4}}}],
|
||||
[ivse, {}, [(
|
||||
'worker_ng_vol', 1, ['HDFS_DATANODE', 'YARN_NODEMANAGER'],
|
||||
3, None, 20,
|
||||
{'DATANODE': {'dfs_datanode_du_reserved': 22548578304}})]],
|
||||
[None, {}, [(
|
||||
'worker_ng_vol', 1, ['HDFS_DATANODE', 'YARN_NODEMANAGER'],
|
||||
3, None, 22,
|
||||
{'DATANODE': {'dfs_datanode_du_reserved': 22548578304}})]],
|
||||
[None, {'YARN_RESOURCEMANAGER': 1}],
|
||||
[icce, {'YARN_RESOURCEMANAGER': 2}],
|
||||
[None, {'YARN_JOBHISTORY': 1}],
|
||||
|
@ -28,11 +28,14 @@ def create_cluster(name, tenant, plugin, version, node_groups, **kwargs):
|
||||
return r.ClusterResource(dct)
|
||||
|
||||
|
||||
def make_ng_dict(name, flavor, processes, count, instances=None, **kwargs):
|
||||
def make_ng_dict(name, flavor, processes, count, instances=None,
|
||||
volumes_size=None, node_configs=None, **kwargs):
|
||||
node_configs = node_configs or {}
|
||||
instances = instances or []
|
||||
dct = {'id': uuidutils.generate_uuid(), 'name': name,
|
||||
'flavor_id': flavor, 'node_processes': processes,
|
||||
'count': count, 'instances': instances, 'node_configs': {},
|
||||
'volumes_size': volumes_size, 'flavor_id': flavor,
|
||||
'node_processes': processes, 'count': count,
|
||||
'instances': instances, 'node_configs': node_configs,
|
||||
'security_groups': None, 'auto_security_group': False,
|
||||
'availability_zone': None, 'volumes_availability_zone': None,
|
||||
'open_ports': [], 'is_proxy_gateway': False,
|
||||
|
Loading…
Reference in New Issue
Block a user