612 lines
22 KiB
Python
612 lines
22 KiB
Python
# Copyright (c) 2014 Mirantis Inc.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
# implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
import re
|
|
import six
|
|
|
|
from oslo_log import log as logging
|
|
from oslo_utils import encodeutils
|
|
|
|
from heat.common import exception
|
|
from heat.common.i18n import _
|
|
from heat.engine import constraints
|
|
from heat.engine import properties
|
|
from heat.engine import resource
|
|
from heat.engine import rsrc_defn
|
|
from heat.engine import support
|
|
from heat.engine import translation
|
|
|
|
LOG = logging.getLogger(__name__)
|
|
|
|
# NOTE(pshchelo): copied from sahara/utils/api_validator.py
|
|
SAHARA_NAME_REGEX = (r"^(([a-zA-Z]|[a-zA-Z][a-zA-Z0-9\-]"
|
|
r"*[a-zA-Z0-9])\.)*([A-Za-z]|[A-Za-z]"
|
|
r"[A-Za-z0-9\-]*[A-Za-z0-9])$")
|
|
|
|
|
|
class SaharaNodeGroupTemplate(resource.Resource):
|
|
"""A resource for managing Sahara node group templates.
|
|
|
|
A Node Group Template describes a group of nodes within cluster. It
|
|
contains a list of hadoop processes that will be launched on each instance
|
|
in a group. Also a Node Group Template may provide node scoped
|
|
configurations for those processes.
|
|
"""
|
|
|
|
support_status = support.SupportStatus(version='2014.2')
|
|
|
|
PROPERTIES = (
|
|
NAME, PLUGIN_NAME, HADOOP_VERSION, FLAVOR, DESCRIPTION,
|
|
VOLUMES_PER_NODE, VOLUMES_SIZE, VOLUME_TYPE,
|
|
SECURITY_GROUPS, AUTO_SECURITY_GROUP,
|
|
AVAILABILITY_ZONE, VOLUMES_AVAILABILITY_ZONE,
|
|
NODE_PROCESSES, FLOATING_IP_POOL, NODE_CONFIGS, IMAGE_ID,
|
|
IS_PROXY_GATEWAY, VOLUME_LOCAL_TO_INSTANCE, USE_AUTOCONFIG,
|
|
SHARES
|
|
|
|
) = (
|
|
'name', 'plugin_name', 'hadoop_version', 'flavor', 'description',
|
|
'volumes_per_node', 'volumes_size', 'volume_type',
|
|
'security_groups', 'auto_security_group',
|
|
'availability_zone', 'volumes_availability_zone',
|
|
'node_processes', 'floating_ip_pool', 'node_configs', 'image_id',
|
|
'is_proxy_gateway', 'volume_local_to_instance', 'use_autoconfig',
|
|
'shares'
|
|
)
|
|
|
|
_SHARE_KEYS = (
|
|
SHARE_ID, PATH, ACCESS_LEVEL
|
|
) = (
|
|
'id', 'path', 'access_level'
|
|
)
|
|
|
|
properties_schema = {
|
|
NAME: properties.Schema(
|
|
properties.Schema.STRING,
|
|
_("Name for the Sahara Node Group Template."),
|
|
constraints=[
|
|
constraints.Length(min=1, max=50),
|
|
constraints.AllowedPattern(SAHARA_NAME_REGEX),
|
|
],
|
|
update_allowed=True
|
|
),
|
|
DESCRIPTION: properties.Schema(
|
|
properties.Schema.STRING,
|
|
_('Description of the Node Group Template.'),
|
|
default="",
|
|
update_allowed=True
|
|
),
|
|
PLUGIN_NAME: properties.Schema(
|
|
properties.Schema.STRING,
|
|
_('Plugin name.'),
|
|
required=True,
|
|
constraints=[
|
|
constraints.CustomConstraint('sahara.plugin')
|
|
],
|
|
update_allowed=True
|
|
),
|
|
HADOOP_VERSION: properties.Schema(
|
|
properties.Schema.STRING,
|
|
_('Version of Hadoop running on instances.'),
|
|
required=True,
|
|
update_allowed=True
|
|
),
|
|
FLAVOR: properties.Schema(
|
|
properties.Schema.STRING,
|
|
_('Name or ID Nova flavor for the nodes.'),
|
|
required=True,
|
|
constraints=[
|
|
constraints.CustomConstraint('nova.flavor')
|
|
],
|
|
update_allowed=True
|
|
),
|
|
VOLUMES_PER_NODE: properties.Schema(
|
|
properties.Schema.INTEGER,
|
|
_("Volumes per node."),
|
|
constraints=[
|
|
constraints.Range(min=0),
|
|
],
|
|
default=0,
|
|
update_allowed=True
|
|
),
|
|
VOLUMES_SIZE: properties.Schema(
|
|
properties.Schema.INTEGER,
|
|
_("Size of the volumes, in GB."),
|
|
constraints=[
|
|
constraints.Range(min=1),
|
|
],
|
|
update_allowed=True
|
|
),
|
|
VOLUME_TYPE: properties.Schema(
|
|
properties.Schema.STRING,
|
|
_("Type of the volume to create on Cinder backend."),
|
|
constraints=[
|
|
constraints.CustomConstraint('cinder.vtype')
|
|
],
|
|
update_allowed=True
|
|
),
|
|
SECURITY_GROUPS: properties.Schema(
|
|
properties.Schema.LIST,
|
|
_("List of security group names or IDs to assign to this "
|
|
"Node Group template."),
|
|
schema=properties.Schema(
|
|
properties.Schema.STRING,
|
|
),
|
|
update_allowed=True
|
|
),
|
|
AUTO_SECURITY_GROUP: properties.Schema(
|
|
properties.Schema.BOOLEAN,
|
|
_("Defines whether auto-assign security group to this "
|
|
"Node Group template."),
|
|
update_allowed=True
|
|
),
|
|
AVAILABILITY_ZONE: properties.Schema(
|
|
properties.Schema.STRING,
|
|
_("Availability zone to create servers in."),
|
|
update_allowed=True
|
|
),
|
|
VOLUMES_AVAILABILITY_ZONE: properties.Schema(
|
|
properties.Schema.STRING,
|
|
_("Availability zone to create volumes in."),
|
|
update_allowed=True
|
|
),
|
|
NODE_PROCESSES: properties.Schema(
|
|
properties.Schema.LIST,
|
|
_("List of processes to run on every node."),
|
|
required=True,
|
|
constraints=[
|
|
constraints.Length(min=1),
|
|
],
|
|
schema=properties.Schema(
|
|
properties.Schema.STRING,
|
|
),
|
|
update_allowed=True
|
|
),
|
|
FLOATING_IP_POOL: properties.Schema(
|
|
properties.Schema.STRING,
|
|
_("Name or UUID of the Neutron floating IP network or "
|
|
"name of the Nova floating ip pool to use. "
|
|
"Should not be provided when used with Nova-network "
|
|
"that auto-assign floating IPs."),
|
|
update_allowed=True
|
|
),
|
|
NODE_CONFIGS: properties.Schema(
|
|
properties.Schema.MAP,
|
|
_("Dictionary of node configurations."),
|
|
update_allowed=True
|
|
),
|
|
IMAGE_ID: properties.Schema(
|
|
properties.Schema.STRING,
|
|
_("ID of the image to use for the template."),
|
|
constraints=[
|
|
constraints.CustomConstraint('sahara.image'),
|
|
],
|
|
update_allowed=True
|
|
),
|
|
IS_PROXY_GATEWAY: properties.Schema(
|
|
properties.Schema.BOOLEAN,
|
|
_("Provide access to nodes using other nodes of the cluster "
|
|
"as proxy gateways."),
|
|
support_status=support.SupportStatus(version='5.0.0'),
|
|
update_allowed=True
|
|
),
|
|
VOLUME_LOCAL_TO_INSTANCE: properties.Schema(
|
|
properties.Schema.BOOLEAN,
|
|
_("Create volumes on the same physical port as an instance."),
|
|
support_status=support.SupportStatus(version='5.0.0'),
|
|
update_allowed=True
|
|
),
|
|
USE_AUTOCONFIG: properties.Schema(
|
|
properties.Schema.BOOLEAN,
|
|
_("Configure most important configs automatically."),
|
|
support_status=support.SupportStatus(version='5.0.0'),
|
|
update_allowed=True
|
|
),
|
|
SHARES: properties.Schema(
|
|
properties.Schema.LIST,
|
|
_("List of manila shares to be mounted."),
|
|
schema=properties.Schema(
|
|
properties.Schema.MAP,
|
|
schema={
|
|
SHARE_ID: properties.Schema(
|
|
properties.Schema.STRING,
|
|
_("Id of the manila share."),
|
|
required=True
|
|
),
|
|
PATH: properties.Schema(
|
|
properties.Schema.STRING,
|
|
_("Local path on each cluster node on which to mount "
|
|
"the share. Defaults to '/mnt/{share_id}'.")
|
|
),
|
|
ACCESS_LEVEL: properties.Schema(
|
|
properties.Schema.STRING,
|
|
_("Governs permissions set in manila for the cluster "
|
|
"ips."),
|
|
constraints=[
|
|
constraints.AllowedValues(['rw', 'ro']),
|
|
],
|
|
default='rw'
|
|
)
|
|
}
|
|
),
|
|
support_status=support.SupportStatus(version='6.0.0'),
|
|
update_allowed=True
|
|
)
|
|
}
|
|
|
|
default_client_name = 'sahara'
|
|
|
|
physical_resource_name_limit = 50
|
|
|
|
entity = 'node_group_templates'
|
|
|
|
def translation_rules(self, props):
|
|
return [
|
|
translation.TranslationRule(
|
|
props,
|
|
translation.TranslationRule.RESOLVE,
|
|
[self.FLAVOR],
|
|
client_plugin=self.client_plugin('nova'),
|
|
finder='find_flavor_by_name_or_id'),
|
|
translation.TranslationRule(
|
|
props,
|
|
translation.TranslationRule.RESOLVE,
|
|
[self.FLOATING_IP_POOL],
|
|
client_plugin=self.client_plugin('neutron'),
|
|
finder='find_resourceid_by_name_or_id',
|
|
entity='network')
|
|
]
|
|
|
|
def _ngt_name(self, name):
|
|
if name:
|
|
return name
|
|
return re.sub('[^a-zA-Z0-9-]', '', self.physical_resource_name())
|
|
|
|
def _prepare_properties(self, props):
|
|
"""Prepares the property values."""
|
|
if self.NAME in props:
|
|
props['name'] = self._ngt_name(props[self.NAME])
|
|
if self.FLAVOR in props:
|
|
props['flavor_id'] = props.pop(self.FLAVOR)
|
|
return props
|
|
|
|
def handle_create(self):
|
|
props = dict((k, v) for k, v in six.iteritems(self.properties))
|
|
args = self._prepare_properties(props)
|
|
node_group_template = self.client().node_group_templates.create(**args)
|
|
LOG.info("Node Group Template '%s' has been created",
|
|
node_group_template.name)
|
|
self.resource_id_set(node_group_template.id)
|
|
return self.resource_id
|
|
|
|
def handle_update(self, json_snippet, tmpl_diff, prop_diff):
|
|
if prop_diff:
|
|
args = self._prepare_properties(prop_diff)
|
|
self.client().node_group_templates.update(self.resource_id, **args)
|
|
|
|
def validate(self):
|
|
res = super(SaharaNodeGroupTemplate, self).validate()
|
|
if res:
|
|
return res
|
|
pool = self.properties[self.FLOATING_IP_POOL]
|
|
if pool:
|
|
if self.is_using_neutron():
|
|
try:
|
|
self.client_plugin(
|
|
'neutron').find_resourceid_by_name_or_id('network',
|
|
pool)
|
|
except Exception as ex:
|
|
if (self.client_plugin('neutron').is_not_found(ex)
|
|
or self.client_plugin('neutron').is_no_unique(ex)):
|
|
err_msg = encodeutils.exception_to_unicode(ex)
|
|
raise exception.StackValidationFailed(message=err_msg)
|
|
raise
|
|
|
|
else:
|
|
try:
|
|
self.client('nova').floating_ip_pools.find(name=pool)
|
|
except Exception as ex:
|
|
if self.client_plugin('nova').is_not_found(ex):
|
|
err_msg = encodeutils.exception_to_unicode(ex)
|
|
raise exception.StackValidationFailed(message=err_msg)
|
|
raise
|
|
|
|
self.client_plugin().validate_hadoop_version(
|
|
self.properties[self.PLUGIN_NAME],
|
|
self.properties[self.HADOOP_VERSION]
|
|
)
|
|
|
|
# validate node processes
|
|
plugin = self.client().plugins.get_version_details(
|
|
self.properties[self.PLUGIN_NAME],
|
|
self.properties[self.HADOOP_VERSION])
|
|
allowed_processes = [item for sublist in
|
|
list(six.itervalues(plugin.node_processes))
|
|
for item in sublist]
|
|
unsupported_processes = []
|
|
for process in self.properties[self.NODE_PROCESSES]:
|
|
if process not in allowed_processes:
|
|
unsupported_processes.append(process)
|
|
if unsupported_processes:
|
|
msg = (_("Plugin %(plugin)s doesn't support the following "
|
|
"node processes: %(unsupported)s. Allowed processes are: "
|
|
"%(allowed)s") %
|
|
{'plugin': self.properties[self.PLUGIN_NAME],
|
|
'unsupported': ', '.join(unsupported_processes),
|
|
'allowed': ', '.join(allowed_processes)})
|
|
raise exception.StackValidationFailed(
|
|
path=[self.stack.t.RESOURCES,
|
|
self.name,
|
|
self.stack.t.get_section_name(rsrc_defn.PROPERTIES)],
|
|
message=msg)
|
|
|
|
def parse_live_resource_data(self, resource_properties, resource_data):
|
|
result = super(SaharaNodeGroupTemplate, self).parse_live_resource_data(
|
|
resource_properties, resource_data)
|
|
|
|
for group in result[self.SHARES] or []:
|
|
remove_keys = set(group.keys()) - set(self._SHARE_KEYS)
|
|
for key in remove_keys:
|
|
del group[key]
|
|
result[self.FLAVOR] = resource_data.get('flavor_id')
|
|
return result
|
|
|
|
|
|
class SaharaClusterTemplate(resource.Resource):
|
|
"""A resource for managing Sahara cluster templates.
|
|
|
|
A Cluster Template is designed to bring Node Group Templates together to
|
|
form a Cluster. A Cluster Template defines what Node Groups will be
|
|
included and how many instances will be created in each. Some data
|
|
processing framework configurations can not be applied to a single node,
|
|
but to a whole Cluster. A user can specify these kinds of configurations in
|
|
a Cluster Template. Sahara enables users to specify which processes should
|
|
be added to an anti-affinity group within a Cluster Template. If a process
|
|
is included into an anti-affinity group, it means that VMs where this
|
|
process is going to be launched should be scheduled to different hardware
|
|
hosts.
|
|
"""
|
|
|
|
support_status = support.SupportStatus(version='2014.2')
|
|
|
|
PROPERTIES = (
|
|
NAME, PLUGIN_NAME, HADOOP_VERSION, DESCRIPTION,
|
|
ANTI_AFFINITY, MANAGEMENT_NETWORK,
|
|
CLUSTER_CONFIGS, NODE_GROUPS, IMAGE_ID, USE_AUTOCONFIG,
|
|
SHARES
|
|
) = (
|
|
'name', 'plugin_name', 'hadoop_version', 'description',
|
|
'anti_affinity', 'neutron_management_network',
|
|
'cluster_configs', 'node_groups', 'default_image_id', 'use_autoconfig',
|
|
'shares'
|
|
)
|
|
|
|
_NODE_GROUP_KEYS = (
|
|
NG_NAME, COUNT, NG_TEMPLATE_ID,
|
|
) = (
|
|
'name', 'count', 'node_group_template_id',
|
|
)
|
|
|
|
_SHARE_KEYS = (
|
|
SHARE_ID, PATH, ACCESS_LEVEL
|
|
) = (
|
|
'id', 'path', 'access_level'
|
|
)
|
|
|
|
properties_schema = {
|
|
NAME: properties.Schema(
|
|
properties.Schema.STRING,
|
|
_("Name for the Sahara Cluster Template."),
|
|
constraints=[
|
|
constraints.Length(min=1, max=50),
|
|
constraints.AllowedPattern(SAHARA_NAME_REGEX),
|
|
],
|
|
update_allowed=True
|
|
),
|
|
DESCRIPTION: properties.Schema(
|
|
properties.Schema.STRING,
|
|
_('Description of the Sahara Group Template.'),
|
|
default="",
|
|
update_allowed=True
|
|
),
|
|
PLUGIN_NAME: properties.Schema(
|
|
properties.Schema.STRING,
|
|
_('Plugin name.'),
|
|
required=True,
|
|
constraints=[
|
|
constraints.CustomConstraint('sahara.plugin')
|
|
],
|
|
update_allowed=True
|
|
),
|
|
HADOOP_VERSION: properties.Schema(
|
|
properties.Schema.STRING,
|
|
_('Version of Hadoop running on instances.'),
|
|
required=True,
|
|
update_allowed=True
|
|
),
|
|
IMAGE_ID: properties.Schema(
|
|
properties.Schema.STRING,
|
|
_("ID of the default image to use for the template."),
|
|
constraints=[
|
|
constraints.CustomConstraint('sahara.image'),
|
|
],
|
|
update_allowed=True
|
|
),
|
|
MANAGEMENT_NETWORK: properties.Schema(
|
|
properties.Schema.STRING,
|
|
_('Name or UUID of network.'),
|
|
constraints=[
|
|
constraints.CustomConstraint('neutron.network')
|
|
],
|
|
update_allowed=True
|
|
),
|
|
ANTI_AFFINITY: properties.Schema(
|
|
properties.Schema.LIST,
|
|
_("List of processes to enable anti-affinity for."),
|
|
schema=properties.Schema(
|
|
properties.Schema.STRING,
|
|
),
|
|
update_allowed=True
|
|
),
|
|
CLUSTER_CONFIGS: properties.Schema(
|
|
properties.Schema.MAP,
|
|
_('Cluster configs dictionary.'),
|
|
update_allowed=True
|
|
),
|
|
NODE_GROUPS: properties.Schema(
|
|
properties.Schema.LIST,
|
|
_('Node groups.'),
|
|
schema=properties.Schema(
|
|
properties.Schema.MAP,
|
|
schema={
|
|
NG_NAME: properties.Schema(
|
|
properties.Schema.STRING,
|
|
_('Name of the Node group.'),
|
|
required=True
|
|
),
|
|
COUNT: properties.Schema(
|
|
properties.Schema.INTEGER,
|
|
_("Number of instances in the Node group."),
|
|
required=True,
|
|
constraints=[
|
|
constraints.Range(min=1)
|
|
]
|
|
),
|
|
NG_TEMPLATE_ID: properties.Schema(
|
|
properties.Schema.STRING,
|
|
_("ID of the Node Group Template."),
|
|
required=True
|
|
),
|
|
}
|
|
),
|
|
update_allowed=True
|
|
),
|
|
USE_AUTOCONFIG: properties.Schema(
|
|
properties.Schema.BOOLEAN,
|
|
_("Configure most important configs automatically."),
|
|
support_status=support.SupportStatus(version='5.0.0')
|
|
),
|
|
SHARES: properties.Schema(
|
|
properties.Schema.LIST,
|
|
_("List of manila shares to be mounted."),
|
|
schema=properties.Schema(
|
|
properties.Schema.MAP,
|
|
schema={
|
|
SHARE_ID: properties.Schema(
|
|
properties.Schema.STRING,
|
|
_("Id of the manila share."),
|
|
required=True
|
|
),
|
|
PATH: properties.Schema(
|
|
properties.Schema.STRING,
|
|
_("Local path on each cluster node on which to mount "
|
|
"the share. Defaults to '/mnt/{share_id}'.")
|
|
),
|
|
ACCESS_LEVEL: properties.Schema(
|
|
properties.Schema.STRING,
|
|
_("Governs permissions set in manila for the cluster "
|
|
"ips."),
|
|
constraints=[
|
|
constraints.AllowedValues(['rw', 'ro']),
|
|
],
|
|
default='rw'
|
|
)
|
|
}
|
|
),
|
|
support_status=support.SupportStatus(version='6.0.0'),
|
|
update_allowed=True
|
|
)
|
|
}
|
|
|
|
default_client_name = 'sahara'
|
|
|
|
physical_resource_name_limit = 50
|
|
|
|
entity = 'cluster_templates'
|
|
|
|
def translation_rules(self, props):
|
|
return [
|
|
translation.TranslationRule(
|
|
props,
|
|
translation.TranslationRule.RESOLVE,
|
|
[self.MANAGEMENT_NETWORK],
|
|
client_plugin=self.client_plugin('neutron'),
|
|
finder='find_resourceid_by_name_or_id',
|
|
entity='network')]
|
|
|
|
def _cluster_template_name(self, name):
|
|
if name:
|
|
return name
|
|
return re.sub('[^a-zA-Z0-9-]', '', self.physical_resource_name())
|
|
|
|
def _prepare_properties(self, props):
|
|
"""Prepares the property values."""
|
|
if self.NAME in props:
|
|
props['name'] = self._cluster_template_name(props[self.NAME])
|
|
if self.MANAGEMENT_NETWORK in props:
|
|
props['net_id'] = props.pop(self.MANAGEMENT_NETWORK)
|
|
return props
|
|
|
|
def handle_create(self):
|
|
props = dict((k, v) for k, v in six.iteritems(self.properties))
|
|
args = self._prepare_properties(props)
|
|
cluster_template = self.client().cluster_templates.create(**args)
|
|
LOG.info("Cluster Template '%s' has been created",
|
|
cluster_template.name)
|
|
self.resource_id_set(cluster_template.id)
|
|
return self.resource_id
|
|
|
|
def handle_update(self, json_snippet, tmpl_diff, prop_diff):
|
|
if prop_diff:
|
|
args = self._prepare_properties(prop_diff)
|
|
self.client().cluster_templates.update(self.resource_id, **args)
|
|
|
|
def validate(self):
|
|
res = super(SaharaClusterTemplate, self).validate()
|
|
if res:
|
|
return res
|
|
# check if running on neutron and MANAGEMENT_NETWORK missing
|
|
if (self.is_using_neutron() and
|
|
not self.properties[self.MANAGEMENT_NETWORK]):
|
|
msg = _("%s must be provided"
|
|
) % self.MANAGEMENT_NETWORK
|
|
raise exception.StackValidationFailed(message=msg)
|
|
|
|
self.client_plugin().validate_hadoop_version(
|
|
self.properties[self.PLUGIN_NAME],
|
|
self.properties[self.HADOOP_VERSION]
|
|
)
|
|
|
|
def parse_live_resource_data(self, resource_properties, resource_data):
|
|
result = super(SaharaClusterTemplate, self).parse_live_resource_data(
|
|
resource_properties, resource_data)
|
|
|
|
for group in result[self.NODE_GROUPS] or []:
|
|
remove_keys = set(group.keys()) - set(self._NODE_GROUP_KEYS)
|
|
for key in remove_keys:
|
|
del group[key]
|
|
for group in result[self.SHARES] or []:
|
|
remove_keys = set(group.keys()) - set(self._SHARE_KEYS)
|
|
for key in remove_keys:
|
|
del group[key]
|
|
return result
|
|
|
|
|
|
def resource_mapping():
|
|
return {
|
|
'OS::Sahara::NodeGroupTemplate': SaharaNodeGroupTemplate,
|
|
'OS::Sahara::ClusterTemplate': SaharaClusterTemplate,
|
|
}
|