ef3815988e
This change renames the service api modules to be organized under the sahara.service.api package. A v2 package has been added which holds the equivalent v10 and v11 functions. To make the api version setup transparent to the caller, the global OPS variable has been refactored into the base sahara.service.api package. Changes * create sahara/service/api package * rename sahara/service/api.py to sahara/service/api/v10.py * rename sahara/service/edp/api.py to sahara/service/api/v11.py * correct occurances of old imports in code and tests * rename sahara/tests/unit/service/test_api.py to sahara/tests/unit/service/api/test_v10.py * add initial v2 equivalents for current api services * move global OPS object into the sahara.service.api package * add documentation for the api service layer Partial-Implements: bp v2-api-experimental-impl Depends-On: I16918a30a862b42edd7a982caf555be618199ac3 Change-Id: Iefbedbc76ac620ff012bcaf536c17637b6252a15
398 lines
14 KiB
Python
398 lines
14 KiB
Python
# Copyright (c) 2013 Mirantis Inc.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
# implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
import collections
|
|
|
|
import novaclient.exceptions as nova_ex
|
|
from oslo_config import cfg
|
|
import six
|
|
|
|
from sahara import conductor as cond
|
|
from sahara import context
|
|
import sahara.exceptions as ex
|
|
from sahara.i18n import _
|
|
import sahara.plugins.base as plugin_base
|
|
from sahara.service.api import v10 as api
|
|
from sahara.utils import general as g
|
|
import sahara.utils.openstack.cinder as cinder
|
|
import sahara.utils.openstack.nova as nova
|
|
|
|
|
|
CONF = cfg.CONF
|
|
conductor = cond.API
|
|
|
|
MAX_HOSTNAME_LENGTH = 64
|
|
|
|
|
|
def _get_plugin_configs(plugin_name, hadoop_version, scope=None):
|
|
pl_confs = {}
|
|
for config in plugin_base.PLUGINS.get_plugin(
|
|
plugin_name).get_all_configs(hadoop_version):
|
|
if pl_confs.get(config.applicable_target):
|
|
pl_confs[config.applicable_target].append(config.name)
|
|
else:
|
|
pl_confs[config.applicable_target] = [config.name]
|
|
return pl_confs
|
|
|
|
|
|
def _check_duplicates(lst, message):
|
|
invalid = []
|
|
lst = collections.Counter(lst)
|
|
for key, value in six.iteritems(lst):
|
|
if value > 1:
|
|
invalid.append(key)
|
|
|
|
if len(invalid) > 0:
|
|
raise ex.InvalidDataException(message % invalid)
|
|
|
|
|
|
# Common validation checks
|
|
|
|
def check_plugin_name_exists(name):
|
|
if name not in [p.name for p in api.get_plugins()]:
|
|
raise ex.InvalidReferenceException(
|
|
_("Sahara doesn't contain plugin with name '%s'") % name)
|
|
|
|
|
|
def check_plugin_supports_version(p_name, version):
|
|
if version not in plugin_base.PLUGINS.get_plugin(p_name).get_versions():
|
|
raise ex.InvalidReferenceException(
|
|
_("Requested plugin '%(name)s' doesn't support version "
|
|
"'%(version)s'") % {'name': p_name, 'version': version})
|
|
|
|
|
|
def check_image_registered(image_id):
|
|
if image_id not in [i.id for i in nova.client().images.list_registered()]:
|
|
raise ex.InvalidReferenceException(
|
|
_("Requested image '%s' is not registered") % image_id)
|
|
|
|
|
|
def check_node_group_configs(plugin_name, hadoop_version, ng_configs,
|
|
plugin_configs=None):
|
|
# TODO(aignatov): Should have scope and config type validations
|
|
pl_confs = plugin_configs or _get_plugin_configs(plugin_name,
|
|
hadoop_version)
|
|
for app_target, configs in ng_configs.items():
|
|
if app_target not in pl_confs:
|
|
raise ex.InvalidReferenceException(
|
|
_("Plugin doesn't contain applicable target '%s'")
|
|
% app_target)
|
|
for name, values in configs.items():
|
|
if name not in pl_confs[app_target]:
|
|
raise ex.InvalidReferenceException(
|
|
_("Plugin's applicable target '%(target)s' doesn't "
|
|
"contain config with name '%(name)s'") %
|
|
{'target': app_target, 'name': name})
|
|
|
|
|
|
def check_all_configurations(data):
|
|
pl_confs = _get_plugin_configs(data['plugin_name'], data['hadoop_version'])
|
|
|
|
if data.get('cluster_configs'):
|
|
check_node_group_configs(data['plugin_name'], data['hadoop_version'],
|
|
data['cluster_configs'],
|
|
plugin_configs=pl_confs)
|
|
|
|
if data.get('node_groups'):
|
|
check_duplicates_node_groups_names(data['node_groups'])
|
|
for ng in data['node_groups']:
|
|
check_node_group_basic_fields(data['plugin_name'],
|
|
data['hadoop_version'],
|
|
ng, pl_confs)
|
|
|
|
# NodeGroup related checks
|
|
|
|
|
|
def check_node_group_basic_fields(plugin_name, hadoop_version, ng,
|
|
plugin_configs=None):
|
|
|
|
if ng.get('node_group_template_id'):
|
|
ng_tmpl_id = ng['node_group_template_id']
|
|
check_node_group_template_exists(ng_tmpl_id)
|
|
ng_tmpl = api.get_node_group_template(ng_tmpl_id).to_wrapped_dict()
|
|
check_node_group_basic_fields(plugin_name, hadoop_version,
|
|
ng_tmpl['node_group_template'],
|
|
plugin_configs)
|
|
|
|
if ng.get('node_configs'):
|
|
check_node_group_configs(plugin_name, hadoop_version,
|
|
ng['node_configs'], plugin_configs)
|
|
if ng.get('flavor_id'):
|
|
check_flavor_exists(ng['flavor_id'])
|
|
|
|
if ng.get('node_processes'):
|
|
check_node_processes(plugin_name, hadoop_version, ng['node_processes'])
|
|
|
|
if ng.get('image_id'):
|
|
check_image_registered(ng['image_id'])
|
|
|
|
if ng.get('volumes_per_node'):
|
|
if not cinder.check_cinder_exists():
|
|
raise ex.InvalidReferenceException(_("Cinder is not supported"))
|
|
|
|
if ng.get('volumes_availability_zone'):
|
|
check_volume_availability_zone_exist(
|
|
ng['volumes_availability_zone'])
|
|
|
|
if ng.get('volume_type'):
|
|
check_volume_type_exists(ng['volume_type'])
|
|
|
|
if not ng.get('volumes_size'):
|
|
raise ex.InvalidReferenceException(
|
|
_("You must specify a volumes_size parameter"))
|
|
|
|
if ng.get('floating_ip_pool'):
|
|
check_floatingip_pool_exists(ng['name'], ng['floating_ip_pool'])
|
|
|
|
if ng.get('security_groups'):
|
|
check_security_groups_exist(ng['security_groups'])
|
|
|
|
if ng.get('availability_zone'):
|
|
check_availability_zone_exist(ng['availability_zone'])
|
|
|
|
|
|
def check_flavor_exists(flavor_id):
|
|
flavor_list = nova.client().flavors.list()
|
|
if flavor_id not in [flavor.id for flavor in flavor_list]:
|
|
raise ex.NotFoundException(
|
|
flavor_id, _("Requested flavor '%s' not found"))
|
|
|
|
|
|
def check_security_groups_exist(security_groups):
|
|
security_group_list = nova.client().security_groups.list()
|
|
allowed_groups = set()
|
|
for sg in security_group_list:
|
|
allowed_groups.add(six.text_type(sg.id))
|
|
allowed_groups.add(sg.name)
|
|
|
|
for sg in security_groups:
|
|
if sg not in allowed_groups:
|
|
raise ex.NotFoundException(
|
|
sg, _("Security group '%s' not found"))
|
|
|
|
|
|
def check_floatingip_pool_exists(ng_name, pool_id):
|
|
network = None
|
|
if CONF.use_neutron:
|
|
network = nova.get_network(id=pool_id)
|
|
else:
|
|
for net in nova.client().floating_ip_pools.list():
|
|
if net.name == pool_id:
|
|
network = net.name
|
|
break
|
|
|
|
if not network:
|
|
raise ex.NotFoundException(pool_id, _("Floating IP pool %s not found"))
|
|
|
|
|
|
def check_node_processes(plugin_name, version, node_processes):
|
|
_check_duplicates(node_processes, _("Duplicates in node processes have "
|
|
"been detected: %s"))
|
|
|
|
plugin_processes = []
|
|
for process in plugin_base.PLUGINS.get_plugin(
|
|
plugin_name).get_node_processes(version).values():
|
|
plugin_processes += process
|
|
plugin_processes = set(plugin_processes)
|
|
|
|
invalid_processes = []
|
|
for node_process in node_processes:
|
|
if node_process not in plugin_processes:
|
|
invalid_processes.append(node_process)
|
|
|
|
if len(invalid_processes) > 0:
|
|
raise ex.InvalidReferenceException(
|
|
_("Plugin doesn't support the following node processes: %s")
|
|
% sorted(invalid_processes))
|
|
|
|
|
|
def check_duplicates_node_groups_names(node_groups):
|
|
ng_names = [ng['name'] for ng in node_groups]
|
|
_check_duplicates(
|
|
ng_names, _("Duplicates in node group names are detected: %s"))
|
|
|
|
|
|
def check_availability_zone_exist(az):
|
|
az_list = nova.client().availability_zones.list(False)
|
|
az_names = [a.zoneName for a in az_list]
|
|
if az not in az_names:
|
|
raise ex.NotFoundException(
|
|
az, _("Nova availability zone '%s' not found"))
|
|
|
|
|
|
def check_volume_availability_zone_exist(az):
|
|
az_list = cinder.client().availability_zones.list()
|
|
az_names = [a.zoneName for a in az_list]
|
|
if az not in az_names:
|
|
raise ex.NotFoundException(
|
|
az, _("Cinder availability zone '%s' not found"))
|
|
|
|
|
|
def check_volume_type_exists(volume_type):
|
|
volume_types = cinder.client().volume_types.list()
|
|
volume_types = list(filter(lambda x: x.name == volume_type, volume_types))
|
|
if len(volume_types) == 1 and volume_types[0].name == volume_type:
|
|
return
|
|
raise ex.NotFoundException(volume_type, _("Volume type '%s' not found"))
|
|
|
|
|
|
# Cluster creation related checks
|
|
|
|
def check_cluster_unique_name(name):
|
|
if name in [cluster.name for cluster in api.get_clusters()]:
|
|
raise ex.NameAlreadyExistsException(
|
|
_("Cluster with name '%s' already exists") % name)
|
|
|
|
|
|
def check_cluster_hostnames_lengths(cluster_name, node_groups):
|
|
for ng in node_groups:
|
|
longest_hostname = g.generate_instance_name(cluster_name,
|
|
ng['name'], ng['count'])
|
|
longest_hostname += '.'
|
|
longest_hostname += CONF.node_domain
|
|
if len(longest_hostname) > MAX_HOSTNAME_LENGTH:
|
|
raise ex.InvalidDataException(
|
|
_("Composite hostname %(host)s in provisioned cluster exceeds"
|
|
" maximum limit %(limit)s characters") %
|
|
{'host': longest_hostname,
|
|
'limit': MAX_HOSTNAME_LENGTH})
|
|
|
|
|
|
def check_keypair_exists(keypair):
|
|
try:
|
|
nova.client().keypairs.get(keypair)
|
|
except nova_ex.NotFound:
|
|
raise ex.NotFoundException(
|
|
keypair, _("Requested keypair '%s' not found"))
|
|
|
|
|
|
def check_network_exists(net_id):
|
|
if not nova.get_network(id=net_id):
|
|
raise ex.NotFoundException(net_id, _("Network %s not found"))
|
|
|
|
|
|
# Cluster templates related checks
|
|
|
|
def check_cluster_template_unique_name(name):
|
|
if name in [t.name for t in api.get_cluster_templates()]:
|
|
raise ex.NameAlreadyExistsException(
|
|
_("Cluster template with name '%s' already exists") % name)
|
|
|
|
|
|
def check_cluster_template_exists(cluster_template_id):
|
|
if not api.get_cluster_template(id=cluster_template_id):
|
|
raise ex.NotFoundException(
|
|
cluster_template_id,
|
|
_("Cluster template with id '%s' not found"))
|
|
|
|
|
|
def check_node_groups_in_cluster_templates(cluster_name, plugin_name,
|
|
hadoop_version,
|
|
cluster_template_id):
|
|
c_t = api.get_cluster_template(id=cluster_template_id)
|
|
n_groups = c_t.to_wrapped_dict()['cluster_template']['node_groups']
|
|
proxy_gateway_used = len([ng for ng in n_groups if
|
|
ng.get('is_proxy_gateway', False)]) > 0
|
|
check_network_config(n_groups, proxy_gateway_used)
|
|
for node_group in n_groups:
|
|
check_node_group_basic_fields(plugin_name, hadoop_version, node_group)
|
|
check_cluster_hostnames_lengths(cluster_name, n_groups)
|
|
|
|
# NodeGroup templates related checks
|
|
|
|
|
|
def check_node_group_template_unique_name(name):
|
|
if name in [t.name for t in api.get_node_group_templates()]:
|
|
raise ex.NameAlreadyExistsException(
|
|
_("NodeGroup template with name '%s' already exists") % name)
|
|
|
|
|
|
def check_node_group_template_exists(ng_tmpl_id):
|
|
if not api.get_node_group_template(id=ng_tmpl_id):
|
|
raise ex.NotFoundException(
|
|
ng_tmpl_id, _("NodeGroup template with id '%s' not found"))
|
|
|
|
|
|
def check_network_config(node_groups, proxy_gateway_used=False):
|
|
if CONF.use_floating_ips and CONF.use_neutron:
|
|
for ng in node_groups:
|
|
require_floating = True
|
|
if proxy_gateway_used:
|
|
require_floating = ng.get('is_proxy_gateway', False)
|
|
|
|
if require_floating and not _get_floating_ip_pool(ng):
|
|
raise ex.MissingFloatingNetworkException(ng.get('name'))
|
|
|
|
|
|
def _get_floating_ip_pool(node_group):
|
|
if node_group.get('floating_ip_pool'):
|
|
return node_group['floating_ip_pool']
|
|
|
|
if node_group.get('node_group_template_id'):
|
|
ctx = context.ctx()
|
|
ngt = conductor.node_group_template_get(
|
|
ctx,
|
|
node_group['node_group_template_id'])
|
|
if ngt.get('floating_ip_pool'):
|
|
return ngt['floating_ip_pool']
|
|
|
|
return None
|
|
|
|
|
|
# Cluster scaling
|
|
|
|
def check_resize(cluster, r_node_groups):
|
|
cluster_ng_names = [ng.name for ng in cluster.node_groups]
|
|
|
|
check_duplicates_node_groups_names(r_node_groups)
|
|
|
|
for ng in r_node_groups:
|
|
if ng['name'] not in cluster_ng_names:
|
|
raise ex.InvalidReferenceException(
|
|
_("Cluster doesn't contain node group with name '%s'")
|
|
% ng['name'])
|
|
|
|
|
|
def check_add_node_groups(cluster, add_node_groups):
|
|
cluster_ng_names = [ng.name for ng in cluster.node_groups]
|
|
|
|
check_duplicates_node_groups_names(add_node_groups)
|
|
|
|
pl_confs = _get_plugin_configs(cluster.plugin_name, cluster.hadoop_version)
|
|
|
|
for ng in add_node_groups:
|
|
if ng['name'] in cluster_ng_names:
|
|
raise ex.InvalidReferenceException(
|
|
_("Can't add new nodegroup. Cluster already has nodegroup with"
|
|
" name '%s'") % ng['name'])
|
|
|
|
check_node_group_basic_fields(cluster.plugin_name,
|
|
cluster.hadoop_version, ng, pl_confs)
|
|
|
|
|
|
# Tags
|
|
|
|
|
|
def check_required_image_tags(plugin_name, hadoop_version, image_id):
|
|
image = api.get_image(id=image_id)
|
|
plugin = plugin_base.PLUGINS.get_plugin(plugin_name)
|
|
req_tags = set(plugin.get_required_image_tags(hadoop_version))
|
|
req_tags = list(req_tags.difference(set(image.tags)))
|
|
if req_tags:
|
|
raise ex.InvalidReferenceException(
|
|
_("Requested image '%(image)s' doesn't contain required"
|
|
" tags: %(tags)s")
|
|
% {'image': image_id, 'tags': req_tags})
|