Sahara aims to provide users with simple means to provision a Hadoop cluster by specifying several parameters like Hadoop version, cluster topology, nodes hardware details and a few more.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

v10.py 8.9KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288
  1. # Copyright (c) 2013 Mirantis Inc.
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License");
  4. # you may not use this file except in compliance with the License.
  5. # You may obtain a copy of the License at
  6. #
  7. # http://www.apache.org/licenses/LICENSE-2.0
  8. #
  9. # Unless required by applicable law or agreed to in writing, software
  10. # distributed under the License is distributed on an "AS IS" BASIS,
  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  12. # implied.
  13. # See the License for the specific language governing permissions and
  14. # limitations under the License.
  15. from oslo_config import cfg
  16. from oslo_utils import excutils
  17. import six
  18. from sahara import conductor as c
  19. from sahara import context
  20. from sahara.plugins import base as plugin_base
  21. from sahara.service import api
  22. from sahara.service.health import verification_base
  23. from sahara.service import quotas
  24. from sahara.utils import cluster as c_u
  25. from sahara.utils import general as g
  26. from sahara.utils.notification import sender
  27. from sahara.utils.openstack import base as b
  28. from sahara.utils.openstack import images as sahara_images
  29. conductor = c.API
  30. CONF = cfg.CONF
  31. # Cluster ops
  32. def get_clusters(**kwargs):
  33. return conductor.cluster_get_all(context.ctx(),
  34. regex_search=True, **kwargs)
  35. def get_cluster(id, show_progress=False):
  36. return conductor.cluster_get(context.ctx(), id, show_progress)
  37. def scale_cluster(id, data):
  38. context.set_current_cluster_id(id)
  39. ctx = context.ctx()
  40. cluster = conductor.cluster_get(ctx, id)
  41. plugin = plugin_base.PLUGINS.get_plugin(cluster.plugin_name)
  42. existing_node_groups = data.get('resize_node_groups', [])
  43. additional_node_groups = data.get('add_node_groups', [])
  44. # the next map is the main object we will work with
  45. # to_be_enlarged : {node_group_id: desired_amount_of_instances}
  46. to_be_enlarged = {}
  47. for ng in existing_node_groups:
  48. ng_id = g.find(cluster.node_groups, name=ng['name'])['id']
  49. to_be_enlarged.update({ng_id: ng['count']})
  50. additional = construct_ngs_for_scaling(cluster, additional_node_groups)
  51. cluster = conductor.cluster_get(ctx, cluster)
  52. _add_ports_for_auto_sg(ctx, cluster, plugin)
  53. try:
  54. cluster = c_u.change_cluster_status(
  55. cluster, c_u.CLUSTER_STATUS_VALIDATING)
  56. quotas.check_scaling(cluster, to_be_enlarged, additional)
  57. plugin.recommend_configs(cluster, scaling=True)
  58. plugin.validate_scaling(cluster, to_be_enlarged, additional)
  59. except Exception as e:
  60. with excutils.save_and_reraise_exception():
  61. c_u.clean_cluster_from_empty_ng(cluster)
  62. c_u.change_cluster_status(
  63. cluster, c_u.CLUSTER_STATUS_ACTIVE, six.text_type(e))
  64. # If we are here validation is successful.
  65. # So let's update to_be_enlarged map:
  66. to_be_enlarged.update(additional)
  67. for node_group in cluster.node_groups:
  68. if node_group.id not in to_be_enlarged:
  69. to_be_enlarged[node_group.id] = node_group.count
  70. api.OPS.provision_scaled_cluster(id, to_be_enlarged)
  71. return cluster
  72. def create_cluster(values):
  73. plugin = plugin_base.PLUGINS.get_plugin(values['plugin_name'])
  74. return _cluster_create(values, plugin)
  75. def create_multiple_clusters(values):
  76. num_of_clusters = values['count']
  77. clusters = []
  78. plugin = plugin_base.PLUGINS.get_plugin(values['plugin_name'])
  79. for counter in range(num_of_clusters):
  80. cluster_dict = values.copy()
  81. cluster_name = cluster_dict['name']
  82. cluster_dict['name'] = get_multiple_cluster_name(num_of_clusters,
  83. cluster_name,
  84. counter + 1)
  85. cluster = _cluster_create(cluster_dict, plugin)
  86. clusters.append(cluster.id)
  87. clusters_dict = {'clusters': clusters}
  88. return clusters_dict
  89. def _cluster_create(values, plugin):
  90. ctx = context.ctx()
  91. cluster = conductor.cluster_create(ctx, values)
  92. context.set_current_cluster_id(cluster.id)
  93. sender.status_notify(cluster.id, cluster.name, "New",
  94. "create")
  95. _add_ports_for_auto_sg(ctx, cluster, plugin)
  96. # validating cluster
  97. try:
  98. plugin.recommend_configs(cluster)
  99. cluster = c_u.change_cluster_status(
  100. cluster, c_u.CLUSTER_STATUS_VALIDATING)
  101. quotas.check_cluster(cluster)
  102. plugin.validate(cluster)
  103. except Exception as e:
  104. with excutils.save_and_reraise_exception():
  105. c_u.change_cluster_status(
  106. cluster, c_u.CLUSTER_STATUS_ERROR, six.text_type(e))
  107. api.OPS.provision_cluster(cluster.id)
  108. return cluster
  109. def get_multiple_cluster_name(num_of_clusters, name, counter):
  110. return "%%s-%%0%dd" % len(str(num_of_clusters)) % (name, counter)
  111. def _add_ports_for_auto_sg(ctx, cluster, plugin):
  112. for ng in cluster.node_groups:
  113. if ng.auto_security_group:
  114. ports = {'open_ports': plugin.get_open_ports(ng)}
  115. conductor.node_group_update(ctx, ng, ports)
  116. def terminate_cluster(id):
  117. context.set_current_cluster_id(id)
  118. cluster = c_u.change_cluster_status(id, c_u.CLUSTER_STATUS_DELETING)
  119. if cluster is None:
  120. return
  121. api.OPS.terminate_cluster(id)
  122. sender.status_notify(cluster.id, cluster.name, cluster.status,
  123. "delete")
  124. def update_cluster(id, values):
  125. if verification_base.update_verification_required(values):
  126. api.OPS.handle_verification(id, values)
  127. return conductor.cluster_get(context.ctx(), id)
  128. return conductor.cluster_update(context.ctx(), id, values)
  129. # ClusterTemplate ops
  130. def get_cluster_templates(**kwargs):
  131. return conductor.cluster_template_get_all(context.ctx(),
  132. regex_search=True, **kwargs)
  133. def get_cluster_template(id):
  134. return conductor.cluster_template_get(context.ctx(), id)
  135. def create_cluster_template(values):
  136. return conductor.cluster_template_create(context.ctx(), values)
  137. def terminate_cluster_template(id):
  138. return conductor.cluster_template_destroy(context.ctx(), id)
  139. def update_cluster_template(id, values):
  140. return conductor.cluster_template_update(context.ctx(), id, values)
  141. # NodeGroupTemplate ops
  142. def get_node_group_templates(**kwargs):
  143. return conductor.node_group_template_get_all(context.ctx(),
  144. regex_search=True, **kwargs)
  145. def get_node_group_template(id):
  146. return conductor.node_group_template_get(context.ctx(), id)
  147. def create_node_group_template(values):
  148. return conductor.node_group_template_create(context.ctx(), values)
  149. def terminate_node_group_template(id):
  150. return conductor.node_group_template_destroy(context.ctx(), id)
  151. def update_node_group_template(id, values):
  152. return conductor.node_group_template_update(context.ctx(), id, values)
  153. def export_node_group_template(id):
  154. return conductor.node_group_template_get(context.ctx(), id)
  155. # Plugins ops
  156. def get_plugins():
  157. return plugin_base.PLUGINS.get_plugins(serialized=True)
  158. def get_plugin(plugin_name, version=None):
  159. return plugin_base.PLUGINS.serialize_plugin(plugin_name, version)
  160. def update_plugin(plugin_name, values):
  161. return plugin_base.PLUGINS.update_plugin(plugin_name, values)
  162. def construct_ngs_for_scaling(cluster, additional_node_groups):
  163. ctx = context.ctx()
  164. additional = {}
  165. for ng in additional_node_groups:
  166. count = ng['count']
  167. ng['count'] = 0
  168. ng_id = conductor.node_group_add(ctx, cluster, ng)
  169. additional.update({ng_id: count})
  170. return additional
  171. # Image Registry
  172. def get_images(name, tags):
  173. return b.execute_with_retries(
  174. sahara_images.image_manager().list_registered, name, tags)
  175. def get_image(**kwargs):
  176. if len(kwargs) == 1 and 'id' in kwargs:
  177. return b.execute_with_retries(
  178. sahara_images.image_manager().get, kwargs['id'])
  179. else:
  180. return b.execute_with_retries(
  181. sahara_images.image_manager().find, **kwargs)
  182. def get_registered_image(image_id):
  183. return b.execute_with_retries(
  184. sahara_images.image_manager().get_registered_image, image_id)
  185. def register_image(image_id, username, description=None):
  186. manager = sahara_images.image_manager()
  187. b.execute_with_retries(
  188. manager.set_image_info, image_id, username, description)
  189. return b.execute_with_retries(manager.get, image_id)
  190. def unregister_image(image_id):
  191. manager = sahara_images.image_manager()
  192. b.execute_with_retries(manager.unset_image_info, image_id)
  193. return b.execute_with_retries(manager.get, image_id)
  194. def add_image_tags(image_id, tags):
  195. manager = sahara_images.image_manager()
  196. b.execute_with_retries(manager.tag, image_id, tags)
  197. return b.execute_with_retries(manager.get, image_id)
  198. def remove_image_tags(image_id, tags):
  199. manager = sahara_images.image_manager()
  200. b.execute_with_retries(manager.untag, image_id, tags)
  201. return b.execute_with_retries(manager.get, image_id)