Sahara Horizon plugin.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

create.py 24KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619
  1. # Licensed under the Apache License, Version 2.0 (the "License");
  2. # you may not use this file except in compliance with the License.
  3. # You may obtain a copy of the License at
  4. #
  5. # http://www.apache.org/licenses/LICENSE-2.0
  6. #
  7. # Unless required by applicable law or agreed to in writing, software
  8. # distributed under the License is distributed on an "AS IS" BASIS,
  9. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  10. # implied.
  11. # See the License for the specific language governing permissions and
  12. # limitations under the License.
  13. import itertools
  14. import uuid
  15. from django.utils import encoding
  16. from django.utils import html
  17. from django.utils import safestring
  18. from django.utils.translation import ugettext_lazy as _
  19. from oslo_log import log as logging
  20. from saharaclient.api import base as api_base
  21. from horizon import exceptions
  22. from horizon import forms
  23. from horizon import workflows
  24. from openstack_dashboard.api import cinder
  25. from openstack_dashboard.api import network
  26. from openstack_dashboard.dashboards.project.instances \
  27. import utils as nova_utils
  28. from openstack_dashboard.dashboards.project.volumes \
  29. import utils as cinder_utils
  30. from sahara_dashboard.api import manila as manilaclient
  31. from sahara_dashboard.api import sahara as saharaclient
  32. from sahara_dashboard.content.data_processing.utils \
  33. import acl as acl_utils
  34. from sahara_dashboard.content.data_processing.utils \
  35. import helpers
  36. from sahara_dashboard.content.data_processing.utils \
  37. import workflow_helpers
  38. LOG = logging.getLogger(__name__)
  39. BASE_IMAGE_URL = "horizon:project:data_processing.clusters:register"
  40. def is_cinder_enabled(request):
  41. for service in ['volumev3', 'volumev2', 'volume']:
  42. if saharaclient.base.is_service_enabled(request, service):
  43. return True
  44. return False
  45. def storage_choices(request):
  46. choices = [("ephemeral_drive", _("Ephemeral Drive")), ]
  47. if is_cinder_enabled(request):
  48. choices.append(("cinder_volume", _("Cinder Volume")))
  49. else:
  50. LOG.warning(_("Cinder service is unavailable now"))
  51. return choices
  52. class GeneralConfigAction(workflows.Action):
  53. nodegroup_name = forms.CharField(label=_("Template Name"))
  54. description = forms.CharField(label=_("Description"),
  55. required=False,
  56. widget=forms.Textarea(attrs={'rows': 4}))
  57. flavor = forms.ChoiceField(label=_("OpenStack Flavor"))
  58. availability_zone = forms.ChoiceField(
  59. label=_("Availability Zone"),
  60. help_text=_("Launch instances in this availability zone."),
  61. required=False,
  62. widget=forms.Select(attrs={"class": "availability_zone_field"})
  63. )
  64. storage = forms.ChoiceField(
  65. label=_("Storage location"),
  66. help_text=_("Choose a storage location"),
  67. choices=[],
  68. widget=forms.Select(attrs={
  69. "class": "storage_field switchable",
  70. 'data-slug': 'storage_loc'
  71. }))
  72. volumes_per_node = forms.IntegerField(
  73. label=_("Volumes per node"),
  74. required=False,
  75. initial=1,
  76. widget=forms.TextInput(attrs={
  77. "class": "volume_per_node_field switched",
  78. "data-switch-on": "storage_loc",
  79. "data-storage_loc-cinder_volume": _('Volumes per node')
  80. })
  81. )
  82. volumes_size = forms.IntegerField(
  83. label=_("Volumes size (GB)"),
  84. required=False,
  85. initial=10,
  86. widget=forms.TextInput(attrs={
  87. "class": "volume_size_field switched",
  88. "data-switch-on": "storage_loc",
  89. "data-storage_loc-cinder_volume": _('Volumes size (GB)')
  90. })
  91. )
  92. volume_type = forms.ChoiceField(
  93. label=_("Volumes type"),
  94. required=False,
  95. widget=forms.Select(attrs={
  96. "class": "volume_type_field switched",
  97. "data-switch-on": "storage_loc",
  98. "data-storage_loc-cinder_volume": _('Volumes type')
  99. })
  100. )
  101. volume_local_to_instance = forms.BooleanField(
  102. label=_("Volume local to instance"),
  103. required=False,
  104. help_text=_("Instance and attached volumes will be created on the "
  105. "same physical host"),
  106. widget=forms.CheckboxInput(attrs={
  107. "class": "volume_local_to_instance_field switched",
  108. "data-switch-on": "storage_loc",
  109. "data-storage_loc-cinder_volume": _('Volume local to instance')
  110. })
  111. )
  112. volumes_availability_zone = forms.ChoiceField(
  113. label=_("Volumes Availability Zone"),
  114. help_text=_("Create volumes in this availability zone."),
  115. required=False,
  116. widget=forms.Select(attrs={
  117. "class": "volumes_availability_zone_field switched",
  118. "data-switch-on": "storage_loc",
  119. "data-storage_loc-cinder_volume": _('Volumes Availability Zone')
  120. })
  121. )
  122. image = forms.DynamicChoiceField(label=_("Base Image"),
  123. required=False,
  124. add_item_link=BASE_IMAGE_URL)
  125. hidden_configure_field = forms.CharField(
  126. required=False,
  127. widget=forms.HiddenInput(attrs={"class": "hidden_configure_field"}))
  128. def __init__(self, request, *args, **kwargs):
  129. super(GeneralConfigAction, self).__init__(request, *args, **kwargs)
  130. hlps = helpers.Helpers(request)
  131. plugin, hadoop_version = (
  132. workflow_helpers.get_plugin_and_hadoop_version(request))
  133. if not saharaclient.SAHARA_AUTO_IP_ALLOCATION_ENABLED:
  134. pools = network.floating_ip_pools_list(request)
  135. pool_choices = [(pool.id, pool.name) for pool in pools]
  136. pool_choices.insert(0, (None, "Do not assign floating IPs"))
  137. self.fields['floating_ip_pool'] = forms.ChoiceField(
  138. label=_("Floating IP Pool"),
  139. choices=pool_choices,
  140. required=False)
  141. self.fields["use_autoconfig"] = forms.BooleanField(
  142. label=_("Auto-configure"),
  143. help_text=_("If selected, instances of a node group will be "
  144. "automatically configured during cluster "
  145. "creation. Otherwise you should manually specify "
  146. "configuration values."),
  147. required=False,
  148. widget=forms.CheckboxInput(),
  149. initial=True,
  150. )
  151. self.fields["proxygateway"] = forms.BooleanField(
  152. label=_("Proxy Gateway"),
  153. widget=forms.CheckboxInput(),
  154. help_text=_("Sahara will use instances of this node group to "
  155. "access other cluster instances."),
  156. required=False)
  157. self.fields['is_public'] = acl_utils.get_is_public_form(
  158. _("node group template"))
  159. self.fields['is_protected'] = acl_utils.get_is_protected_form(
  160. _("node group template"))
  161. self.fields["plugin_name"] = forms.CharField(
  162. widget=forms.HiddenInput(),
  163. initial=plugin
  164. )
  165. self.fields["hadoop_version"] = forms.CharField(
  166. widget=forms.HiddenInput(),
  167. initial=hadoop_version
  168. )
  169. self.fields["storage"].choices = storage_choices(request)
  170. node_parameters = hlps.get_general_node_group_configs(plugin,
  171. hadoop_version)
  172. for param in node_parameters:
  173. self.fields[param.name] = workflow_helpers.build_control(param)
  174. # when we copy or edit a node group template then
  175. # request contains valuable info in both GET and POST methods
  176. req = request.GET.copy()
  177. req.update(request.POST)
  178. if req.get("guide_template_type"):
  179. self.fields["guide_template_type"] = forms.CharField(
  180. required=False,
  181. widget=forms.HiddenInput(),
  182. initial=req.get("guide_template_type"))
  183. if is_cinder_enabled(request):
  184. volume_types = cinder.volume_type_list(request)
  185. else:
  186. volume_types = []
  187. self.fields['volume_type'].choices = [(None, _("No volume type"))] + \
  188. [(type.name, type.name)
  189. for type in volume_types]
  190. def populate_flavor_choices(self, request, context):
  191. flavors = nova_utils.flavor_list(request)
  192. if flavors:
  193. return nova_utils.sort_flavor_list(request, flavors)
  194. return []
  195. def populate_availability_zone_choices(self, request, context):
  196. # The default is None, i.e. not specifying any availability zone
  197. az_list = [(None, _('No availability zone specified'))]
  198. az_list.extend([(az.zoneName, az.zoneName)
  199. for az in nova_utils.availability_zone_list(request)
  200. if az.zoneState['available']])
  201. return az_list
  202. def populate_volumes_availability_zone_choices(self, request, context):
  203. az_list = [(None, _('No availability zone specified'))]
  204. if is_cinder_enabled(request):
  205. az_list.extend([(az.zoneName, az.zoneName)
  206. for az in cinder_utils.availability_zone_list(
  207. request) if az.zoneState['available']])
  208. return az_list
  209. def populate_image_choices(self, request, context):
  210. return workflow_helpers.populate_image_choices(self, request, context,
  211. empty_choice=True)
  212. def get_help_text(self):
  213. extra = dict()
  214. plugin_name, hadoop_version = (
  215. workflow_helpers.get_plugin_and_hadoop_version(self.request))
  216. extra["plugin_name"] = plugin_name
  217. extra["hadoop_version"] = hadoop_version
  218. plugin = saharaclient.plugin_get_version_details(
  219. self.request, plugin_name, hadoop_version)
  220. extra["deprecated"] = workflow_helpers.is_version_of_plugin_deprecated(
  221. plugin, hadoop_version)
  222. return super(GeneralConfigAction, self).get_help_text(extra)
  223. class Meta(object):
  224. name = _("Configure Node Group Template")
  225. help_text_template = "nodegroup_templates/_configure_general_help.html"
  226. class SecurityConfigAction(workflows.Action):
  227. def __init__(self, request, *args, **kwargs):
  228. super(SecurityConfigAction, self).__init__(request, *args, **kwargs)
  229. self.fields["security_autogroup"] = forms.BooleanField(
  230. label=_("Auto Security Group"),
  231. widget=forms.CheckboxInput(),
  232. help_text=_("Create security group for this Node Group."),
  233. required=False,
  234. initial=True)
  235. try:
  236. groups = network.security_group_list(request)
  237. except Exception:
  238. exceptions.handle(request,
  239. _("Unable to get security group list."))
  240. raise
  241. security_group_list = [(sg.id, sg.name) for sg in groups]
  242. self.fields["security_groups"] = forms.MultipleChoiceField(
  243. label=_("Security Groups"),
  244. widget=forms.CheckboxSelectMultiple(),
  245. help_text=_("Launch instances in these security groups."),
  246. choices=security_group_list,
  247. required=False)
  248. class Meta(object):
  249. name = _("Security")
  250. help_text = _("Control access to instances of the node group.")
  251. class CheckboxSelectMultiple(forms.CheckboxSelectMultiple):
  252. def render(self, name, value, attrs=None, choices=()):
  253. if value is None:
  254. value = []
  255. has_id = attrs and 'id' in attrs
  256. final_attrs = self.build_attrs(attrs, name=name)
  257. output = []
  258. initial_service = uuid.uuid4()
  259. str_values = set([encoding.force_text(v) for v in value])
  260. for i, (option_value, option_label) in enumerate(
  261. itertools.chain(self.choices, choices)):
  262. current_service = option_value.split(':')[0]
  263. if current_service != initial_service:
  264. if i > 0:
  265. output.append("</ul>")
  266. service_description = _("%s processes: ") % current_service
  267. service_description = html.conditional_escape(
  268. encoding.force_text(service_description))
  269. output.append("<label>%s</label>" % service_description)
  270. initial_service = current_service
  271. output.append(encoding.force_text("<ul>"))
  272. if has_id:
  273. final_attrs = dict(final_attrs, id='%s_%s' % (attrs['id'], i))
  274. label_for = ' for="%s"' % final_attrs['id']
  275. else:
  276. label_for = ''
  277. cb = forms.CheckboxInput(
  278. final_attrs, check_test=lambda value: value in str_values)
  279. option_value = encoding.force_text(option_value)
  280. rendered_cb = cb.render(name, option_value)
  281. option_label = html.conditional_escape(
  282. encoding.force_text(option_label))
  283. output.append(
  284. '<li><label%s>%s %s</label></li>' %
  285. (label_for, rendered_cb, option_label))
  286. output.append('</ul>')
  287. return safestring.mark_safe('\n'.join(output))
  288. class SelectNodeProcessesAction(workflows.Action):
  289. def __init__(self, request, *args, **kwargs):
  290. super(SelectNodeProcessesAction, self).__init__(
  291. request, *args, **kwargs)
  292. plugin, hadoop_version = (
  293. workflow_helpers.get_plugin_and_hadoop_version(request))
  294. node_processes = {}
  295. try:
  296. version_details = saharaclient.plugin_get_version_details(
  297. request, plugin, hadoop_version)
  298. node_processes = version_details.node_processes
  299. except Exception:
  300. exceptions.handle(request,
  301. _("Unable to generate process choices."))
  302. process_choices = []
  303. for service, processes in node_processes.items():
  304. for process in processes:
  305. choice_label = str(service) + ":" + str(process)
  306. process_choices.append((choice_label, process))
  307. self.fields["processes"] = forms.MultipleChoiceField(
  308. label=_("Select Node Group Processes"),
  309. widget=CheckboxSelectMultiple(),
  310. choices=process_choices,
  311. required=True)
  312. class Meta(object):
  313. name = _("Node Processes")
  314. help_text = _("Select node processes for the node group")
  315. class SelectNodeGroupSharesAction(workflows.Action):
  316. def __init__(self, request, *args, **kwargs):
  317. super(SelectNodeGroupSharesAction, self).__init__(
  318. request, *args, **kwargs)
  319. possible_shares = self.get_possible_shares(request)
  320. self.fields["shares"] = workflow_helpers.MultipleShareChoiceField(
  321. label=_("Select Shares"),
  322. widget=workflow_helpers.ShareWidget(choices=possible_shares),
  323. required=False,
  324. choices=possible_shares
  325. )
  326. def get_possible_shares(self, request):
  327. try:
  328. shares = manilaclient.share_list(request)
  329. choices = [(s.id, s.name) for s in shares]
  330. except Exception:
  331. exceptions.handle(request, _("Failed to get list of shares"))
  332. choices = []
  333. return choices
  334. class Meta(object):
  335. name = _("Shares")
  336. help_text = _("Select the manila shares for this node group")
  337. class GeneralConfig(workflows.Step):
  338. action_class = GeneralConfigAction
  339. contributes = ("general_nodegroup_name", )
  340. def contribute(self, data, context):
  341. for k, v in data.items():
  342. if "hidden" in k:
  343. continue
  344. context["general_" + k] = v if v != "None" else None
  345. return context
  346. class SecurityConfig(workflows.Step):
  347. action_class = SecurityConfigAction
  348. contributes = ("security_autogroup", "security_groups")
  349. class SelectNodeProcesses(workflows.Step):
  350. action_class = SelectNodeProcessesAction
  351. def contribute(self, data, context):
  352. post = self.workflow.request.POST
  353. context['general_processes'] = post.getlist('processes')
  354. return context
  355. class SelectNodeGroupShares(workflows.Step):
  356. action_class = SelectNodeGroupSharesAction
  357. def contribute(self, data, context):
  358. post = self.workflow.request.POST
  359. shares_details = []
  360. for index in range(0, len(self.action.fields['shares'].choices) * 3):
  361. if index % 3 == 0:
  362. share = post.get("shares_{0}".format(index))
  363. if share:
  364. path = post.get("shares_{0}".format(index + 1))
  365. permissions = post.get("shares_{0}".format(index + 2))
  366. shares_details.append({
  367. "id": share,
  368. "path": path,
  369. "access_level": permissions
  370. })
  371. context['ngt_shares'] = shares_details
  372. return context
  373. class ConfigureNodegroupTemplate(workflow_helpers.ServiceParametersWorkflow,
  374. workflow_helpers.StatusFormatMixin):
  375. slug = "configure_nodegroup_template"
  376. name = _("Create Node Group Template")
  377. finalize_button_name = _("Create")
  378. success_message = _("Created Node Group Template %s")
  379. name_property = "general_nodegroup_name"
  380. success_url = ("horizon:project:data_processing.clusters:"
  381. "nodegroup-templates-tab")
  382. default_steps = (GeneralConfig, SelectNodeProcesses, SecurityConfig, )
  383. def __init__(self, request, context_seed, entry_point, *args, **kwargs):
  384. hlps = helpers.Helpers(request)
  385. plugin, hadoop_version = (
  386. workflow_helpers.get_plugin_and_hadoop_version(request))
  387. general_parameters, service_parameters = \
  388. hlps.get_general_and_service_nodegroups_parameters(plugin,
  389. hadoop_version)
  390. if saharaclient.base.is_service_enabled(request, 'share'):
  391. ConfigureNodegroupTemplate._register_step(self,
  392. SelectNodeGroupShares)
  393. self._populate_tabs(general_parameters, service_parameters)
  394. super(ConfigureNodegroupTemplate, self).__init__(request,
  395. context_seed,
  396. entry_point,
  397. *args, **kwargs)
  398. def is_valid(self):
  399. missing = self.depends_on - set(self.context.keys())
  400. if missing:
  401. raise exceptions.WorkflowValidationError(
  402. "Unable to complete the workflow. The values %s are "
  403. "required but not present." % ", ".join(missing))
  404. checked_steps = []
  405. if "general_processes" in self.context:
  406. checked_steps = self.context["general_processes"]
  407. enabled_services = set([])
  408. for process_name in checked_steps:
  409. enabled_services.add(str(process_name).split(":")[0])
  410. steps_valid = True
  411. for step in self.steps:
  412. process_name = str(getattr(step, "process_name", None))
  413. if process_name not in enabled_services and \
  414. not isinstance(step, GeneralConfig):
  415. continue
  416. if not step.action.is_valid():
  417. steps_valid = False
  418. step.has_errors = True
  419. if not steps_valid:
  420. return steps_valid
  421. return self.validate(self.context)
  422. def handle(self, request, context):
  423. try:
  424. processes = []
  425. for service_process in context["general_processes"]:
  426. processes.append(str(service_process).split(":")[1])
  427. configs_dict = (
  428. workflow_helpers.parse_configs_from_context(
  429. context, self.defaults))
  430. plugin, hadoop_version = (
  431. workflow_helpers.get_plugin_and_hadoop_version(request))
  432. volumes_per_node = None
  433. volumes_size = None
  434. volumes_availability_zone = None
  435. volume_type = None
  436. volume_local_to_instance = False
  437. if context["general_storage"] == "cinder_volume":
  438. volumes_per_node = context["general_volumes_per_node"]
  439. volumes_size = context["general_volumes_size"]
  440. volumes_availability_zone = \
  441. context["general_volumes_availability_zone"]
  442. volume_type = context["general_volume_type"]
  443. volume_local_to_instance = \
  444. context["general_volume_local_to_instance"]
  445. ngt_shares = context.get('ngt_shares', [])
  446. image_id = context["general_image"] or None
  447. ngt = saharaclient.nodegroup_template_create(
  448. request,
  449. name=context["general_nodegroup_name"],
  450. plugin_name=plugin,
  451. hadoop_version=hadoop_version,
  452. description=context["general_description"],
  453. flavor_id=context["general_flavor"],
  454. volumes_per_node=volumes_per_node,
  455. volumes_size=volumes_size,
  456. volumes_availability_zone=volumes_availability_zone,
  457. volume_type=volume_type,
  458. volume_local_to_instance=volume_local_to_instance,
  459. node_processes=processes,
  460. node_configs=configs_dict,
  461. floating_ip_pool=context.get("general_floating_ip_pool"),
  462. security_groups=context["security_groups"],
  463. auto_security_group=context["security_autogroup"],
  464. is_proxy_gateway=context["general_proxygateway"],
  465. availability_zone=context["general_availability_zone"],
  466. use_autoconfig=context['general_use_autoconfig'],
  467. shares=ngt_shares,
  468. is_public=context['general_is_public'],
  469. is_protected=context['general_is_protected'],
  470. image_id=image_id)
  471. hlps = helpers.Helpers(request)
  472. if hlps.is_from_guide():
  473. guide_type = context["general_guide_template_type"]
  474. request.session[guide_type + "_name"] = (
  475. context["general_nodegroup_name"])
  476. request.session[guide_type + "_id"] = ngt.id
  477. self.success_url = (
  478. "horizon:project:data_processing.clusters:cluster_guide")
  479. return True
  480. except api_base.APIException as e:
  481. self.error_description = str(e)
  482. return False
  483. except Exception:
  484. exceptions.handle(request)
  485. class SelectPluginAction(workflows.Action,
  486. workflow_helpers.PluginAndVersionMixin):
  487. hidden_create_field = forms.CharField(
  488. required=False,
  489. widget=forms.HiddenInput(attrs={"class": "hidden_create_field"}))
  490. def __init__(self, request, *args, **kwargs):
  491. super(SelectPluginAction, self).__init__(request, *args, **kwargs)
  492. sahara = saharaclient.client(request)
  493. self._generate_plugin_version_fields(sahara)
  494. class Meta(object):
  495. name = _("Select plugin and hadoop version")
  496. help_text_template = "nodegroup_templates/_create_general_help.html"
  497. class SelectPlugin(workflows.Step):
  498. action_class = SelectPluginAction
  499. contributes = ("plugin_name", "hadoop_version")
  500. def contribute(self, data, context):
  501. context = super(SelectPlugin, self).contribute(data, context)
  502. context["plugin_name"] = data.get('plugin_name', None)
  503. context["hadoop_version"] = \
  504. data.get(context["plugin_name"] + "_version", None)
  505. return context
  506. class CreateNodegroupTemplate(workflows.Workflow):
  507. slug = "create_nodegroup_template"
  508. name = _("Create Node Group Template")
  509. finalize_button_name = _("Next")
  510. success_message = _("Created")
  511. failure_message = _("Could not create")
  512. success_url = ("horizon:project:data_processing.clusters:"
  513. "nodegroup-templates-tab")
  514. default_steps = (SelectPlugin,)