Fuel UI
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

cluster.py 61KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733
  1. # -*- coding: utf-8 -*-
  2. # Copyright 2013 Mirantis, Inc.
  3. #
  4. # Licensed under the Apache License, Version 2.0 (the "License"); you may
  5. # not use this file except in compliance with the License. You may obtain
  6. # a copy of the License at
  7. #
  8. # http://www.apache.org/licenses/LICENSE-2.0
  9. #
  10. # Unless required by applicable law or agreed to in writing, software
  11. # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  12. # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  13. # License for the specific language governing permissions and limitations
  14. # under the License.
  15. """
  16. Cluster-related objects and collections
  17. """
  18. import copy
  19. from distutils.version import StrictVersion
  20. import itertools
  21. import six
  22. import sqlalchemy as sa
  23. from sqlalchemy.dialects import postgresql as psql
  24. from sqlalchemy.orm.exc import MultipleResultsFound
  25. from sqlalchemy.orm.exc import NoResultFound
  26. from nailgun import consts
  27. from nailgun.db import db
  28. from nailgun.db.sqlalchemy import models
  29. from nailgun import errors
  30. from nailgun.extensions import callback_wrapper
  31. from nailgun.extensions import fire_callback_on_cluster_create
  32. from nailgun.extensions import fire_callback_on_cluster_delete
  33. from nailgun.extensions import fire_callback_on_cluster_patch_attributes
  34. from nailgun.extensions import fire_callback_on_node_collection_delete
  35. from nailgun.logger import logger
  36. from nailgun.objects import DeploymentGraph
  37. from nailgun.objects import NailgunCollection
  38. from nailgun.objects import NailgunObject
  39. from nailgun.objects.plugin import ClusterPlugin
  40. from nailgun.objects import Release
  41. from nailgun.objects.serializers.cluster import ClusterSerializer
  42. from nailgun.plugins.manager import PluginManager
  43. from nailgun.policy.merge import NetworkRoleMergePolicy
  44. from nailgun.settings import settings
  45. from nailgun.utils import AttributesGenerator
  46. from nailgun.utils import dict_merge
  47. from nailgun.utils import dict_update
  48. from nailgun.utils import text_format_safe
  49. from nailgun.utils import traverse
  50. class Attributes(NailgunObject):
  51. """Cluster attributes object."""
  52. #: SQLAlchemy model for Cluster attributes
  53. model = models.Attributes
  54. @classmethod
  55. def generate_fields(cls, instance):
  56. """Generate field values for Cluster attributes using generators.
  57. :param instance: Attributes instance
  58. :returns: None
  59. """
  60. instance.generated = traverse(
  61. instance.generated or {},
  62. formatter_context={
  63. 'cluster': instance.cluster, 'settings': settings,
  64. },
  65. keywords={
  66. 'generator': AttributesGenerator.evaluate
  67. }
  68. )
  69. # TODO(ikalnitsky):
  70. #
  71. # Think about traversing "editable" attributes. It might be very
  72. # useful to generate default values for editable attribute at
  73. # cluster creation time.
  74. @classmethod
  75. def merged_attrs(cls, instance):
  76. """Generates merged dict of attributes
  77. Result includes generated Cluster attributes recursively updated
  78. by new values from editable attributes
  79. :param instance: Attributes instance
  80. :returns: dict of merged attributes
  81. """
  82. return dict_merge(
  83. instance.generated,
  84. instance.editable
  85. )
  86. @classmethod
  87. def merged_attrs_values(cls, instance):
  88. """Transforms raw dict of attributes into dict of facts
  89. Raw dict is taken from :func:`merged_attrs`
  90. The result of this function is a dict of facts that wil be sent to
  91. orchestrator
  92. :param instance: Attributes instance
  93. :returns: dict of merged attributes
  94. """
  95. attrs = cls.merged_attrs(instance)
  96. for group_attrs in six.itervalues(attrs):
  97. for attr, value in six.iteritems(group_attrs):
  98. if isinstance(value, dict) and 'value' in value:
  99. group_attrs[attr] = value['value']
  100. if 'common' in attrs:
  101. attrs.update(attrs.pop('common'))
  102. if 'additional_components' in attrs:
  103. for comp, enabled in six.iteritems(attrs['additional_components']):
  104. if isinstance(enabled, bool):
  105. attrs.setdefault(comp, {}).update({
  106. "enabled": enabled
  107. })
  108. attrs.pop('additional_components')
  109. return attrs
  110. class Cluster(NailgunObject):
  111. """Cluster object."""
  112. #: SQLAlchemy model for Cluster
  113. model = models.Cluster
  114. #: Serializer for Cluster
  115. serializer = ClusterSerializer
  116. @classmethod
  117. def create(cls, data):
  118. """Create Cluster instance with specified parameters in DB.
  119. This includes:
  120. * creating Cluster attributes and generating default values \
  121. (see :func:`create_attributes`)
  122. * creating NetworkGroups for Cluster
  123. * adding default pending changes (see :func:`add_pending_changes`)
  124. * if "nodes" are specified in data then they are added to Cluster \
  125. (see :func:`update_nodes`)
  126. :param data: dictionary of key-value pairs as object fields
  127. :returns: Cluster instance
  128. """
  129. # TODO(enchantner): fix this temporary hack in clients
  130. if "release_id" not in data:
  131. release_id = data.pop("release", None)
  132. data["release_id"] = release_id
  133. # remove read-only attribute
  134. data.pop("is_locked", None)
  135. assign_nodes = data.pop("nodes", [])
  136. enabled_editable_attributes = None
  137. if 'components' in data:
  138. enabled_core_attributes = cls.get_cluster_attributes_by_components(
  139. data['components'], data["release_id"])
  140. data = dict_merge(data, enabled_core_attributes['cluster'])
  141. enabled_editable_attributes = enabled_core_attributes['editable']
  142. data["fuel_version"] = settings.VERSION["release"]
  143. deployment_tasks = data.pop("deployment_tasks", [])
  144. cluster = super(Cluster, cls).create(data)
  145. cls.create_default_group(cluster)
  146. cls.create_attributes(cluster, enabled_editable_attributes)
  147. cls.create_vmware_attributes(cluster)
  148. cls.create_default_extensions(cluster)
  149. # default graph should be created in any case
  150. DeploymentGraph.create_for_model({"tasks": deployment_tasks}, cluster)
  151. cls.add_pending_changes(
  152. cluster, consts.CLUSTER_CHANGES.attributes)
  153. cls.add_pending_changes(
  154. cluster, consts.CLUSTER_CHANGES.vmware_attributes)
  155. ClusterPlugin.add_compatible_plugins(cluster)
  156. PluginManager.enable_plugins_by_components(cluster)
  157. fire_callback_on_cluster_create(cluster, data)
  158. if assign_nodes:
  159. cls.update_nodes(cluster, assign_nodes)
  160. return cluster
  161. @classmethod
  162. def get_cluster_attributes_by_components(cls, components, release_id):
  163. """Enable cluster attributes by given components
  164. :param components: list of component names
  165. :type components: list of strings
  166. :param release_id: Release model id
  167. :type release_id: str
  168. :returns: dict -- objects with enabled attributes for cluster
  169. """
  170. def _update_attributes_dict_by_binds_exp(bind_exp, value):
  171. """Update cluster and attributes data with bound values
  172. :param bind_exp: path to specific attribute for model in format
  173. model:some.attribute.value. Model can be
  174. settings|cluster
  175. :type bind_exp: str
  176. :param value: value for specific attribute
  177. :type value: bool|str|int
  178. :returns: None
  179. """
  180. model, attr_expr = bind_exp.split(':')
  181. if model not in ('settings', 'cluster'):
  182. return
  183. path_items = attr_expr.split('.')
  184. path_items.insert(0, model)
  185. attributes = cluster_attributes
  186. for i in six.moves.range(0, len(path_items) - 1):
  187. attributes = attributes.setdefault(path_items[i], {})
  188. attributes[path_items[-1]] = value
  189. release = Release.get_by_uid(release_id)
  190. cluster_attributes = {}
  191. for component in Release.get_all_components(release):
  192. if component['name'] in components:
  193. for bind_item in component.get('bind', []):
  194. if isinstance(bind_item, six.string_types):
  195. _update_attributes_dict_by_binds_exp(bind_item, True)
  196. elif isinstance(bind_item, list):
  197. _update_attributes_dict_by_binds_exp(bind_item[0],
  198. bind_item[1])
  199. return {
  200. 'editable': cluster_attributes.get('settings', {}),
  201. 'cluster': cluster_attributes.get('cluster', {})
  202. }
  203. @classmethod
  204. @callback_wrapper('cluster_delete', ['instance'])
  205. def delete(cls, instance):
  206. """Delete cluster.
  207. :param instance: Cluster model instance
  208. :type instance: models.Cluster
  209. """
  210. DeploymentGraph.delete_for_parent(instance)
  211. node_ids = [
  212. _id for (_id,) in
  213. db().query(models.Node.id).
  214. filter_by(cluster_id=instance.id).
  215. order_by(models.Node.id)]
  216. fire_callback_on_node_collection_delete(node_ids)
  217. fire_callback_on_cluster_delete(instance)
  218. super(Cluster, cls).delete(instance)
  219. @classmethod
  220. def get_default_kernel_params(cls, instance):
  221. kernel_params = instance.attributes.editable.get("kernel_params", {})
  222. return kernel_params.get("kernel", {}).get("value")
  223. @classmethod
  224. def create_attributes(cls, instance, editable_attributes=None):
  225. """Create attributes for Cluster instance, generate their values
  226. (see :func:`Attributes.generate_fields`)
  227. :param instance: Cluster instance
  228. :param editable_attributes: key-value dictionary represents editable
  229. attributes that will be merged with default editable attributes
  230. :returns: None
  231. """
  232. merged_editable_attributes = \
  233. cls.get_default_editable_attributes(instance)
  234. if editable_attributes:
  235. merged_editable_attributes = dict_merge(
  236. merged_editable_attributes, editable_attributes)
  237. attributes = Attributes.create(
  238. {
  239. "editable": merged_editable_attributes,
  240. "generated": instance.release.attributes_metadata.get(
  241. "generated"
  242. ),
  243. "cluster_id": instance.id
  244. }
  245. )
  246. Attributes.generate_fields(attributes)
  247. db().flush()
  248. return attributes
  249. @classmethod
  250. def create_default_extensions(cls, instance):
  251. """Sets default extensions list from release model
  252. :param instance: Cluster instance
  253. :returns: None
  254. """
  255. instance.extensions = instance.release.extensions
  256. db().flush()
  257. @classmethod
  258. def get_default_editable_attributes(cls, instance):
  259. """Get editable attributes from release metadata
  260. :param instance: Cluster instance
  261. :returns: Dict object
  262. """
  263. editable = instance.release.attributes_metadata.get("editable")
  264. # Add default attributes of connected plugins
  265. plugin_attrs = PluginManager.get_plugins_attributes(
  266. instance, all_versions=True, default=True)
  267. editable = dict(plugin_attrs, **editable)
  268. editable = traverse(
  269. editable,
  270. formatter_context={'cluster': instance, 'settings': settings},
  271. keywords={'generator': AttributesGenerator.evaluate}
  272. )
  273. return editable
  274. @classmethod
  275. def get_attributes(cls, instance, all_plugins_versions=False):
  276. """Get attributes for current Cluster instance.
  277. :param instance: Cluster instance
  278. :param all_plugins_versions: Get attributes of all versions of plugins
  279. :returns: dict
  280. """
  281. try:
  282. attrs = db().query(models.Attributes).filter(
  283. models.Attributes.cluster_id == instance.id
  284. ).one()
  285. except MultipleResultsFound:
  286. raise errors.InvalidData(
  287. u"Multiple rows with attributes were found for cluster '{0}'"
  288. .format(instance.name)
  289. )
  290. except NoResultFound:
  291. raise errors.InvalidData(
  292. u"No attributes were found for cluster '{0}'"
  293. .format(instance.name)
  294. )
  295. attrs = copy.deepcopy(attrs)
  296. # Merge plugins attributes into editable ones
  297. plugin_attrs = PluginManager.get_plugins_attributes(
  298. instance, all_versions=all_plugins_versions)
  299. plugin_attrs = traverse(
  300. plugin_attrs,
  301. formatter=text_format_safe,
  302. formatter_context={'cluster': instance, 'settings': settings},
  303. keywords={'generator': AttributesGenerator.evaluate}
  304. )
  305. attrs['editable'].update(plugin_attrs)
  306. return attrs
  307. @classmethod
  308. def get_editable_attributes(cls, instance, all_plugins_versions=False):
  309. """Get editable attributes for current Cluster instance.
  310. :param instance: Cluster instance
  311. :param all_plugins_versions: Get attributes of all versions of plugins
  312. :return: dict
  313. """
  314. return cls.get_attributes(instance, all_plugins_versions)['editable']
  315. @classmethod
  316. def update_attributes(cls, instance, data):
  317. PluginManager.process_cluster_attributes(instance, data['editable'])
  318. for key, value in six.iteritems(data):
  319. setattr(instance.attributes, key, value)
  320. cls.add_pending_changes(instance, "attributes")
  321. db().flush()
  322. @classmethod
  323. def update_role(cls, instance, role):
  324. """Update existing Cluster instance with specified role.
  325. Previous ones are deleted.
  326. :param instance: a Cluster instance
  327. :param role: a role dict
  328. :returns: None
  329. """
  330. old_role = instance.roles_metadata.get(role['name'], {})
  331. instance.roles_metadata[role['name']] = role['meta']
  332. if old_role:
  333. deleted_tags = set(old_role['tags']) - set(role['meta']['tags'])
  334. for tag in deleted_tags:
  335. cls.remove_primary_tag(instance, tag)
  336. instance.volumes_metadata.setdefault(
  337. 'volumes_roles_mapping', {}).update(
  338. {role['name']: role.get('volumes_roles_mapping', [])})
  339. # notify about changes
  340. instance.volumes_metadata.changed()
  341. @classmethod
  342. def remove_role(cls, instance, role_name):
  343. result = instance.roles_metadata.pop(role_name, None)
  344. instance.volumes_metadata['volumes_roles_mapping'].pop(role_name, None)
  345. # notify about changes
  346. instance.volumes_metadata.changed()
  347. return bool(result)
  348. @classmethod
  349. def update_tag(cls, instance, tag):
  350. """Update existing Cluster instance with specified tag.
  351. Previous ones are deleted.
  352. :param instance: a Cluster instance
  353. :param tag: a tag dict
  354. :returns: None
  355. """
  356. instance.tags_metadata[tag['name']] = tag['meta']
  357. @classmethod
  358. def remove_tag(cls, instance, tag_name):
  359. res = instance.tags_metadata.pop(tag_name, None)
  360. if tag_name not in instance.release.tags_metadata:
  361. cls.remove_tag_from_roles(instance, tag_name)
  362. cls.remove_primary_tag(instance, tag_name)
  363. return bool(res)
  364. @classmethod
  365. def remove_tag_from_roles(cls, instance, tag_name):
  366. for role, meta in six.iteritems(cls.get_own_roles(instance)):
  367. tags = meta.get('tags', [])
  368. if tag_name in tags:
  369. tags.remove(tag_name)
  370. instance.roles_metadata.changed()
  371. @classmethod
  372. def _create_public_map(cls, instance, roles_metadata=None):
  373. if instance.network_config.configuration_template is not None:
  374. return
  375. from nailgun import objects
  376. public_map = {}
  377. for node in instance.nodes:
  378. public_map[node.id] = objects.Node.should_have_public(
  379. node, roles_metadata)
  380. return public_map
  381. @classmethod
  382. def patch_attributes(cls, instance, data):
  383. """Applyes changes to Cluster attributes and updates networks.
  384. :param instance: Cluster object
  385. :param data: dict
  386. """
  387. roles_metadata = Cluster.get_roles(instance)
  388. # Note(kszukielojc): We need to create status map of public networks
  389. # to avoid updating networks if there was no change to node public
  390. # network after patching attributes
  391. public_map = cls._create_public_map(instance, roles_metadata)
  392. PluginManager.process_cluster_attributes(instance, data['editable'])
  393. instance.attributes.editable = dict_merge(
  394. instance.attributes.editable, data['editable'])
  395. cls.add_pending_changes(instance, "attributes")
  396. fire_callback_on_cluster_patch_attributes(instance, public_map)
  397. db().flush()
  398. @classmethod
  399. def get_updated_editable_attributes(cls, instance, data):
  400. """Same as get_editable_attributes but also merges given data.
  401. :param instance: Cluster object
  402. :param data: dict
  403. :returns: dict
  404. """
  405. attributes = {'editable': dict_merge(
  406. cls.get_editable_attributes(instance),
  407. data.get('editable', {})
  408. )}
  409. # plugin attributes should be updated with values to provide
  410. # consitency between new data and plugin attributes data from DB
  411. PluginManager.inject_plugin_attribute_values(attributes['editable'])
  412. return attributes
  413. @classmethod
  414. def get_network_manager(cls, instance=None):
  415. """Get network manager for Cluster instance.
  416. If instance is None then the default NetworkManager is returned
  417. :param instance: Cluster instance
  418. :returns: NetworkManager/NovaNetworkManager/NeutronManager
  419. """
  420. if not instance:
  421. from nailgun.extensions.network_manager.manager import \
  422. NetworkManager
  423. return NetworkManager
  424. ver = instance.release.environment_version
  425. net_provider = instance.net_provider
  426. if net_provider == consts.CLUSTER_NET_PROVIDERS.neutron:
  427. from nailgun.extensions.network_manager.managers import neutron
  428. if StrictVersion(ver) < StrictVersion('6.1'):
  429. return neutron.NeutronManagerLegacy
  430. if StrictVersion(ver) == StrictVersion('6.1'):
  431. return neutron.NeutronManager61
  432. if StrictVersion(ver) == StrictVersion('7.0'):
  433. return neutron.NeutronManager70
  434. if StrictVersion(ver) >= StrictVersion('8.0'):
  435. return neutron.NeutronManager80
  436. return neutron.NeutronManager
  437. elif net_provider == consts.CLUSTER_NET_PROVIDERS.nova_network:
  438. from nailgun.extensions.network_manager.managers import \
  439. nova_network
  440. if StrictVersion(ver) < StrictVersion('6.1'):
  441. return nova_network.NovaNetworkManagerLegacy
  442. if StrictVersion(ver) == StrictVersion('6.1'):
  443. return nova_network.NovaNetworkManager61
  444. if StrictVersion(ver) == StrictVersion('7.0'):
  445. return nova_network.NovaNetworkManager70
  446. if StrictVersion(ver) >= StrictVersion('8.0'):
  447. raise errors.NovaNetworkNotSupported()
  448. return nova_network.NovaNetworkManager
  449. raise ValueError(
  450. 'The network provider "{0}" is not supported.'
  451. .format(net_provider)
  452. )
  453. @classmethod
  454. def add_pending_changes(cls, instance, changes_type, node_id=None):
  455. """Add pending changes for current Cluster.
  456. If node_id is specified then links created changes with node.
  457. :param instance: Cluster instance
  458. :param changes_type: name of changes to add
  459. :param node_id: node id for changes
  460. :returns: None
  461. """
  462. logger.debug(
  463. u"New pending changes in environment {0}: {1}{2}".format(
  464. instance.id,
  465. changes_type,
  466. u" node_id={0}".format(node_id) if node_id else u""
  467. )
  468. )
  469. # TODO(enchantner): check if node belongs to cluster
  470. ex_chs = db().query(models.ClusterChanges).filter_by(
  471. cluster=instance,
  472. name=changes_type
  473. )
  474. if not node_id:
  475. ex_chs = ex_chs.first()
  476. else:
  477. ex_chs = ex_chs.filter_by(node_id=node_id).first()
  478. # do nothing if changes with the same name already pending
  479. if ex_chs:
  480. return
  481. ch = models.ClusterChanges(
  482. cluster_id=instance.id,
  483. name=changes_type
  484. )
  485. if node_id:
  486. ch.node_id = node_id
  487. db().add(ch)
  488. db().flush()
  489. @classmethod
  490. def get_nodes_not_for_deletion(cls, cluster):
  491. """All clusters nodes except nodes for deletion."""
  492. return db().query(models.Node).filter_by(
  493. cluster=cluster, pending_deletion=False).order_by(models.Node.id)
  494. @classmethod
  495. def clear_pending_changes(cls, instance, node_id=None):
  496. """Clear pending changes for current Cluster.
  497. If node_id is specified then only clears changes connected
  498. to this node.
  499. :param instance: Cluster instance
  500. :param node_id: node id for changes
  501. :returns: None
  502. """
  503. logger.debug(
  504. u"Removing pending changes in environment {0}{1}".format(
  505. instance.id,
  506. u" where node_id={0}".format(node_id) if node_id else u""
  507. )
  508. )
  509. chs = db().query(models.ClusterChanges).filter_by(
  510. cluster_id=instance.id
  511. )
  512. if node_id:
  513. chs = chs.filter_by(node_id=node_id)
  514. for ch in chs.all():
  515. db().delete(ch)
  516. db().flush()
  517. @classmethod
  518. def update(cls, instance, data):
  519. """Update Cluster object instance with specified parameters in DB
  520. If "nodes" are specified in data then they will replace existing ones
  521. (see :func:`update_nodes`)
  522. :param instance: Cluster instance
  523. :param data: dictionary of key-value pairs as object fields
  524. :returns: Cluster instance
  525. """
  526. # remove read-only attributes
  527. data.pop("fuel_version", None)
  528. data.pop("is_locked", None)
  529. nodes = data.pop("nodes", None)
  530. changes = data.pop("changes", None)
  531. deployment_tasks = data.pop("deployment_tasks", None)
  532. super(Cluster, cls).update(instance, data)
  533. if deployment_tasks:
  534. deployment_graph_instance = DeploymentGraph.get_for_model(instance)
  535. DeploymentGraph.update(
  536. deployment_graph_instance, {"tasks": deployment_tasks})
  537. if nodes is not None:
  538. cls.update_nodes(instance, nodes)
  539. if changes is not None:
  540. cls.update_changes(instance, changes)
  541. return instance
  542. @classmethod
  543. def update_nodes(cls, instance, nodes_ids):
  544. """Update Cluster nodes by specified node IDs
  545. Nodes with specified IDs will replace existing ones in Cluster
  546. :param instance: Cluster instance
  547. :param nodes_ids: list of nodes ids
  548. :returns: None
  549. """
  550. from nailgun import objects
  551. # TODO(NAME): sepatate nodes
  552. # for deletion and addition by set().
  553. new_nodes = []
  554. if nodes_ids:
  555. new_nodes = db().query(models.Node).filter(
  556. models.Node.id.in_(nodes_ids)
  557. )
  558. nodes_to_remove = [n for n in instance.nodes
  559. if n not in new_nodes]
  560. nodes_to_add = [n for n in new_nodes
  561. if n not in instance.nodes]
  562. for node in nodes_to_add:
  563. if not node.online:
  564. raise errors.NodeOffline(
  565. u"Cannot add offline node "
  566. u"'{0}' to environment".format(node.id)
  567. )
  568. # we should reset hostname to default value to guarantee
  569. # hostnames uniqueness for nodes outside clusters
  570. for node in nodes_to_remove:
  571. node.hostname = objects.Node.default_slave_name(node)
  572. instance.nodes.remove(node)
  573. for node in nodes_to_add:
  574. instance.nodes.append(node)
  575. net_manager = cls.get_network_manager(instance)
  576. for node in nodes_to_remove:
  577. net_manager.clear_assigned_networks(node)
  578. net_manager.clear_bond_configuration(node)
  579. cls.replace_provisioning_info_on_nodes(instance, {}, nodes_to_remove)
  580. cls.replace_deployment_info_on_nodes(instance, {}, nodes_to_remove)
  581. objects.NodeCollection.reset_network_template(nodes_to_remove)
  582. objects.NodeCollection.reset_attributes(nodes_to_remove)
  583. objects.OpenstackConfig.disable_by_nodes(nodes_to_remove)
  584. for node in nodes_to_add:
  585. objects.Node.assign_group(node)
  586. net_manager.assign_networks_by_default(node)
  587. objects.Node.set_default_attributes(node)
  588. objects.Node.create_nic_attributes(node)
  589. objects.Node.refresh_dpdk_properties(node)
  590. cls.update_nodes_network_template(instance, nodes_to_add)
  591. db().flush()
  592. @classmethod
  593. def update_changes(cls, instance, changes):
  594. instance.changes_list = [
  595. models.ClusterChanges(**change) for change in changes
  596. ]
  597. db().flush()
  598. @classmethod
  599. def get_ifaces_for_network_in_cluster(cls, cluster, net):
  600. """Method for receiving node_id:iface pairs for all nodes in cluster
  601. :param instance: Cluster instance
  602. :param net: Nailgun specific network name
  603. :type net: str
  604. :returns: List of node_id, iface pairs for all nodes in cluster.
  605. """
  606. nics_db = db().query(
  607. models.NodeNICInterface.node_id,
  608. models.NodeNICInterface.name
  609. ).filter(
  610. models.NodeNICInterface.node.has(cluster_id=cluster.id),
  611. models.NodeNICInterface.assigned_networks_list.any(name=net)
  612. )
  613. bonds_db = db().query(
  614. models.NodeBondInterface.node_id,
  615. models.NodeBondInterface.name
  616. ).filter(
  617. models.NodeBondInterface.node.has(cluster_id=cluster.id),
  618. models.NodeBondInterface.assigned_networks_list.any(name=net)
  619. )
  620. return nics_db.union(bonds_db)
  621. @classmethod
  622. def replace_provisioning_info_on_nodes(cls, instance, data, nodes):
  623. if isinstance(data, list):
  624. data = {n.get('uid'): n for n in data}
  625. for node in nodes:
  626. node.replaced_provisioning_info = data.get(node.uid, {})
  627. @classmethod
  628. def replace_deployment_info_on_nodes(cls, instance, data, nodes):
  629. if isinstance(data, list):
  630. data = {n.get('uid'): n for n in data}
  631. for node in nodes:
  632. node_data = data.get(node.uid, [])
  633. # replaced deployment info for node should be list
  634. # because before in previous versions of nailgun
  635. # node info will be per role, not per node
  636. if isinstance(node_data, dict):
  637. node_data = [node_data]
  638. node.replaced_deployment_info = node_data
  639. @classmethod
  640. def replace_provisioning_info(cls, instance, data):
  641. received_nodes = data.pop('nodes', [])
  642. instance.is_customized = True
  643. instance.replaced_provisioning_info = data
  644. cls.replace_provisioning_info_on_nodes(
  645. instance, received_nodes, instance.nodes)
  646. return cls.get_provisioning_info(instance)
  647. @classmethod
  648. def replace_deployment_info(cls, instance, data):
  649. instance.is_customized = True
  650. instance.replaced_deployment_info = data.get('common', {})
  651. cls.replace_deployment_info_on_nodes(
  652. instance, data.get('nodes', {}), instance.nodes
  653. )
  654. return cls.get_deployment_info(instance)
  655. @classmethod
  656. def get_provisioning_info(cls, instance):
  657. data = {}
  658. if instance.replaced_provisioning_info:
  659. data.update(instance.replaced_provisioning_info)
  660. nodes = []
  661. for node in instance.nodes:
  662. if node.replaced_provisioning_info:
  663. nodes.append(node.replaced_provisioning_info)
  664. if data:
  665. data['nodes'] = nodes
  666. return data
  667. @classmethod
  668. def get_deployment_info(cls, instance):
  669. nodes = []
  670. for node in instance.nodes:
  671. if node.replaced_deployment_info:
  672. nodes.extend(node.replaced_deployment_info)
  673. data = {}
  674. if nodes:
  675. data['nodes'] = nodes
  676. if instance.replaced_deployment_info:
  677. data['common'] = instance.replaced_deployment_info
  678. return data
  679. @classmethod
  680. def get_creds(cls, instance):
  681. return instance.attributes.editable['access']
  682. @classmethod
  683. def should_assign_public_to_all_nodes(cls, instance):
  684. """Check if Public network is to be assigned to all nodes in cluster
  685. :param instance: cluster instance
  686. :returns: True when Public network is to be assigned to all nodes
  687. """
  688. if instance.net_provider == \
  689. consts.CLUSTER_NET_PROVIDERS.nova_network:
  690. return True
  691. assignment = instance.attributes.editable.get(
  692. 'public_network_assignment')
  693. if not assignment or assignment['assign_to_all_nodes']['value']:
  694. return True
  695. return False
  696. @classmethod
  697. def neutron_dvr_enabled(cls, instance):
  698. neutron_attrs = instance.attributes.editable.get(
  699. 'neutron_advanced_configuration')
  700. if neutron_attrs:
  701. return neutron_attrs['neutron_dvr']['value']
  702. else:
  703. return False
  704. @classmethod
  705. def dpdk_enabled(cls, instance):
  706. # Had to do this due to issues with modules imports in current
  707. # nailgun __init__.py which cannot be resolved easily
  708. from nailgun.objects import Node
  709. if Release.is_nfv_supported(instance.release):
  710. for node in cls.get_nodes_not_for_deletion(instance):
  711. if Node.dpdk_enabled(node):
  712. return True
  713. return False
  714. @classmethod
  715. def get_roles(cls, instance):
  716. """Returns a dictionary of node roles available for deployment.
  717. :param instance: cluster instance
  718. :returns: a dictionary of roles metadata
  719. """
  720. available_roles = copy.deepcopy(cls.get_own_roles(instance.release))
  721. available_roles.update(cls.get_own_roles(instance))
  722. available_roles.update(
  723. PluginManager.get_plugins_node_roles(instance))
  724. return available_roles
  725. @classmethod
  726. def get_roles_by_tag(cls, tag_name, instance):
  727. roles = set()
  728. for role, meta in six.iteritems(cls.get_own_roles(instance)):
  729. if tag_name in meta.get('tags', {}):
  730. roles.add(role)
  731. return roles
  732. @classmethod
  733. def get_own_roles(cls, instance):
  734. return instance.roles_metadata
  735. @classmethod
  736. def get_own_tags(cls, instance):
  737. return instance.tags_metadata
  738. @classmethod
  739. def remove_primary_tag(cls, instance, tag):
  740. node = cls.get_primary_node(instance, tag)
  741. if node:
  742. node.primary_tags.remove(tag)
  743. @classmethod
  744. def set_primary_tag(cls, instance, nodes, tag):
  745. """Method for assigning primary attribute for specific tag.
  746. :param instance: Cluster db objects
  747. :param nodes: list of Node db objects
  748. :param tag: string with known tag name
  749. """
  750. from nailgun.objects import Node
  751. node = cls.get_primary_node(instance, tag)
  752. if not node:
  753. # get nodes with a given role name which are not going to be
  754. # removed
  755. filtered_nodes = []
  756. for node in nodes:
  757. if (not node.pending_deletion and (
  758. tag in Node.get_tags(node))):
  759. filtered_nodes.append(node)
  760. filtered_nodes = sorted(filtered_nodes, key=lambda node: node.id)
  761. if filtered_nodes:
  762. primary_node = next((
  763. node for node in filtered_nodes
  764. if node.status == consts.NODE_STATUSES.ready),
  765. filtered_nodes[0])
  766. primary_node.primary_tags.append(tag)
  767. db().flush()
  768. @classmethod
  769. def set_primary_tags(cls, instance, nodes):
  770. """Assignment of all primary attribute for all tags that requires it.
  771. This method is idempotent
  772. To mark tag as primary add "has_primary: true" attribute to tag meta
  773. :param instance: Cluster db object
  774. :param nodes: list of Node db objects
  775. """
  776. if not instance.is_ha_mode:
  777. return
  778. tags_meta = cls.get_tags_metadata(instance)
  779. for role, meta in six.iteritems(cls.get_roles(instance)):
  780. for tag in meta.get('tags', []):
  781. if tags_meta[tag].get('has_primary'):
  782. cls.set_primary_tag(instance, nodes, tag)
  783. @classmethod
  784. def get_nodes_by_role(cls, instance, role_name):
  785. """Get nodes related to some specific role
  786. :param instance: cluster db object
  787. :type: python object
  788. :param role_name: node role name
  789. :type: string
  790. """
  791. from nailgun.objects import Node
  792. if role_name not in cls.get_roles(instance):
  793. logger.warning("%s role doesn't exist", role_name)
  794. return []
  795. return Node.get_nodes_by_role([instance.id], role_name).all()
  796. @classmethod
  797. def get_node_by_role(cls, instance, role_name):
  798. from nailgun.objects import Node
  799. return Node.get_nodes_by_role([instance.id], role_name).first()
  800. @classmethod
  801. def get_nodes_by_status(cls, instance, status, exclude=None):
  802. """Get cluster nodes with particular status
  803. :param instance: cluster instance
  804. :param status: node status
  805. :param exclude: the list of uids to exclude
  806. :return: filtered query on nodes
  807. """
  808. query = db().query(models.Node).filter_by(
  809. cluster_id=instance.id,
  810. status=status
  811. )
  812. if exclude:
  813. query = query.filter(sa.not_(models.Node.id.in_(exclude)))
  814. return query
  815. @classmethod
  816. def get_primary_node(cls, instance, tag):
  817. """Get primary node for tag
  818. If primary node is not found None will be returned
  819. :param instance: cluster db object
  820. :type: python object
  821. :param tag: node tag name
  822. :type: string
  823. :returns: node db object or None
  824. """
  825. logger.debug("Getting primary node for tag: %s", tag)
  826. primary_node = db().query(models.Node).filter_by(
  827. pending_deletion=False,
  828. cluster_id=instance.id
  829. ).filter(
  830. models.Node.primary_tags.any(tag)
  831. ).first()
  832. if primary_node is None:
  833. logger.debug("Not found primary node for tag: %s", tag)
  834. else:
  835. logger.debug("Found primary node: %s for tag: %s",
  836. primary_node.id, tag)
  837. return primary_node
  838. @classmethod
  839. def get_controllers_group_id(cls, instance):
  840. return cls.get_controllers_node_group(instance).id
  841. @classmethod
  842. def get_controllers_node_group(cls, instance):
  843. return cls.get_common_node_group(instance, ['controller'])
  844. @classmethod
  845. def get_common_node_group(cls, instance, noderoles):
  846. """Returns a common node group for a given node roles.
  847. If a given node roles have different node groups, the error
  848. will be raised, so it's mandatory to have them the same
  849. node group.
  850. :param instance: a Cluster instance
  851. :param noderoles: a list of node roles
  852. :returns: a common NodeGroup instance
  853. """
  854. nodegroups = cls.get_node_groups(instance, noderoles).all()
  855. if not nodegroups:
  856. return None
  857. if len(nodegroups) > 1:
  858. raise errors.CanNotFindCommonNodeGroup(
  859. 'Node roles [{0}] has more than one common node group'.format(
  860. ', '.join(noderoles)))
  861. return nodegroups[0]
  862. @classmethod
  863. def get_node_groups(cls, instance, noderoles):
  864. """Returns node groups for given node roles.
  865. :param instance: a Cluster instance
  866. :param noderoles: a list of node roles
  867. :returns: a query for list of NodeGroup instances
  868. """
  869. psql_noderoles = sa.cast(
  870. psql.array(noderoles),
  871. psql.ARRAY(sa.String(consts.ROLE_NAME_MAX_SIZE)))
  872. nodegroups = db().query(models.NodeGroup).join(models.Node).filter(
  873. models.Node.cluster_id == instance.id,
  874. models.Node.pending_deletion.is_(False)
  875. ).filter(sa.or_(
  876. models.Node.roles.overlap(psql_noderoles),
  877. models.Node.pending_roles.overlap(psql_noderoles)
  878. ))
  879. return nodegroups
  880. @classmethod
  881. def get_default_group(cls, instance):
  882. return next(g for g in instance.node_groups if g.is_default)
  883. @classmethod
  884. def create_default_group(cls, instance):
  885. node_group = models.NodeGroup(name=consts.NODE_GROUPS.default,
  886. is_default=True)
  887. instance.node_groups.append(node_group)
  888. db.add(node_group)
  889. db().flush()
  890. return node_group
  891. @classmethod
  892. def get_own_deployment_graph(
  893. cls, instance, graph_type=None):
  894. """Return only cluster own deployment graph.
  895. :param instance: models.Cluster instance
  896. :type instance: models.Cluster
  897. :param graph_type: deployment graph type
  898. :type graph_type: basestring|None
  899. :return: deployment tasks list
  900. :rtype: list[dict]
  901. """
  902. cluster_deployment_graph = DeploymentGraph.get_for_model(
  903. instance, graph_type=graph_type)
  904. if cluster_deployment_graph:
  905. graph_metadata = DeploymentGraph.get_metadata(
  906. cluster_deployment_graph
  907. )
  908. graph_metadata['tasks'] = DeploymentGraph.get_tasks(
  909. cluster_deployment_graph
  910. )
  911. else:
  912. graph_metadata = {'tasks': []}
  913. return graph_metadata
  914. @classmethod
  915. def get_own_deployment_tasks(
  916. cls, instance, graph_type=None):
  917. """Return only cluster own deployment graph.
  918. :param instance: models.Cluster instance
  919. :type instance: models.Cluster
  920. :param graph_type: deployment graph type
  921. :type graph_type: basestring|None
  922. :return: deployment tasks list
  923. :rtype: list[dict]
  924. """
  925. return cls.get_own_deployment_graph(instance, graph_type)['tasks']
  926. @classmethod
  927. def get_plugins_deployment_graph(
  928. cls, instance, graph_type=None):
  929. """Get merged deployment tasks for plugins enabled to given cluster.
  930. :param instance: models.Cluster instance
  931. :type instance: models.Cluster
  932. :param graph_type: deployment graph type
  933. :type graph_type: basestring|None
  934. :return: deployment tasks list
  935. :rtype: list[dict]
  936. """
  937. return PluginManager.get_plugins_deployment_graph(
  938. instance, graph_type=graph_type)
  939. @classmethod
  940. def get_plugins_deployment_tasks(
  941. cls, instance, graph_type=None):
  942. """Get merged deployment tasks for plugins enabled to given cluster.
  943. :param instance: models.Cluster instance
  944. :type instance: models.Cluster
  945. :param graph_type: deployment graph type
  946. :type graph_type: basestring|None
  947. :return: deployment tasks list
  948. :rtype: list[dict]
  949. """
  950. return PluginManager.get_plugins_deployment_tasks(
  951. instance, graph_type=graph_type)
  952. @classmethod
  953. def get_release_deployment_graph(
  954. cls, instance, graph_type=None):
  955. """Get merged deployment graph for release related to the cluster.
  956. :param instance: models.Cluster instance
  957. :type instance: models.Cluster
  958. :param graph_type: deployment graph type
  959. :type graph_type: basestring|None
  960. :return: deployment tasks list
  961. :rtype: list[dict]
  962. """
  963. return Release.get_deployment_graph(
  964. instance.release, graph_type=graph_type)
  965. @classmethod
  966. def get_release_deployment_tasks(
  967. cls, instance, graph_type=None):
  968. """Get merged deployment tasks for release related to the cluster.
  969. :param instance: models.Cluster instance
  970. :type instance: models.Cluster
  971. :param graph_type: deployment graph type
  972. :type graph_type: basestring|None
  973. :return: deployment tasks list
  974. :rtype: list[dict]
  975. """
  976. return Release.get_deployment_tasks(
  977. instance.release, graph_type=graph_type)
  978. @classmethod
  979. def _merge_tasks_lists(cls, tasks_lists):
  980. """Merge several tasks lists.
  981. Every next list will override tasks in previous one by `task_name` key.
  982. :param tasks_lists: tasks lists is order of increasing priority
  983. task from next will override task in previous
  984. if ID is same
  985. :type tasks_lists: list[list]
  986. :return: merged list
  987. :rtype: list[dict]
  988. """
  989. result = []
  990. seen = set()
  991. for task in itertools.chain(*reversed(tasks_lists)):
  992. if not task['id'] in seen:
  993. seen.add(task['id'])
  994. result.append(task)
  995. return result
  996. @classmethod
  997. def get_deployment_graph(cls, instance, graph_type=None):
  998. """Return deployment graph for cluster considering release and plugins.
  999. :param instance: models.Cluster instance
  1000. :type instance: models.Cluster
  1001. :param graph_type: deployment graph type
  1002. :type graph_type: basestring|None
  1003. :return: deployment graph which includes metadata and tasks
  1004. :rtype: dict
  1005. """
  1006. if graph_type is None:
  1007. graph_type = consts.DEFAULT_DEPLOYMENT_GRAPH_TYPE
  1008. release_deployment_graph = cls.get_release_deployment_graph(
  1009. instance, graph_type=graph_type)
  1010. plugins_deployment_graph = cls.get_plugins_deployment_graph(
  1011. instance, graph_type=graph_type)
  1012. cluster_deployment_graph = cls.get_own_deployment_graph(
  1013. instance, graph_type=graph_type)
  1014. tasks = cls._merge_tasks_lists([
  1015. release_deployment_graph.pop('tasks'),
  1016. plugins_deployment_graph.pop('tasks'),
  1017. cluster_deployment_graph.pop('tasks')
  1018. ])
  1019. graph_metadata = {}
  1020. # added metadata from each type of graphs according
  1021. # to merge priority
  1022. dict_update(graph_metadata, release_deployment_graph)
  1023. dict_update(graph_metadata, plugins_deployment_graph)
  1024. dict_update(graph_metadata, cluster_deployment_graph)
  1025. graph_metadata['tasks'] = tasks
  1026. graph_metadata['type'] = graph_type
  1027. return graph_metadata
  1028. @classmethod
  1029. def get_deployment_tasks(cls, instance, graph_type=None):
  1030. """Return deployment tasks for cluster considering release and plugins.
  1031. :param instance: models.Cluster instance
  1032. :type instance: models.Cluster
  1033. :param graph_type: deployment graph type
  1034. :type graph_type: basestring|None
  1035. :return: deployment tasks list
  1036. :rtype: list[dict]
  1037. """
  1038. return cls.get_deployment_graph(instance, graph_type)['tasks']
  1039. @classmethod
  1040. def get_legacy_plugin_tasks(cls, instance):
  1041. """Get legacy deployment tasks from tasks.yaml."""
  1042. return PluginManager.get_legacy_tasks_for_cluster(instance)
  1043. @classmethod
  1044. def get_refreshable_tasks(cls, instance, filter_by_configs=None,
  1045. graph_type=None):
  1046. """Return list of refreshable tasks
  1047. If 'filter_by_configs' specified then only tasks needed to update
  1048. these config resources will be returned as a result, otherwise
  1049. all refreshable tasks will be returned
  1050. :param instance: a Cluster instance
  1051. :param filter_by_configs: a list with configs resources
  1052. :param graph_type: deployment graph type
  1053. :return: list of tasks
  1054. """
  1055. if filter_by_configs:
  1056. filter_by_configs = set(filter_by_configs)
  1057. tasks = []
  1058. for task in cls.get_deployment_tasks(instance, graph_type):
  1059. refresh_on = task.get(consts.TASK_REFRESH_FIELD)
  1060. if (refresh_on
  1061. and (filter_by_configs is None
  1062. or filter_by_configs.intersection(set(refresh_on)))):
  1063. tasks.append(task)
  1064. return tasks
  1065. @classmethod
  1066. def get_tags_metadata(cls, instance):
  1067. """Return proper tags metadata for cluster
  1068. Metadata consists of general tags metadata from release,
  1069. tags metadata from cluster and tags metadata from
  1070. plugins which are enabled for this cluster.
  1071. :param instance: Cluster DB instance
  1072. :returns: dict -- object with merged tags metadata
  1073. """
  1074. tags_meta = dict(instance.release.tags_metadata)
  1075. cluster_tags_meta = instance.tags_metadata
  1076. tags_meta.update(cluster_tags_meta)
  1077. plugins_tags_meta = PluginManager.get_tags_metadata(instance)
  1078. tags_meta.update(plugins_tags_meta)
  1079. return tags_meta
  1080. @classmethod
  1081. def get_volumes_metadata(cls, instance):
  1082. """Return proper volumes metadata for cluster
  1083. Metadata consists of general volumes metadata from release,
  1084. volumes metadata from cluster and volumes metadata from
  1085. plugins which are related to this cluster.
  1086. :param instance: Cluster DB instance
  1087. :returns: dict -- object with merged volumes metadata
  1088. """
  1089. def _update_volumes_meta(to_update, data):
  1090. to_update.get('volumes_roles_mapping', {}).update(
  1091. data.get('volumes_roles_mapping', {}))
  1092. to_update.get('volumes', []).extend(
  1093. data.get('volumes', []))
  1094. to_update.setdefault('rule_to_pick_boot_disk', []).extend(
  1095. data.get('rule_to_pick_boot_disk', []))
  1096. volumes_metadata = copy.deepcopy(
  1097. Release.get_volumes_metadata(instance.release))
  1098. cluster_volumes = instance.volumes_metadata
  1099. _update_volumes_meta(volumes_metadata, cluster_volumes)
  1100. plugin_volumes = PluginManager.get_volumes_metadata(instance)
  1101. _update_volumes_meta(volumes_metadata, plugin_volumes)
  1102. return volumes_metadata
  1103. @classmethod
  1104. def create_vmware_attributes(cls, instance):
  1105. """Store VmwareAttributes instance into DB."""
  1106. vmware_metadata = instance.release.vmware_attributes_metadata
  1107. if vmware_metadata:
  1108. return VmwareAttributes.create(
  1109. {
  1110. "editable": vmware_metadata.get("editable"),
  1111. "cluster_id": instance.id
  1112. }
  1113. )
  1114. return None
  1115. @classmethod
  1116. def get_create_data(cls, instance):
  1117. """Return common parameters cluster was created with.
  1118. This method is compatible with :func:`create` and used to create
  1119. a new cluster with the same settings including the network
  1120. configuration.
  1121. :returns: a dict of key-value pairs as a cluster create data
  1122. """
  1123. data = {
  1124. "name": instance.name,
  1125. "mode": instance.mode,
  1126. "net_provider": instance.net_provider,
  1127. "release_id": instance.release.id,
  1128. }
  1129. data.update(cls.get_network_manager(instance).
  1130. get_network_config_create_data(instance))
  1131. return data
  1132. @classmethod
  1133. def get_vmware_attributes(cls, instance):
  1134. """Get VmwareAttributes instance from DB.
  1135. Now we have relation with cluster 1:1.
  1136. """
  1137. return db().query(models.VmwareAttributes).filter(
  1138. models.VmwareAttributes.cluster_id == instance.id
  1139. ).first()
  1140. @classmethod
  1141. def get_default_vmware_attributes(cls, instance):
  1142. """Get metadata from release with empty value section."""
  1143. editable = instance.release.vmware_attributes_metadata.get("editable")
  1144. editable = traverse(
  1145. editable,
  1146. formatter_context={'cluster': instance, 'settings': settings},
  1147. keywords={'generator': AttributesGenerator.evaluate}
  1148. )
  1149. return editable
  1150. @classmethod
  1151. def update_vmware_attributes(cls, instance, data):
  1152. """Update Vmware attributes.
  1153. Actually we allways update only value section in editable.
  1154. """
  1155. metadata = instance.vmware_attributes.editable['metadata']
  1156. value = data.get('editable', {}).get('value')
  1157. vmware_attr = {
  1158. 'metadata': metadata,
  1159. 'value': value
  1160. }
  1161. setattr(instance.vmware_attributes, 'editable', vmware_attr)
  1162. cls.add_pending_changes(instance, "vmware_attributes")
  1163. db().flush()
  1164. vmware_attr.pop('metadata')
  1165. return vmware_attr
  1166. @classmethod
  1167. def is_vmware_enabled(cls, instance):
  1168. """Check if current cluster supports vmware configuration."""
  1169. attributes = cls.get_editable_attributes(instance)
  1170. return attributes.get('common', {}).get('use_vcenter', {}).get('value')
  1171. @staticmethod
  1172. def adjust_nodes_lists_on_controller_removing(instance, nodes_to_delete,
  1173. nodes_to_deploy):
  1174. """Adds controllers to nodes_to_deploy if deleting other controllers
  1175. :param instance: instance of SqlAlchemy cluster
  1176. :param nodes_to_delete: list of nodes to be deleted
  1177. :param nodes_to_deploy: list of nodes to be deployed
  1178. :return:
  1179. """
  1180. if instance is None:
  1181. return
  1182. controllers_ids_to_delete = set([n.id for n in nodes_to_delete
  1183. if 'controller' in n.all_roles])
  1184. if controllers_ids_to_delete:
  1185. ids_to_deploy = set([n.id for n in nodes_to_deploy])
  1186. controllers_to_deploy = set(
  1187. filter(lambda n: (n.id not in controllers_ids_to_delete
  1188. and n.id not in ids_to_deploy
  1189. and 'controller' in n.all_roles),
  1190. instance.nodes))
  1191. nodes_to_deploy.extend(controllers_to_deploy)
  1192. @classmethod
  1193. def get_repo_urls(self, instance):
  1194. repos = instance.attributes.editable['repo_setup']['repos']['value']
  1195. return tuple(set([r['uri'] for r in repos]))
  1196. @classmethod
  1197. def get_nodes_to_spawn_vms(cls, instance):
  1198. nodes = []
  1199. for node in cls.get_nodes_by_role(instance,
  1200. consts.VIRTUAL_NODE_TYPES.virt):
  1201. for vm in node.vms_conf:
  1202. if not vm.get('created'):
  1203. nodes.append(node)
  1204. return nodes
  1205. @classmethod
  1206. def set_vms_created_state(cls, instance):
  1207. nodes = cls.get_nodes_by_role(instance, consts.VIRTUAL_NODE_TYPES.virt)
  1208. for node in nodes:
  1209. for vm in node.vms_conf:
  1210. if not vm.get('created'):
  1211. vm['created'] = True
  1212. # notify about changes
  1213. node.vms_conf.changed()
  1214. db().flush()
  1215. @classmethod
  1216. def get_network_roles(
  1217. cls, instance, merge_policy=NetworkRoleMergePolicy()):
  1218. """Method for receiving network roles for particular cluster
  1219. :param instance: nailgun.db.sqlalchemy.models.Cluster instance
  1220. :param merge_policy: the policy to merge same roles
  1221. :returns: List of network roles' descriptions
  1222. """
  1223. return PluginManager.get_network_roles(instance, merge_policy)
  1224. @classmethod
  1225. def set_network_template(cls, instance, template):
  1226. instance.network_config.configuration_template = template
  1227. cls.update_nodes_network_template(instance, instance.nodes)
  1228. db().flush()
  1229. if template is None:
  1230. net_manager = cls.get_network_manager(instance)
  1231. for node in instance.nodes:
  1232. net_manager.clear_bond_configuration(node)
  1233. net_manager.assign_networks_by_default(node)
  1234. @classmethod
  1235. def update_nodes_network_template(cls, instance, nodes):
  1236. from nailgun.objects import Node
  1237. template = instance.network_config.configuration_template
  1238. for node in nodes:
  1239. Node.apply_network_template(node, template)
  1240. @classmethod
  1241. def get_nodes_ids(cls, instance):
  1242. return [x[0] for x in db().query(models.Node.id).filter(
  1243. models.Node.cluster_id == instance.id).all()]
  1244. @classmethod
  1245. def get_vips(cls, instance):
  1246. net_roles = cls.get_network_roles(instance)
  1247. cluster_vips = []
  1248. for nr in net_roles:
  1249. cluster_vips.extend(nr['properties']['vip'])
  1250. return cluster_vips
  1251. @classmethod
  1252. def get_assigned_roles(cls, instance):
  1253. """Get list of all roles currently assigned to nodes in cluster
  1254. :param instance: nailgun.db.sqlalchemy.models.Cluster instance
  1255. :returns: List of node roles currently assigned
  1256. """
  1257. pending_roles = db().query(
  1258. sa.func.unnest(models.Node.pending_roles)
  1259. ).filter_by(
  1260. cluster_id=instance.id
  1261. ).distinct().all()
  1262. pending_roles = [pr[0] for pr in pending_roles]
  1263. roles = db().query(
  1264. sa.func.unnest(models.Node.roles)
  1265. ).filter_by(
  1266. cluster_id=instance.id
  1267. ).distinct().all()
  1268. roles = [r[0] for r in roles]
  1269. return set(pending_roles + roles)
  1270. @classmethod
  1271. def is_network_modification_locked(cls, instance):
  1272. """Checks whether network settings can be modified or deleted.
  1273. The result depends on the current status of cluster.
  1274. """
  1275. return instance.is_locked
  1276. @classmethod
  1277. def is_component_enabled(cls, instance, component):
  1278. """Checks is specified additional component enabled in cluster
  1279. :param instance: nailgun.db.sqlalchemy.models.Cluster instance
  1280. :param component: name of additional component
  1281. :returns: The result depends on current component status in settings
  1282. """
  1283. return bool(instance.attributes.editable['additional_components'].
  1284. get((component), {}).get('value'))
  1285. @classmethod
  1286. def get_nodes_to_update_config(cls, cluster, node_ids=None, node_role=None,
  1287. only_ready_nodes=True):
  1288. """Get nodes for specified cluster that should be updated.
  1289. Configuration update can be executed for all nodes in the cluster,
  1290. or for single node, or for all nodes with specified role.
  1291. If :param only_ready_nodes set by True function returns list of nodes
  1292. that will be updated during next config update execution.
  1293. If :param only_ready_nodes set by False function returns list of all
  1294. nodes that will finally get an updated configuration.
  1295. """
  1296. query = cls.get_nodes_not_for_deletion(cluster)
  1297. if only_ready_nodes:
  1298. query = query.filter_by(status=consts.NODE_STATUSES.ready)
  1299. if node_ids:
  1300. query = query.filter(models.Node.id.in_(node_ids))
  1301. elif node_role:
  1302. query = query.filter(
  1303. models.Node.roles.any(node_role))
  1304. return query.all()
  1305. @classmethod
  1306. def prepare_for_deployment(cls, instance, nodes=None):
  1307. """Shortcut for NetworkManager.prepare_for_deployment.
  1308. :param instance: nailgun.db.sqlalchemy.models.Cluster instance
  1309. :param nodes: the list of Nodes, None means for all nodes
  1310. """
  1311. cls.get_network_manager(instance).prepare_for_deployment(
  1312. instance, instance.nodes if nodes is None else nodes
  1313. )
  1314. @classmethod
  1315. def has_compute_vmware_changes(cls, instance):
  1316. """Checks if any 'compute-vmware' nodes are waiting for deployment.
  1317. :param instance: cluster for checking
  1318. :type instance: nailgun.db.sqlalchemy.models.Cluster instance
  1319. """
  1320. compute_vmware_nodes_query = db().query(models.Node).filter_by(
  1321. cluster_id=instance.id
  1322. ).filter(sa.or_(
  1323. sa.and_(models.Node.roles.any('compute-vmware'),
  1324. models.Node.pending_deletion),
  1325. models.Node.pending_roles.any('compute-vmware')
  1326. ))
  1327. return db().query(compute_vmware_nodes_query.exists()).scalar()
  1328. @classmethod
  1329. def get_operational_vmware_compute_nodes(cls, instance):
  1330. return db().query(models.Node).filter_by(
  1331. cluster_id=instance.id
  1332. ).filter(
  1333. models.Node.roles.any('compute-vmware'),
  1334. sa.not_(models.Node.pending_deletion)
  1335. ).all()
  1336. @classmethod
  1337. def is_task_deploy_enabled(cls, instance):
  1338. """Tests that task based deploy is enabled.
  1339. :param instance: cluster for checking
  1340. :type instance: nailgun.db.sqlalchemy.models.Cluster instance
  1341. """
  1342. attrs = cls.get_editable_attributes(instance)
  1343. return attrs['common'].get('task_deploy', {}).get('value')
  1344. @classmethod
  1345. def is_propagate_task_deploy_enabled(cls, instance):
  1346. """Tests that task based deployment propagation enabled.
  1347. :param instance: cluster for checking
  1348. :type instance: nailgun.db.sqlalchemy.models.Cluster instance
  1349. """
  1350. attrs = cls.get_editable_attributes(instance)
  1351. return attrs['common'].get('propagate_task_deploy', {}).get('value')
  1352. # FIXME(aroma): remove updating of 'deployed_before'
  1353. # when stop action is reworked. 'deployed_before'
  1354. # flag identifies whether stop action is allowed for the
  1355. # cluster. Please, refer to [1] for more details.
  1356. # [1]: https://bugs.launchpad.net/fuel/+bug/1529691
  1357. @classmethod
  1358. def set_deployed_before_flag(cls, instance, value):
  1359. """Change value for before_deployed if needed
  1360. :param instance: nailgun.db.sqlalchemy.models.Cluster instance
  1361. :param value: new value for flag
  1362. :type value: bool
  1363. """
  1364. generated = copy.deepcopy(instance.attributes.generated)
  1365. if 'deployed_before' not in generated:
  1366. # NOTE(aroma): this is needed for case when master node has
  1367. # been upgraded and there is attempt to re/deploy previously
  1368. # existing clusters. As long as setting the flag is temporary
  1369. # solution data base migration code should not be mangled
  1370. # in order to support it
  1371. generated['deployed_before'] = {'value': value}
  1372. elif generated['deployed_before']['value'] != value:
  1373. generated['deployed_before']['value'] = value
  1374. instance.attributes.generated = generated
  1375. db.flush()
  1376. @classmethod
  1377. def get_nodes_count_unmet_status(cls, instance, status):
  1378. """Gets the number of nodes, that does not have specified status.
  1379. :param instance: nailgun.db.sqlalchemy.models.Cluster instance
  1380. :param status: the expected status
  1381. :return: the number of nodes that does not have specified status
  1382. """
  1383. q = db().query(models.Node).filter_by(cluster_id=instance.id)
  1384. return q.filter(models.Node.status != status).count()
  1385. @classmethod
  1386. def get_network_attributes(cls, instance):
  1387. # use local import to avoid recursive imports
  1388. from nailgun.extensions.network_manager.objects.serializers import \
  1389. network_configuration
  1390. if instance.net_provider == consts.CLUSTER_NET_PROVIDERS.nova_network:
  1391. serializer = \
  1392. network_configuration.NovaNetworkConfigurationSerializer
  1393. else:
  1394. serializer = \
  1395. network_configuration.NeutronNetworkConfigurationSerializer
  1396. return serializer.serialize_for_cluster(instance)
  1397. @classmethod
  1398. def get_restrictions_models(cls, instance, attrs=None):
  1399. """Return models which are used in restrictions mechanism
  1400. :param instance: nailgun.db.sqlalchemy.models.Cluster instance
  1401. :param attrs: models' settings will be overwritten with this value
  1402. :return: dict with models
  1403. """
  1404. return {
  1405. 'settings': attrs or cls.get_editable_attributes(instance),
  1406. 'cluster': instance,
  1407. 'version': settings.VERSION,
  1408. 'networking_parameters': instance.network_config,
  1409. }
  1410. @classmethod
  1411. def is_dpdk_supported_for_segmentation(cls, instance):
  1412. return (instance.network_config.segmentation_type in
  1413. (consts.NEUTRON_SEGMENT_TYPES.vlan,
  1414. consts.NEUTRON_SEGMENT_TYPES.tun))
  1415. class ClusterCollection(NailgunCollection):
  1416. """Cluster collection."""
  1417. #: Single Cluster object class
  1418. single = Cluster
  1419. class VmwareAttributes(NailgunObject):
  1420. model = models.VmwareAttributes
  1421. @staticmethod
  1422. def get_nova_computes_attrs(attributes):
  1423. return attributes.get('value', {}).get(
  1424. 'availability_zones', [{}])[0].get('nova_computes', [])
  1425. @classmethod
  1426. def get_nova_computes_target_nodes(cls, instance):
  1427. """Get data of targets node for all nova computes.
  1428. :param instance: nailgun.db.sqlalchemy.models.Cluster instance
  1429. :returns: list of dicts that represents nova compute targets
  1430. """
  1431. nova_compute_target_nodes = []
  1432. for nova_compute in cls.get_nova_computes_attrs(instance.editable):
  1433. target = nova_compute['target_node']['current']
  1434. if target['id'] != 'controllers':
  1435. nova_compute_target_nodes.append(target)
  1436. return nova_compute_target_nodes