Fuel UI
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

cluster.py 57KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622
  1. # -*- coding: utf-8 -*-
  2. # Copyright 2013 Mirantis, Inc.
  3. #
  4. # Licensed under the Apache License, Version 2.0 (the "License"); you may
  5. # not use this file except in compliance with the License. You may obtain
  6. # a copy of the License at
  7. #
  8. # http://www.apache.org/licenses/LICENSE-2.0
  9. #
  10. # Unless required by applicable law or agreed to in writing, software
  11. # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  12. # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  13. # License for the specific language governing permissions and limitations
  14. # under the License.
  15. """
  16. Cluster-related objects and collections
  17. """
  18. import copy
  19. from distutils.version import StrictVersion
  20. import itertools
  21. import six
  22. import sqlalchemy as sa
  23. from sqlalchemy.dialects import postgresql as psql
  24. from sqlalchemy.orm.exc import MultipleResultsFound
  25. from sqlalchemy.orm.exc import NoResultFound
  26. from nailgun import consts
  27. from nailgun.db import db
  28. from nailgun.db.sqlalchemy import models
  29. from nailgun import errors
  30. from nailgun.extensions import callback_wrapper
  31. from nailgun.extensions import fire_callback_on_cluster_create
  32. from nailgun.extensions import fire_callback_on_cluster_delete
  33. from nailgun.extensions import fire_callback_on_cluster_patch_attributes
  34. from nailgun.extensions import fire_callback_on_node_collection_delete
  35. from nailgun.logger import logger
  36. from nailgun.objects import DeploymentGraph
  37. from nailgun.objects import NailgunCollection
  38. from nailgun.objects import NailgunObject
  39. from nailgun.objects.plugin import ClusterPlugin
  40. from nailgun.objects import Release
  41. from nailgun.objects.serializers.cluster import ClusterSerializer
  42. from nailgun.plugins.manager import PluginManager
  43. from nailgun.policy.merge import NetworkRoleMergePolicy
  44. from nailgun.settings import settings
  45. from nailgun.utils import AttributesGenerator
  46. from nailgun.utils import dict_merge
  47. from nailgun.utils import dict_update
  48. from nailgun.utils import text_format_safe
  49. from nailgun.utils import traverse
  50. class Attributes(NailgunObject):
  51. """Cluster attributes object."""
  52. #: SQLAlchemy model for Cluster attributes
  53. model = models.Attributes
  54. @classmethod
  55. def generate_fields(cls, instance):
  56. """Generate field values for Cluster attributes using generators.
  57. :param instance: Attributes instance
  58. :returns: None
  59. """
  60. instance.generated = traverse(
  61. instance.generated or {},
  62. formatter_context={
  63. 'cluster': instance.cluster, 'settings': settings,
  64. },
  65. keywords={
  66. 'generator': AttributesGenerator.evaluate
  67. }
  68. )
  69. # TODO(ikalnitsky):
  70. #
  71. # Think about traversing "editable" attributes. It might be very
  72. # useful to generate default values for editable attribute at
  73. # cluster creation time.
  74. @classmethod
  75. def merged_attrs(cls, instance):
  76. """Generates merged dict of attributes
  77. Result includes generated Cluster attributes recursively updated
  78. by new values from editable attributes
  79. :param instance: Attributes instance
  80. :returns: dict of merged attributes
  81. """
  82. return dict_merge(
  83. instance.generated,
  84. instance.editable
  85. )
  86. @classmethod
  87. def merged_attrs_values(cls, instance):
  88. """Transforms raw dict of attributes into dict of facts
  89. Raw dict is taken from :func:`merged_attrs`
  90. The result of this function is a dict of facts that wil be sent to
  91. orchestrator
  92. :param instance: Attributes instance
  93. :returns: dict of merged attributes
  94. """
  95. attrs = cls.merged_attrs(instance)
  96. for group_attrs in six.itervalues(attrs):
  97. for attr, value in six.iteritems(group_attrs):
  98. if isinstance(value, dict) and 'value' in value:
  99. group_attrs[attr] = value['value']
  100. if 'common' in attrs:
  101. attrs.update(attrs.pop('common'))
  102. if 'additional_components' in attrs:
  103. for comp, enabled in six.iteritems(attrs['additional_components']):
  104. if isinstance(enabled, bool):
  105. attrs.setdefault(comp, {}).update({
  106. "enabled": enabled
  107. })
  108. attrs.pop('additional_components')
  109. return attrs
  110. class Cluster(NailgunObject):
  111. """Cluster object."""
  112. #: SQLAlchemy model for Cluster
  113. model = models.Cluster
  114. #: Serializer for Cluster
  115. serializer = ClusterSerializer
  116. @classmethod
  117. def create(cls, data):
  118. """Create Cluster instance with specified parameters in DB.
  119. This includes:
  120. * creating Cluster attributes and generating default values \
  121. (see :func:`create_attributes`)
  122. * creating NetworkGroups for Cluster
  123. * adding default pending changes (see :func:`add_pending_changes`)
  124. * if "nodes" are specified in data then they are added to Cluster \
  125. (see :func:`update_nodes`)
  126. :param data: dictionary of key-value pairs as object fields
  127. :returns: Cluster instance
  128. """
  129. # TODO(enchantner): fix this temporary hack in clients
  130. if "release_id" not in data:
  131. release_id = data.pop("release", None)
  132. data["release_id"] = release_id
  133. # remove read-only attribute
  134. data.pop("is_locked", None)
  135. assign_nodes = data.pop("nodes", [])
  136. enabled_editable_attributes = None
  137. if 'components' in data:
  138. enabled_core_attributes = cls.get_cluster_attributes_by_components(
  139. data['components'], data["release_id"])
  140. data = dict_merge(data, enabled_core_attributes['cluster'])
  141. enabled_editable_attributes = enabled_core_attributes['editable']
  142. data["fuel_version"] = settings.VERSION["release"]
  143. deployment_tasks = data.pop("deployment_tasks", [])
  144. cluster = super(Cluster, cls).create(data)
  145. cls.create_default_group(cluster)
  146. cls.create_attributes(cluster, enabled_editable_attributes)
  147. cls.create_default_extensions(cluster)
  148. # default graph should be created in any case
  149. DeploymentGraph.create_for_model({"tasks": deployment_tasks}, cluster)
  150. cls.add_pending_changes(
  151. cluster, consts.CLUSTER_CHANGES.attributes)
  152. ClusterPlugin.add_compatible_plugins(cluster)
  153. PluginManager.enable_plugins_by_components(cluster)
  154. fire_callback_on_cluster_create(cluster, data)
  155. if assign_nodes:
  156. cls.update_nodes(cluster, assign_nodes)
  157. return cluster
  158. @classmethod
  159. def get_cluster_attributes_by_components(cls, components, release_id):
  160. """Enable cluster attributes by given components
  161. :param components: list of component names
  162. :type components: list of strings
  163. :param release_id: Release model id
  164. :type release_id: str
  165. :returns: dict -- objects with enabled attributes for cluster
  166. """
  167. def _update_attributes_dict_by_binds_exp(bind_exp, value):
  168. """Update cluster and attributes data with bound values
  169. :param bind_exp: path to specific attribute for model in format
  170. model:some.attribute.value. Model can be
  171. settings|cluster
  172. :type bind_exp: str
  173. :param value: value for specific attribute
  174. :type value: bool|str|int
  175. :returns: None
  176. """
  177. model, attr_expr = bind_exp.split(':')
  178. if model not in ('settings', 'cluster'):
  179. return
  180. path_items = attr_expr.split('.')
  181. path_items.insert(0, model)
  182. attributes = cluster_attributes
  183. for i in six.moves.range(0, len(path_items) - 1):
  184. attributes = attributes.setdefault(path_items[i], {})
  185. attributes[path_items[-1]] = value
  186. release = Release.get_by_uid(release_id)
  187. cluster_attributes = {}
  188. for component in Release.get_all_components(release):
  189. if component['name'] in components:
  190. for bind_item in component.get('bind', []):
  191. if isinstance(bind_item, six.string_types):
  192. _update_attributes_dict_by_binds_exp(bind_item, True)
  193. elif isinstance(bind_item, list):
  194. _update_attributes_dict_by_binds_exp(bind_item[0],
  195. bind_item[1])
  196. return {
  197. 'editable': cluster_attributes.get('settings', {}),
  198. 'cluster': cluster_attributes.get('cluster', {})
  199. }
  200. @classmethod
  201. @callback_wrapper('cluster_delete', ['instance'])
  202. def delete(cls, instance):
  203. """Delete cluster.
  204. :param instance: Cluster model instance
  205. :type instance: models.Cluster
  206. """
  207. DeploymentGraph.delete_for_parent(instance)
  208. node_ids = [
  209. _id for (_id,) in
  210. db().query(models.Node.id).
  211. filter_by(cluster_id=instance.id).
  212. order_by(models.Node.id)]
  213. fire_callback_on_node_collection_delete(node_ids)
  214. fire_callback_on_cluster_delete(instance)
  215. super(Cluster, cls).delete(instance)
  216. @classmethod
  217. def get_default_kernel_params(cls, instance):
  218. kernel_params = instance.attributes.editable.get("kernel_params", {})
  219. return kernel_params.get("kernel", {}).get("value")
  220. @classmethod
  221. def create_attributes(cls, instance, editable_attributes=None):
  222. """Create attributes for Cluster instance, generate their values
  223. (see :func:`Attributes.generate_fields`)
  224. :param instance: Cluster instance
  225. :param editable_attributes: key-value dictionary represents editable
  226. attributes that will be merged with default editable attributes
  227. :returns: None
  228. """
  229. merged_editable_attributes = \
  230. cls.get_default_editable_attributes(instance)
  231. if editable_attributes:
  232. merged_editable_attributes = dict_merge(
  233. merged_editable_attributes, editable_attributes)
  234. attributes = Attributes.create(
  235. {
  236. "editable": merged_editable_attributes,
  237. "generated": instance.release.attributes_metadata.get(
  238. "generated"
  239. ),
  240. "cluster_id": instance.id
  241. }
  242. )
  243. Attributes.generate_fields(attributes)
  244. db().flush()
  245. return attributes
  246. @classmethod
  247. def create_default_extensions(cls, instance):
  248. """Sets default extensions list from release model
  249. :param instance: Cluster instance
  250. :returns: None
  251. """
  252. instance.extensions = instance.release.extensions
  253. db().flush()
  254. @classmethod
  255. def get_default_editable_attributes(cls, instance):
  256. """Get editable attributes from release metadata
  257. :param instance: Cluster instance
  258. :returns: Dict object
  259. """
  260. editable = instance.release.attributes_metadata.get("editable")
  261. # Add default attributes of connected plugins
  262. plugin_attrs = PluginManager.get_plugins_attributes(
  263. instance, all_versions=True, default=True)
  264. editable = dict(plugin_attrs, **editable)
  265. editable = traverse(
  266. editable,
  267. formatter_context={'cluster': instance, 'settings': settings},
  268. keywords={'generator': AttributesGenerator.evaluate}
  269. )
  270. return editable
  271. @classmethod
  272. def get_attributes(cls, instance, all_plugins_versions=False):
  273. """Get attributes for current Cluster instance.
  274. :param instance: Cluster instance
  275. :param all_plugins_versions: Get attributes of all versions of plugins
  276. :returns: dict
  277. """
  278. try:
  279. attrs = db().query(models.Attributes).filter(
  280. models.Attributes.cluster_id == instance.id
  281. ).one()
  282. except MultipleResultsFound:
  283. raise errors.InvalidData(
  284. u"Multiple rows with attributes were found for cluster '{0}'"
  285. .format(instance.name)
  286. )
  287. except NoResultFound:
  288. raise errors.InvalidData(
  289. u"No attributes were found for cluster '{0}'"
  290. .format(instance.name)
  291. )
  292. attrs = copy.deepcopy(attrs)
  293. # Merge plugins attributes into editable ones
  294. plugin_attrs = PluginManager.get_plugins_attributes(
  295. instance, all_versions=all_plugins_versions)
  296. plugin_attrs = traverse(
  297. plugin_attrs,
  298. formatter=text_format_safe,
  299. formatter_context={'cluster': instance, 'settings': settings},
  300. keywords={'generator': AttributesGenerator.evaluate}
  301. )
  302. attrs['editable'].update(plugin_attrs)
  303. return attrs
  304. @classmethod
  305. def get_editable_attributes(cls, instance, all_plugins_versions=False):
  306. """Get editable attributes for current Cluster instance.
  307. :param instance: Cluster instance
  308. :param all_plugins_versions: Get attributes of all versions of plugins
  309. :return: dict
  310. """
  311. return cls.get_attributes(instance, all_plugins_versions)['editable']
  312. @classmethod
  313. def update_attributes(cls, instance, data):
  314. PluginManager.process_cluster_attributes(instance, data['editable'])
  315. for key, value in six.iteritems(data):
  316. setattr(instance.attributes, key, value)
  317. cls.add_pending_changes(instance, "attributes")
  318. db().flush()
  319. @classmethod
  320. def update_role(cls, instance, role):
  321. """Update existing Cluster instance with specified role.
  322. Previous ones are deleted.
  323. :param instance: a Cluster instance
  324. :param role: a role dict
  325. :returns: None
  326. """
  327. old_role = instance.roles_metadata.get(role['name'], {})
  328. instance.roles_metadata[role['name']] = role['meta']
  329. if old_role:
  330. deleted_tags = set(old_role['tags']) - set(role['meta']['tags'])
  331. for tag in deleted_tags:
  332. cls.remove_primary_tag(instance, tag)
  333. instance.volumes_metadata.setdefault(
  334. 'volumes_roles_mapping', {}).update(
  335. {role['name']: role.get('volumes_roles_mapping', [])})
  336. # notify about changes
  337. instance.volumes_metadata.changed()
  338. @classmethod
  339. def remove_role(cls, instance, role_name):
  340. result = instance.roles_metadata.pop(role_name, None)
  341. instance.volumes_metadata['volumes_roles_mapping'].pop(role_name, None)
  342. # notify about changes
  343. instance.volumes_metadata.changed()
  344. return bool(result)
  345. @classmethod
  346. def update_tag(cls, instance, tag):
  347. """Update existing Cluster instance with specified tag.
  348. Previous ones are deleted.
  349. :param instance: a Cluster instance
  350. :param tag: a tag dict
  351. :returns: None
  352. """
  353. instance.tags_metadata[tag['name']] = tag['meta']
  354. @classmethod
  355. def remove_tag(cls, instance, tag_name):
  356. res = instance.tags_metadata.pop(tag_name, None)
  357. if tag_name not in instance.release.tags_metadata:
  358. cls.remove_tag_from_roles(instance, tag_name)
  359. cls.remove_primary_tag(instance, tag_name)
  360. return bool(res)
  361. @classmethod
  362. def remove_tag_from_roles(cls, instance, tag_name):
  363. for role, meta in six.iteritems(cls.get_own_roles(instance)):
  364. tags = meta.get('tags', [])
  365. if tag_name in tags:
  366. tags.remove(tag_name)
  367. instance.roles_metadata.changed()
  368. @classmethod
  369. def _create_public_map(cls, instance, roles_metadata=None):
  370. if instance.network_config.configuration_template is not None:
  371. return
  372. from nailgun import objects
  373. public_map = {}
  374. for node in instance.nodes:
  375. public_map[node.id] = objects.Node.should_have_public(
  376. node, roles_metadata)
  377. return public_map
  378. @classmethod
  379. def patch_attributes(cls, instance, data):
  380. """Applyes changes to Cluster attributes and updates networks.
  381. :param instance: Cluster object
  382. :param data: dict
  383. """
  384. roles_metadata = Cluster.get_roles(instance)
  385. # Note(kszukielojc): We need to create status map of public networks
  386. # to avoid updating networks if there was no change to node public
  387. # network after patching attributes
  388. public_map = cls._create_public_map(instance, roles_metadata)
  389. PluginManager.process_cluster_attributes(instance, data['editable'])
  390. instance.attributes.editable = dict_merge(
  391. instance.attributes.editable, data['editable'])
  392. cls.add_pending_changes(instance, "attributes")
  393. fire_callback_on_cluster_patch_attributes(instance, public_map)
  394. db().flush()
  395. @classmethod
  396. def get_updated_editable_attributes(cls, instance, data):
  397. """Same as get_editable_attributes but also merges given data.
  398. :param instance: Cluster object
  399. :param data: dict
  400. :returns: dict
  401. """
  402. attributes = {'editable': dict_merge(
  403. cls.get_editable_attributes(instance),
  404. data.get('editable', {})
  405. )}
  406. # plugin attributes should be updated with values to provide
  407. # consitency between new data and plugin attributes data from DB
  408. PluginManager.inject_plugin_attribute_values(attributes['editable'])
  409. return attributes
  410. @classmethod
  411. def get_network_manager(cls, instance=None):
  412. """Get network manager for Cluster instance.
  413. If instance is None then the default NetworkManager is returned
  414. :param instance: Cluster instance
  415. :returns: NetworkManager/NovaNetworkManager/NeutronManager
  416. """
  417. if not instance:
  418. from nailgun.extensions.network_manager.manager import \
  419. NetworkManager
  420. return NetworkManager
  421. ver = instance.release.environment_version
  422. net_provider = instance.net_provider
  423. if net_provider == consts.CLUSTER_NET_PROVIDERS.neutron:
  424. from nailgun.extensions.network_manager.managers import neutron
  425. if StrictVersion(ver) < StrictVersion('6.1'):
  426. return neutron.NeutronManagerLegacy
  427. if StrictVersion(ver) == StrictVersion('6.1'):
  428. return neutron.NeutronManager61
  429. if StrictVersion(ver) == StrictVersion('7.0'):
  430. return neutron.NeutronManager70
  431. if StrictVersion(ver) >= StrictVersion('8.0'):
  432. return neutron.NeutronManager80
  433. return neutron.NeutronManager
  434. elif net_provider == consts.CLUSTER_NET_PROVIDERS.nova_network:
  435. from nailgun.extensions.network_manager.managers import \
  436. nova_network
  437. if StrictVersion(ver) < StrictVersion('6.1'):
  438. return nova_network.NovaNetworkManagerLegacy
  439. if StrictVersion(ver) == StrictVersion('6.1'):
  440. return nova_network.NovaNetworkManager61
  441. if StrictVersion(ver) == StrictVersion('7.0'):
  442. return nova_network.NovaNetworkManager70
  443. if StrictVersion(ver) >= StrictVersion('8.0'):
  444. raise errors.NovaNetworkNotSupported()
  445. return nova_network.NovaNetworkManager
  446. raise ValueError(
  447. 'The network provider "{0}" is not supported.'
  448. .format(net_provider)
  449. )
  450. @classmethod
  451. def add_pending_changes(cls, instance, changes_type, node_id=None):
  452. """Add pending changes for current Cluster.
  453. If node_id is specified then links created changes with node.
  454. :param instance: Cluster instance
  455. :param changes_type: name of changes to add
  456. :param node_id: node id for changes
  457. :returns: None
  458. """
  459. logger.debug(
  460. u"New pending changes in environment {0}: {1}{2}".format(
  461. instance.id,
  462. changes_type,
  463. u" node_id={0}".format(node_id) if node_id else u""
  464. )
  465. )
  466. # TODO(enchantner): check if node belongs to cluster
  467. ex_chs = db().query(models.ClusterChanges).filter_by(
  468. cluster=instance,
  469. name=changes_type
  470. )
  471. if not node_id:
  472. ex_chs = ex_chs.first()
  473. else:
  474. ex_chs = ex_chs.filter_by(node_id=node_id).first()
  475. # do nothing if changes with the same name already pending
  476. if ex_chs:
  477. return
  478. ch = models.ClusterChanges(
  479. cluster_id=instance.id,
  480. name=changes_type
  481. )
  482. if node_id:
  483. ch.node_id = node_id
  484. db().add(ch)
  485. db().flush()
  486. @classmethod
  487. def get_nodes_not_for_deletion(cls, cluster):
  488. """All clusters nodes except nodes for deletion."""
  489. return db().query(models.Node).filter_by(
  490. cluster=cluster, pending_deletion=False).order_by(models.Node.id)
  491. @classmethod
  492. def clear_pending_changes(cls, instance, node_id=None):
  493. """Clear pending changes for current Cluster.
  494. If node_id is specified then only clears changes connected
  495. to this node.
  496. :param instance: Cluster instance
  497. :param node_id: node id for changes
  498. :returns: None
  499. """
  500. logger.debug(
  501. u"Removing pending changes in environment {0}{1}".format(
  502. instance.id,
  503. u" where node_id={0}".format(node_id) if node_id else u""
  504. )
  505. )
  506. chs = db().query(models.ClusterChanges).filter_by(
  507. cluster_id=instance.id
  508. )
  509. if node_id:
  510. chs = chs.filter_by(node_id=node_id)
  511. for ch in chs.all():
  512. db().delete(ch)
  513. db().flush()
  514. @classmethod
  515. def update(cls, instance, data):
  516. """Update Cluster object instance with specified parameters in DB
  517. If "nodes" are specified in data then they will replace existing ones
  518. (see :func:`update_nodes`)
  519. :param instance: Cluster instance
  520. :param data: dictionary of key-value pairs as object fields
  521. :returns: Cluster instance
  522. """
  523. # remove read-only attributes
  524. data.pop("fuel_version", None)
  525. data.pop("is_locked", None)
  526. nodes = data.pop("nodes", None)
  527. changes = data.pop("changes", None)
  528. deployment_tasks = data.pop("deployment_tasks", None)
  529. super(Cluster, cls).update(instance, data)
  530. if deployment_tasks:
  531. deployment_graph_instance = DeploymentGraph.get_for_model(instance)
  532. DeploymentGraph.update(
  533. deployment_graph_instance, {"tasks": deployment_tasks})
  534. if nodes is not None:
  535. cls.update_nodes(instance, nodes)
  536. if changes is not None:
  537. cls.update_changes(instance, changes)
  538. return instance
  539. @classmethod
  540. def update_nodes(cls, instance, nodes_ids):
  541. """Update Cluster nodes by specified node IDs
  542. Nodes with specified IDs will replace existing ones in Cluster
  543. :param instance: Cluster instance
  544. :param nodes_ids: list of nodes ids
  545. :returns: None
  546. """
  547. from nailgun import objects
  548. # TODO(NAME): sepatate nodes
  549. # for deletion and addition by set().
  550. new_nodes = []
  551. if nodes_ids:
  552. new_nodes = db().query(models.Node).filter(
  553. models.Node.id.in_(nodes_ids)
  554. )
  555. nodes_to_remove = [n for n in instance.nodes
  556. if n not in new_nodes]
  557. nodes_to_add = [n for n in new_nodes
  558. if n not in instance.nodes]
  559. for node in nodes_to_add:
  560. if not node.online:
  561. raise errors.NodeOffline(
  562. u"Cannot add offline node "
  563. u"'{0}' to environment".format(node.id)
  564. )
  565. # we should reset hostname to default value to guarantee
  566. # hostnames uniqueness for nodes outside clusters
  567. for node in nodes_to_remove:
  568. node.hostname = objects.Node.default_slave_name(node)
  569. instance.nodes.remove(node)
  570. for node in nodes_to_add:
  571. instance.nodes.append(node)
  572. net_manager = cls.get_network_manager(instance)
  573. for node in nodes_to_remove:
  574. net_manager.clear_assigned_networks(node)
  575. net_manager.clear_bond_configuration(node)
  576. cls.replace_provisioning_info_on_nodes(instance, {}, nodes_to_remove)
  577. cls.replace_deployment_info_on_nodes(instance, {}, nodes_to_remove)
  578. objects.NodeCollection.reset_network_template(nodes_to_remove)
  579. objects.NodeCollection.reset_attributes(nodes_to_remove)
  580. objects.OpenstackConfig.disable_by_nodes(nodes_to_remove)
  581. for node in nodes_to_add:
  582. objects.Node.assign_group(node)
  583. net_manager.assign_networks_by_default(node)
  584. objects.Node.set_default_attributes(node)
  585. objects.Node.create_nic_attributes(node)
  586. objects.Node.refresh_dpdk_properties(node)
  587. cls.update_nodes_network_template(instance, nodes_to_add)
  588. db().flush()
  589. @classmethod
  590. def update_changes(cls, instance, changes):
  591. instance.changes_list = [
  592. models.ClusterChanges(**change) for change in changes
  593. ]
  594. db().flush()
  595. @classmethod
  596. def get_ifaces_for_network_in_cluster(cls, cluster, net):
  597. """Method for receiving node_id:iface pairs for all nodes in cluster
  598. :param instance: Cluster instance
  599. :param net: Nailgun specific network name
  600. :type net: str
  601. :returns: List of node_id, iface pairs for all nodes in cluster.
  602. """
  603. nics_db = db().query(
  604. models.NodeNICInterface.node_id,
  605. models.NodeNICInterface.name
  606. ).filter(
  607. models.NodeNICInterface.node.has(cluster_id=cluster.id),
  608. models.NodeNICInterface.assigned_networks_list.any(name=net)
  609. )
  610. bonds_db = db().query(
  611. models.NodeBondInterface.node_id,
  612. models.NodeBondInterface.name
  613. ).filter(
  614. models.NodeBondInterface.node.has(cluster_id=cluster.id),
  615. models.NodeBondInterface.assigned_networks_list.any(name=net)
  616. )
  617. return nics_db.union(bonds_db)
  618. @classmethod
  619. def replace_provisioning_info_on_nodes(cls, instance, data, nodes):
  620. if isinstance(data, list):
  621. data = {n.get('uid'): n for n in data}
  622. for node in nodes:
  623. node.replaced_provisioning_info = data.get(node.uid, {})
  624. @classmethod
  625. def replace_deployment_info_on_nodes(cls, instance, data, nodes):
  626. if isinstance(data, list):
  627. data = {n.get('uid'): n for n in data}
  628. for node in nodes:
  629. node_data = data.get(node.uid, [])
  630. # replaced deployment info for node should be list
  631. # because before in previous versions of nailgun
  632. # node info will be per role, not per node
  633. if isinstance(node_data, dict):
  634. node_data = [node_data]
  635. node.replaced_deployment_info = node_data
  636. @classmethod
  637. def replace_provisioning_info(cls, instance, data):
  638. received_nodes = data.pop('nodes', [])
  639. instance.is_customized = True
  640. instance.replaced_provisioning_info = data
  641. cls.replace_provisioning_info_on_nodes(
  642. instance, received_nodes, instance.nodes)
  643. return cls.get_provisioning_info(instance)
  644. @classmethod
  645. def replace_deployment_info(cls, instance, data):
  646. instance.is_customized = True
  647. instance.replaced_deployment_info = data.get('common', {})
  648. cls.replace_deployment_info_on_nodes(
  649. instance, data.get('nodes', {}), instance.nodes
  650. )
  651. return cls.get_deployment_info(instance)
  652. @classmethod
  653. def get_provisioning_info(cls, instance):
  654. data = {}
  655. if instance.replaced_provisioning_info:
  656. data.update(instance.replaced_provisioning_info)
  657. nodes = []
  658. for node in instance.nodes:
  659. if node.replaced_provisioning_info:
  660. nodes.append(node.replaced_provisioning_info)
  661. if data:
  662. data['nodes'] = nodes
  663. return data
  664. @classmethod
  665. def get_deployment_info(cls, instance):
  666. nodes = []
  667. for node in instance.nodes:
  668. if node.replaced_deployment_info:
  669. nodes.extend(node.replaced_deployment_info)
  670. data = {}
  671. if nodes:
  672. data['nodes'] = nodes
  673. if instance.replaced_deployment_info:
  674. data['common'] = instance.replaced_deployment_info
  675. return data
  676. @classmethod
  677. def get_creds(cls, instance):
  678. return instance.attributes.editable['access']
  679. @classmethod
  680. def should_assign_public_to_all_nodes(cls, instance):
  681. """Check if Public network is to be assigned to all nodes in cluster
  682. :param instance: cluster instance
  683. :returns: True when Public network is to be assigned to all nodes
  684. """
  685. if instance.net_provider == \
  686. consts.CLUSTER_NET_PROVIDERS.nova_network:
  687. return True
  688. assignment = instance.attributes.editable.get(
  689. 'public_network_assignment')
  690. if not assignment or assignment['assign_to_all_nodes']['value']:
  691. return True
  692. return False
  693. @classmethod
  694. def neutron_dvr_enabled(cls, instance):
  695. neutron_attrs = instance.attributes.editable.get(
  696. 'neutron_advanced_configuration')
  697. if neutron_attrs:
  698. return neutron_attrs['neutron_dvr']['value']
  699. else:
  700. return False
  701. @classmethod
  702. def dpdk_enabled(cls, instance):
  703. # Had to do this due to issues with modules imports in current
  704. # nailgun __init__.py which cannot be resolved easily
  705. from nailgun.objects import Node
  706. if Release.is_nfv_supported(instance.release):
  707. for node in cls.get_nodes_not_for_deletion(instance):
  708. if Node.dpdk_enabled(node):
  709. return True
  710. return False
  711. @classmethod
  712. def get_roles(cls, instance):
  713. """Returns a dictionary of node roles available for deployment.
  714. :param instance: cluster instance
  715. :returns: a dictionary of roles metadata
  716. """
  717. available_roles = copy.deepcopy(cls.get_own_roles(instance.release))
  718. available_roles.update(cls.get_own_roles(instance))
  719. available_roles.update(
  720. PluginManager.get_plugins_node_roles(instance))
  721. return available_roles
  722. @classmethod
  723. def get_roles_by_tag(cls, tag_name, instance):
  724. roles = set()
  725. for role, meta in six.iteritems(cls.get_own_roles(instance)):
  726. if tag_name in meta.get('tags', {}):
  727. roles.add(role)
  728. return roles
  729. @classmethod
  730. def get_own_roles(cls, instance):
  731. return instance.roles_metadata
  732. @classmethod
  733. def get_own_tags(cls, instance):
  734. return instance.tags_metadata
  735. @classmethod
  736. def remove_primary_tag(cls, instance, tag):
  737. node = cls.get_primary_node(instance, tag)
  738. if node:
  739. node.primary_tags.remove(tag)
  740. @classmethod
  741. def set_primary_tag(cls, instance, nodes, tag):
  742. """Method for assigning primary attribute for specific tag.
  743. :param instance: Cluster db objects
  744. :param nodes: list of Node db objects
  745. :param tag: string with known tag name
  746. """
  747. from nailgun.objects import Node
  748. node = cls.get_primary_node(instance, tag)
  749. if not node:
  750. # get nodes with a given role name which are not going to be
  751. # removed
  752. filtered_nodes = []
  753. for node in nodes:
  754. if (not node.pending_deletion and (
  755. tag in Node.get_tags(node))):
  756. filtered_nodes.append(node)
  757. filtered_nodes = sorted(filtered_nodes, key=lambda node: node.id)
  758. if filtered_nodes:
  759. primary_node = next((
  760. node for node in filtered_nodes
  761. if node.status == consts.NODE_STATUSES.ready),
  762. filtered_nodes[0])
  763. primary_node.primary_tags.append(tag)
  764. db().flush()
  765. @classmethod
  766. def set_primary_tags(cls, instance, nodes):
  767. """Assignment of all primary attribute for all tags that requires it.
  768. This method is idempotent
  769. To mark tag as primary add "has_primary: true" attribute to tag meta
  770. :param instance: Cluster db object
  771. :param nodes: list of Node db objects
  772. """
  773. if not instance.is_ha_mode:
  774. return
  775. tags_meta = cls.get_tags_metadata(instance)
  776. for role, meta in six.iteritems(cls.get_roles(instance)):
  777. for tag in meta.get('tags', []):
  778. if tags_meta[tag].get('has_primary'):
  779. cls.set_primary_tag(instance, nodes, tag)
  780. @classmethod
  781. def get_nodes_by_role(cls, instance, role_name):
  782. """Get nodes related to some specific role
  783. :param instance: cluster db object
  784. :type: python object
  785. :param role_name: node role name
  786. :type: string
  787. """
  788. from nailgun.objects import Node
  789. if role_name not in cls.get_roles(instance):
  790. logger.warning("%s role doesn't exist", role_name)
  791. return []
  792. return Node.get_nodes_by_role([instance.id], role_name).all()
  793. @classmethod
  794. def get_node_by_role(cls, instance, role_name):
  795. from nailgun.objects import Node
  796. return Node.get_nodes_by_role([instance.id], role_name).first()
  797. @classmethod
  798. def get_nodes_by_status(cls, instance, status, exclude=None):
  799. """Get cluster nodes with particular status
  800. :param instance: cluster instance
  801. :param status: node status
  802. :param exclude: the list of uids to exclude
  803. :return: filtered query on nodes
  804. """
  805. query = db().query(models.Node).filter_by(
  806. cluster_id=instance.id,
  807. status=status
  808. )
  809. if exclude:
  810. query = query.filter(sa.not_(models.Node.id.in_(exclude)))
  811. return query
  812. @classmethod
  813. def get_primary_node(cls, instance, tag):
  814. """Get primary node for tag
  815. If primary node is not found None will be returned
  816. :param instance: cluster db object
  817. :type: python object
  818. :param tag: node tag name
  819. :type: string
  820. :returns: node db object or None
  821. """
  822. logger.debug("Getting primary node for tag: %s", tag)
  823. primary_node = db().query(models.Node).filter_by(
  824. pending_deletion=False,
  825. cluster_id=instance.id
  826. ).filter(
  827. models.Node.primary_tags.any(tag)
  828. ).first()
  829. if primary_node is None:
  830. logger.debug("Not found primary node for tag: %s", tag)
  831. else:
  832. logger.debug("Found primary node: %s for tag: %s",
  833. primary_node.id, tag)
  834. return primary_node
  835. @classmethod
  836. def get_controllers_group_id(cls, instance):
  837. return cls.get_controllers_node_group(instance).id
  838. @classmethod
  839. def get_controllers_node_group(cls, instance):
  840. return cls.get_common_node_group(instance, ['controller'])
  841. @classmethod
  842. def get_common_node_group(cls, instance, noderoles):
  843. """Returns a common node group for a given node roles.
  844. If a given node roles have different node groups, the error
  845. will be raised, so it's mandatory to have them the same
  846. node group.
  847. :param instance: a Cluster instance
  848. :param noderoles: a list of node roles
  849. :returns: a common NodeGroup instance
  850. """
  851. nodegroups = cls.get_node_groups(instance, noderoles).all()
  852. if not nodegroups:
  853. return None
  854. if len(nodegroups) > 1:
  855. raise errors.CanNotFindCommonNodeGroup(
  856. 'Node roles [{0}] has more than one common node group'.format(
  857. ', '.join(noderoles)))
  858. return nodegroups[0]
  859. @classmethod
  860. def get_node_groups(cls, instance, noderoles):
  861. """Returns node groups for given node roles.
  862. :param instance: a Cluster instance
  863. :param noderoles: a list of node roles
  864. :returns: a query for list of NodeGroup instances
  865. """
  866. psql_noderoles = sa.cast(
  867. psql.array(noderoles),
  868. psql.ARRAY(sa.String(consts.ROLE_NAME_MAX_SIZE)))
  869. nodegroups = db().query(models.NodeGroup).join(models.Node).filter(
  870. models.Node.cluster_id == instance.id,
  871. models.Node.pending_deletion.is_(False)
  872. ).filter(sa.or_(
  873. models.Node.roles.overlap(psql_noderoles),
  874. models.Node.pending_roles.overlap(psql_noderoles)
  875. ))
  876. return nodegroups
  877. @classmethod
  878. def get_default_group(cls, instance):
  879. return next(g for g in instance.node_groups if g.is_default)
  880. @classmethod
  881. def create_default_group(cls, instance):
  882. node_group = models.NodeGroup(name=consts.NODE_GROUPS.default,
  883. is_default=True)
  884. instance.node_groups.append(node_group)
  885. db.add(node_group)
  886. db().flush()
  887. return node_group
  888. @classmethod
  889. def get_own_deployment_graph(
  890. cls, instance, graph_type=None):
  891. """Return only cluster own deployment graph.
  892. :param instance: models.Cluster instance
  893. :type instance: models.Cluster
  894. :param graph_type: deployment graph type
  895. :type graph_type: basestring|None
  896. :return: deployment tasks list
  897. :rtype: list[dict]
  898. """
  899. cluster_deployment_graph = DeploymentGraph.get_for_model(
  900. instance, graph_type=graph_type)
  901. if cluster_deployment_graph:
  902. graph_metadata = DeploymentGraph.get_metadata(
  903. cluster_deployment_graph
  904. )
  905. graph_metadata['tasks'] = DeploymentGraph.get_tasks(
  906. cluster_deployment_graph
  907. )
  908. else:
  909. graph_metadata = {'tasks': []}
  910. return graph_metadata
  911. @classmethod
  912. def get_own_deployment_tasks(
  913. cls, instance, graph_type=None):
  914. """Return only cluster own deployment graph.
  915. :param instance: models.Cluster instance
  916. :type instance: models.Cluster
  917. :param graph_type: deployment graph type
  918. :type graph_type: basestring|None
  919. :return: deployment tasks list
  920. :rtype: list[dict]
  921. """
  922. return cls.get_own_deployment_graph(instance, graph_type)['tasks']
  923. @classmethod
  924. def get_plugins_deployment_graph(
  925. cls, instance, graph_type=None):
  926. """Get merged deployment tasks for plugins enabled to given cluster.
  927. :param instance: models.Cluster instance
  928. :type instance: models.Cluster
  929. :param graph_type: deployment graph type
  930. :type graph_type: basestring|None
  931. :return: deployment tasks list
  932. :rtype: list[dict]
  933. """
  934. return PluginManager.get_plugins_deployment_graph(
  935. instance, graph_type=graph_type)
  936. @classmethod
  937. def get_plugins_deployment_tasks(
  938. cls, instance, graph_type=None):
  939. """Get merged deployment tasks for plugins enabled to given cluster.
  940. :param instance: models.Cluster instance
  941. :type instance: models.Cluster
  942. :param graph_type: deployment graph type
  943. :type graph_type: basestring|None
  944. :return: deployment tasks list
  945. :rtype: list[dict]
  946. """
  947. return PluginManager.get_plugins_deployment_tasks(
  948. instance, graph_type=graph_type)
  949. @classmethod
  950. def get_release_deployment_graph(
  951. cls, instance, graph_type=None):
  952. """Get merged deployment graph for release related to the cluster.
  953. :param instance: models.Cluster instance
  954. :type instance: models.Cluster
  955. :param graph_type: deployment graph type
  956. :type graph_type: basestring|None
  957. :return: deployment tasks list
  958. :rtype: list[dict]
  959. """
  960. return Release.get_deployment_graph(
  961. instance.release, graph_type=graph_type)
  962. @classmethod
  963. def get_release_deployment_tasks(
  964. cls, instance, graph_type=None):
  965. """Get merged deployment tasks for release related to the cluster.
  966. :param instance: models.Cluster instance
  967. :type instance: models.Cluster
  968. :param graph_type: deployment graph type
  969. :type graph_type: basestring|None
  970. :return: deployment tasks list
  971. :rtype: list[dict]
  972. """
  973. return Release.get_deployment_tasks(
  974. instance.release, graph_type=graph_type)
  975. @classmethod
  976. def _merge_tasks_lists(cls, tasks_lists):
  977. """Merge several tasks lists.
  978. Every next list will override tasks in previous one by `task_name` key.
  979. :param tasks_lists: tasks lists is order of increasing priority
  980. task from next will override task in previous
  981. if ID is same
  982. :type tasks_lists: list[list]
  983. :return: merged list
  984. :rtype: list[dict]
  985. """
  986. result = []
  987. seen = set()
  988. for task in itertools.chain(*reversed(tasks_lists)):
  989. if not task['id'] in seen:
  990. seen.add(task['id'])
  991. result.append(task)
  992. return result
  993. @classmethod
  994. def get_deployment_graph(cls, instance, graph_type=None):
  995. """Return deployment graph for cluster considering release and plugins.
  996. :param instance: models.Cluster instance
  997. :type instance: models.Cluster
  998. :param graph_type: deployment graph type
  999. :type graph_type: basestring|None
  1000. :return: deployment graph which includes metadata and tasks
  1001. :rtype: dict
  1002. """
  1003. if graph_type is None:
  1004. graph_type = consts.DEFAULT_DEPLOYMENT_GRAPH_TYPE
  1005. release_deployment_graph = cls.get_release_deployment_graph(
  1006. instance, graph_type=graph_type)
  1007. plugins_deployment_graph = cls.get_plugins_deployment_graph(
  1008. instance, graph_type=graph_type)
  1009. cluster_deployment_graph = cls.get_own_deployment_graph(
  1010. instance, graph_type=graph_type)
  1011. tasks = cls._merge_tasks_lists([
  1012. release_deployment_graph.pop('tasks'),
  1013. plugins_deployment_graph.pop('tasks'),
  1014. cluster_deployment_graph.pop('tasks')
  1015. ])
  1016. graph_metadata = {}
  1017. # added metadata from each type of graphs according
  1018. # to merge priority
  1019. dict_update(graph_metadata, release_deployment_graph)
  1020. dict_update(graph_metadata, plugins_deployment_graph)
  1021. dict_update(graph_metadata, cluster_deployment_graph)
  1022. graph_metadata['tasks'] = tasks
  1023. graph_metadata['type'] = graph_type
  1024. return graph_metadata
  1025. @classmethod
  1026. def get_deployment_tasks(cls, instance, graph_type=None):
  1027. """Return deployment tasks for cluster considering release and plugins.
  1028. :param instance: models.Cluster instance
  1029. :type instance: models.Cluster
  1030. :param graph_type: deployment graph type
  1031. :type graph_type: basestring|None
  1032. :return: deployment tasks list
  1033. :rtype: list[dict]
  1034. """
  1035. return cls.get_deployment_graph(instance, graph_type)['tasks']
  1036. @classmethod
  1037. def get_legacy_plugin_tasks(cls, instance):
  1038. """Get legacy deployment tasks from tasks.yaml."""
  1039. return PluginManager.get_legacy_tasks_for_cluster(instance)
  1040. @classmethod
  1041. def get_refreshable_tasks(cls, instance, filter_by_configs=None,
  1042. graph_type=None):
  1043. """Return list of refreshable tasks
  1044. If 'filter_by_configs' specified then only tasks needed to update
  1045. these config resources will be returned as a result, otherwise
  1046. all refreshable tasks will be returned
  1047. :param instance: a Cluster instance
  1048. :param filter_by_configs: a list with configs resources
  1049. :param graph_type: deployment graph type
  1050. :return: list of tasks
  1051. """
  1052. if filter_by_configs:
  1053. filter_by_configs = set(filter_by_configs)
  1054. tasks = []
  1055. for task in cls.get_deployment_tasks(instance, graph_type):
  1056. refresh_on = task.get(consts.TASK_REFRESH_FIELD)
  1057. if (refresh_on
  1058. and (filter_by_configs is None
  1059. or filter_by_configs.intersection(set(refresh_on)))):
  1060. tasks.append(task)
  1061. return tasks
  1062. @classmethod
  1063. def get_tags_metadata(cls, instance):
  1064. """Return proper tags metadata for cluster
  1065. Metadata consists of general tags metadata from release,
  1066. tags metadata from cluster and tags metadata from
  1067. plugins which are enabled for this cluster.
  1068. :param instance: Cluster DB instance
  1069. :returns: dict -- object with merged tags metadata
  1070. """
  1071. tags_meta = dict(instance.release.tags_metadata)
  1072. cluster_tags_meta = instance.tags_metadata
  1073. tags_meta.update(cluster_tags_meta)
  1074. plugins_tags_meta = PluginManager.get_tags_metadata(instance)
  1075. tags_meta.update(plugins_tags_meta)
  1076. return tags_meta
  1077. @classmethod
  1078. def get_volumes_metadata(cls, instance):
  1079. """Return proper volumes metadata for cluster
  1080. Metadata consists of general volumes metadata from release,
  1081. volumes metadata from cluster and volumes metadata from
  1082. plugins which are related to this cluster.
  1083. :param instance: Cluster DB instance
  1084. :returns: dict -- object with merged volumes metadata
  1085. """
  1086. def _update_volumes_meta(to_update, data):
  1087. to_update.get('volumes_roles_mapping', {}).update(
  1088. data.get('volumes_roles_mapping', {}))
  1089. to_update.get('volumes', []).extend(
  1090. data.get('volumes', []))
  1091. to_update.setdefault('rule_to_pick_boot_disk', []).extend(
  1092. data.get('rule_to_pick_boot_disk', []))
  1093. volumes_metadata = copy.deepcopy(
  1094. Release.get_volumes_metadata(instance.release))
  1095. cluster_volumes = instance.volumes_metadata
  1096. _update_volumes_meta(volumes_metadata, cluster_volumes)
  1097. plugin_volumes = PluginManager.get_volumes_metadata(instance)
  1098. _update_volumes_meta(volumes_metadata, plugin_volumes)
  1099. return volumes_metadata
  1100. @classmethod
  1101. def get_create_data(cls, instance):
  1102. """Return common parameters cluster was created with.
  1103. This method is compatible with :func:`create` and used to create
  1104. a new cluster with the same settings including the network
  1105. configuration.
  1106. :returns: a dict of key-value pairs as a cluster create data
  1107. """
  1108. data = {
  1109. "name": instance.name,
  1110. "mode": instance.mode,
  1111. "net_provider": instance.net_provider,
  1112. "release_id": instance.release.id,
  1113. }
  1114. data.update(cls.get_network_manager(instance).
  1115. get_network_config_create_data(instance))
  1116. return data
  1117. @staticmethod
  1118. def adjust_nodes_lists_on_controller_removing(instance, nodes_to_delete,
  1119. nodes_to_deploy):
  1120. """Adds controllers to nodes_to_deploy if deleting other controllers
  1121. :param instance: instance of SqlAlchemy cluster
  1122. :param nodes_to_delete: list of nodes to be deleted
  1123. :param nodes_to_deploy: list of nodes to be deployed
  1124. :return:
  1125. """
  1126. if instance is None:
  1127. return
  1128. controllers_ids_to_delete = set([n.id for n in nodes_to_delete
  1129. if 'controller' in n.all_roles])
  1130. if controllers_ids_to_delete:
  1131. ids_to_deploy = set([n.id for n in nodes_to_deploy])
  1132. controllers_to_deploy = set(
  1133. filter(lambda n: (n.id not in controllers_ids_to_delete
  1134. and n.id not in ids_to_deploy
  1135. and 'controller' in n.all_roles),
  1136. instance.nodes))
  1137. nodes_to_deploy.extend(controllers_to_deploy)
  1138. @classmethod
  1139. def get_repo_urls(self, instance):
  1140. repos = instance.attributes.editable['repo_setup']['repos']['value']
  1141. return tuple(set([r['uri'] for r in repos]))
  1142. @classmethod
  1143. def get_nodes_to_spawn_vms(cls, instance):
  1144. nodes = []
  1145. for node in cls.get_nodes_by_role(instance,
  1146. consts.VIRTUAL_NODE_TYPES.virt):
  1147. for vm in node.vms_conf:
  1148. if not vm.get('created'):
  1149. nodes.append(node)
  1150. return nodes
  1151. @classmethod
  1152. def set_vms_created_state(cls, instance):
  1153. nodes = cls.get_nodes_by_role(instance, consts.VIRTUAL_NODE_TYPES.virt)
  1154. for node in nodes:
  1155. for vm in node.vms_conf:
  1156. if not vm.get('created'):
  1157. vm['created'] = True
  1158. # notify about changes
  1159. node.vms_conf.changed()
  1160. db().flush()
  1161. @classmethod
  1162. def get_network_roles(
  1163. cls, instance, merge_policy=NetworkRoleMergePolicy()):
  1164. """Method for receiving network roles for particular cluster
  1165. :param instance: nailgun.db.sqlalchemy.models.Cluster instance
  1166. :param merge_policy: the policy to merge same roles
  1167. :returns: List of network roles' descriptions
  1168. """
  1169. return PluginManager.get_network_roles(instance, merge_policy)
  1170. @classmethod
  1171. def set_network_template(cls, instance, template):
  1172. instance.network_config.configuration_template = template
  1173. cls.update_nodes_network_template(instance, instance.nodes)
  1174. db().flush()
  1175. if template is None:
  1176. net_manager = cls.get_network_manager(instance)
  1177. for node in instance.nodes:
  1178. net_manager.clear_bond_configuration(node)
  1179. net_manager.assign_networks_by_default(node)
  1180. @classmethod
  1181. def update_nodes_network_template(cls, instance, nodes):
  1182. from nailgun.objects import Node
  1183. template = instance.network_config.configuration_template
  1184. for node in nodes:
  1185. Node.apply_network_template(node, template)
  1186. @classmethod
  1187. def get_nodes_ids(cls, instance):
  1188. return [x[0] for x in db().query(models.Node.id).filter(
  1189. models.Node.cluster_id == instance.id).all()]
  1190. @classmethod
  1191. def get_vips(cls, instance):
  1192. net_roles = cls.get_network_roles(instance)
  1193. cluster_vips = []
  1194. for nr in net_roles:
  1195. cluster_vips.extend(nr['properties']['vip'])
  1196. return cluster_vips
  1197. @classmethod
  1198. def get_assigned_roles(cls, instance):
  1199. """Get list of all roles currently assigned to nodes in cluster
  1200. :param instance: nailgun.db.sqlalchemy.models.Cluster instance
  1201. :returns: List of node roles currently assigned
  1202. """
  1203. pending_roles = db().query(
  1204. sa.func.unnest(models.Node.pending_roles)
  1205. ).filter_by(
  1206. cluster_id=instance.id
  1207. ).distinct().all()
  1208. pending_roles = [pr[0] for pr in pending_roles]
  1209. roles = db().query(
  1210. sa.func.unnest(models.Node.roles)
  1211. ).filter_by(
  1212. cluster_id=instance.id
  1213. ).distinct().all()
  1214. roles = [r[0] for r in roles]
  1215. return set(pending_roles + roles)
  1216. @classmethod
  1217. def is_network_modification_locked(cls, instance):
  1218. """Checks whether network settings can be modified or deleted.
  1219. The result depends on the current status of cluster.
  1220. """
  1221. return instance.is_locked
  1222. @classmethod
  1223. def is_component_enabled(cls, instance, component):
  1224. """Checks is specified additional component enabled in cluster
  1225. :param instance: nailgun.db.sqlalchemy.models.Cluster instance
  1226. :param component: name of additional component
  1227. :returns: The result depends on current component status in settings
  1228. """
  1229. return bool(instance.attributes.editable['additional_components'].
  1230. get((component), {}).get('value'))
  1231. @classmethod
  1232. def get_nodes_to_update_config(cls, cluster, node_ids=None, node_role=None,
  1233. only_ready_nodes=True):
  1234. """Get nodes for specified cluster that should be updated.
  1235. Configuration update can be executed for all nodes in the cluster,
  1236. or for single node, or for all nodes with specified role.
  1237. If :param only_ready_nodes set by True function returns list of nodes
  1238. that will be updated during next config update execution.
  1239. If :param only_ready_nodes set by False function returns list of all
  1240. nodes that will finally get an updated configuration.
  1241. """
  1242. query = cls.get_nodes_not_for_deletion(cluster)
  1243. if only_ready_nodes:
  1244. query = query.filter_by(status=consts.NODE_STATUSES.ready)
  1245. if node_ids:
  1246. query = query.filter(models.Node.id.in_(node_ids))
  1247. elif node_role:
  1248. query = query.filter(
  1249. models.Node.roles.any(node_role))
  1250. return query.all()
  1251. @classmethod
  1252. def prepare_for_deployment(cls, instance, nodes=None):
  1253. """Shortcut for NetworkManager.prepare_for_deployment.
  1254. :param instance: nailgun.db.sqlalchemy.models.Cluster instance
  1255. :param nodes: the list of Nodes, None means for all nodes
  1256. """
  1257. cls.get_network_manager(instance).prepare_for_deployment(
  1258. instance, instance.nodes if nodes is None else nodes
  1259. )
  1260. @classmethod
  1261. def is_task_deploy_enabled(cls, instance):
  1262. """Tests that task based deploy is enabled.
  1263. :param instance: cluster for checking
  1264. :type instance: nailgun.db.sqlalchemy.models.Cluster instance
  1265. """
  1266. attrs = cls.get_editable_attributes(instance)
  1267. return attrs['common'].get('task_deploy', {}).get('value')
  1268. @classmethod
  1269. def is_propagate_task_deploy_enabled(cls, instance):
  1270. """Tests that task based deployment propagation enabled.
  1271. :param instance: cluster for checking
  1272. :type instance: nailgun.db.sqlalchemy.models.Cluster instance
  1273. """
  1274. attrs = cls.get_editable_attributes(instance)
  1275. return attrs['common'].get('propagate_task_deploy', {}).get('value')
  1276. # FIXME(aroma): remove updating of 'deployed_before'
  1277. # when stop action is reworked. 'deployed_before'
  1278. # flag identifies whether stop action is allowed for the
  1279. # cluster. Please, refer to [1] for more details.
  1280. # [1]: https://bugs.launchpad.net/fuel/+bug/1529691
  1281. @classmethod
  1282. def set_deployed_before_flag(cls, instance, value):
  1283. """Change value for before_deployed if needed
  1284. :param instance: nailgun.db.sqlalchemy.models.Cluster instance
  1285. :param value: new value for flag
  1286. :type value: bool
  1287. """
  1288. generated = copy.deepcopy(instance.attributes.generated)
  1289. if 'deployed_before' not in generated:
  1290. # NOTE(aroma): this is needed for case when master node has
  1291. # been upgraded and there is attempt to re/deploy previously
  1292. # existing clusters. As long as setting the flag is temporary
  1293. # solution data base migration code should not be mangled
  1294. # in order to support it
  1295. generated['deployed_before'] = {'value': value}
  1296. elif generated['deployed_before']['value'] != value:
  1297. generated['deployed_before']['value'] = value
  1298. instance.attributes.generated = generated
  1299. db.flush()
  1300. @classmethod
  1301. def get_nodes_count_unmet_status(cls, instance, status):
  1302. """Gets the number of nodes, that does not have specified status.
  1303. :param instance: nailgun.db.sqlalchemy.models.Cluster instance
  1304. :param status: the expected status
  1305. :return: the number of nodes that does not have specified status
  1306. """
  1307. q = db().query(models.Node).filter_by(cluster_id=instance.id)
  1308. return q.filter(models.Node.status != status).count()
  1309. @classmethod
  1310. def get_network_attributes(cls, instance):
  1311. # use local import to avoid recursive imports
  1312. from nailgun.extensions.network_manager.objects.serializers import \
  1313. network_configuration
  1314. if instance.net_provider == consts.CLUSTER_NET_PROVIDERS.nova_network:
  1315. serializer = \
  1316. network_configuration.NovaNetworkConfigurationSerializer
  1317. else:
  1318. serializer = \
  1319. network_configuration.NeutronNetworkConfigurationSerializer
  1320. return serializer.serialize_for_cluster(instance)
  1321. @classmethod
  1322. def get_restrictions_models(cls, instance, attrs=None):
  1323. """Return models which are used in restrictions mechanism
  1324. :param instance: nailgun.db.sqlalchemy.models.Cluster instance
  1325. :param attrs: models' settings will be overwritten with this value
  1326. :return: dict with models
  1327. """
  1328. return {
  1329. 'settings': attrs or cls.get_editable_attributes(instance),
  1330. 'cluster': instance,
  1331. 'version': settings.VERSION,
  1332. 'networking_parameters': instance.network_config,
  1333. }
  1334. @classmethod
  1335. def is_dpdk_supported_for_segmentation(cls, instance):
  1336. return (instance.network_config.segmentation_type in
  1337. (consts.NEUTRON_SEGMENT_TYPES.vlan,
  1338. consts.NEUTRON_SEGMENT_TYPES.tun))
  1339. class ClusterCollection(NailgunCollection):
  1340. """Cluster collection."""
  1341. #: Single Cluster object class
  1342. single = Cluster