Fuel UI
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

deployment_serializers.py 33KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918
  1. # -*- coding: utf-8 -*-
  2. # Copyright 2013 Mirantis, Inc.
  3. #
  4. # Licensed under the Apache License, Version 2.0 (the "License"); you may
  5. # not use this file except in compliance with the License. You may obtain
  6. # a copy of the License at
  7. #
  8. # http://www.apache.org/licenses/LICENSE-2.0
  9. #
  10. # Unless required by applicable law or agreed to in writing, software
  11. # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  12. # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  13. # License for the specific language governing permissions and limitations
  14. # under the License.
  15. """Deployment serializers for orchestrator"""
  16. from distutils.version import StrictVersion
  17. import six
  18. from nailgun import consts
  19. from nailgun.db import db
  20. from nailgun import extensions
  21. from nailgun.logger import logger
  22. from nailgun import objects
  23. from nailgun import plugins
  24. from nailgun.settings import settings
  25. from nailgun import utils
  26. from nailgun.utils.resolvers import NameMatchingPolicy
  27. from nailgun.utils.resolvers import TagResolver
  28. from nailgun.orchestrator.base_serializers import MuranoMetadataSerializerMixin
  29. from nailgun.orchestrator.provisioning_serializers import \
  30. ProvisionLCMSerializer
  31. from nailgun.extensions.network_manager.serializers import neutron_serializers
  32. from nailgun.extensions.network_manager.serializers import nova_serializers
  33. class DeploymentMultinodeSerializer(object):
  34. nova_network_serializer = nova_serializers.NovaNetworkDeploymentSerializer
  35. neutron_network_serializer = \
  36. neutron_serializers.NeutronNetworkDeploymentSerializer
  37. critical_roles = frozenset(('controller', 'ceph-osd', 'primary-mongo'))
  38. def __init__(self, tasks_graph=None):
  39. self.task_graph = tasks_graph
  40. self.all_nodes = None
  41. self.resolver = None
  42. self.initialized = None
  43. def initialize(self, cluster):
  44. self.all_nodes = objects.Cluster.get_nodes_not_for_deletion(cluster)
  45. self.resolver = TagResolver(self.all_nodes)
  46. self.initialized = cluster.id
  47. def finalize(self):
  48. self.all_nodes = None
  49. self.resolver = None
  50. self.initialized = None
  51. def _ensure_initialized_for(self, cluster):
  52. # TODO(bgaifullin) need to move initialize into __init__
  53. if self.initialized != cluster.id:
  54. self.initialize(cluster)
  55. def serialize(self, cluster, nodes,
  56. ignore_customized=False, skip_extensions=False):
  57. """Method generates facts which are passed to puppet."""
  58. try:
  59. self.initialize(cluster)
  60. common_attrs = self.get_common_attrs(cluster)
  61. if not ignore_customized and cluster.replaced_deployment_info:
  62. # patch common attributes with custom deployment info
  63. utils.dict_update(
  64. common_attrs, cluster.replaced_deployment_info
  65. )
  66. if not skip_extensions:
  67. extensions.\
  68. fire_callback_on_cluster_serialization_for_deployment(
  69. cluster, common_attrs
  70. )
  71. serialized_nodes = []
  72. origin_nodes = []
  73. customized_nodes = []
  74. if ignore_customized:
  75. origin_nodes = nodes
  76. else:
  77. for node in nodes:
  78. if node.replaced_deployment_info:
  79. customized_nodes.append(node)
  80. else:
  81. origin_nodes.append(node)
  82. serialized_nodes.extend(
  83. self.serialize_generated(origin_nodes, skip_extensions)
  84. )
  85. serialized_nodes.extend(
  86. self.serialize_customized(customized_nodes)
  87. )
  88. # NOTE(dshulyak) tasks should not be preserved from replaced
  89. # deployment info, there is different mechanism to control
  90. # changes in tasks introduced during granular deployment,
  91. # and that mech should be used
  92. self.set_tasks(serialized_nodes)
  93. deployment_info = {'common': common_attrs,
  94. 'nodes': serialized_nodes}
  95. finally:
  96. self.finalize()
  97. return deployment_info
  98. def serialize_generated(self, nodes, skip_extensions):
  99. serialized_nodes = self.serialize_nodes(nodes)
  100. nodes_map = {n.uid: n for n in nodes}
  101. self.set_deployment_priorities(serialized_nodes)
  102. for node_data in serialized_nodes:
  103. # the serialized nodes may contain fake nodes like master node
  104. # which does not have related db object. it shall be excluded.
  105. if not skip_extensions and node_data['uid'] in nodes_map:
  106. extensions.fire_callback_on_node_serialization_for_deployment(
  107. nodes_map[node_data['uid']], node_data
  108. )
  109. yield node_data
  110. def serialize_customized(self, nodes):
  111. for node in nodes:
  112. for role_data in node.replaced_deployment_info:
  113. yield role_data
  114. def get_common_attrs(self, cluster):
  115. """Cluster attributes."""
  116. # tests call this method directly.
  117. # and we need this workaround to avoid refactoring a lot of tests.
  118. self._ensure_initialized_for(cluster)
  119. attrs = objects.Cluster.get_attributes(cluster)
  120. attrs = objects.Attributes.merged_attrs_values(attrs)
  121. attrs['deployment_mode'] = cluster.mode
  122. attrs['deployment_id'] = cluster.id
  123. attrs['openstack_version'] = cluster.release.version
  124. attrs['fuel_version'] = cluster.fuel_version
  125. attrs['nodes'] = self.node_list(self.all_nodes)
  126. # Adding params to workloads_collector
  127. if 'workloads_collector' not in attrs:
  128. attrs['workloads_collector'] = {}
  129. attrs['workloads_collector']['create_user'] = \
  130. objects.MasterNodeSettings.must_send_stats()
  131. username = attrs['workloads_collector'].pop('user', None)
  132. attrs['workloads_collector']['username'] = username
  133. if self.resolver.resolve(['cinder']):
  134. attrs['use_cinder'] = True
  135. net_serializer = self.get_net_provider_serializer(cluster)
  136. net_common_attrs = net_serializer.get_common_attrs(cluster, attrs)
  137. utils.dict_update(attrs, net_common_attrs)
  138. self.inject_list_of_plugins(attrs, cluster)
  139. return attrs
  140. @classmethod
  141. def node_list(cls, nodes):
  142. """Generate nodes list. Represents as "nodes" parameter in facts."""
  143. node_list = []
  144. for node in nodes:
  145. for role in objects.Node.all_tags(node):
  146. node_list.append(cls.serialize_node_for_node_list(node, role))
  147. return node_list
  148. @classmethod
  149. def serialize_node_for_node_list(cls, node, role):
  150. return {
  151. 'uid': node.uid,
  152. 'fqdn': objects.Node.get_node_fqdn(node),
  153. 'name': objects.Node.get_slave_name(node),
  154. 'role': role}
  155. # TODO(apopovych): we have more generical method 'filter_by_roles'
  156. def by_role(self, nodes, role):
  157. return filter(lambda node: node['role'] == role, nodes)
  158. def not_roles(self, nodes, roles):
  159. return filter(lambda node: node['role'] not in roles, nodes)
  160. def serialize_nodes(self, nodes):
  161. """Serialize node for each role.
  162. For example if node has two roles then
  163. in orchestrator will be passed two serialized
  164. nodes.
  165. """
  166. serialized_nodes = []
  167. for node in nodes:
  168. for role in objects.Node.all_tags(node):
  169. serialized_nodes.append(
  170. self.serialize_node(node, role)
  171. )
  172. return serialized_nodes
  173. def serialize_node(self, node, role):
  174. """Serialize node, then it will be merged with common attributes."""
  175. node_attrs = {
  176. # Yes, uid is really should be a string
  177. 'uid': node.uid,
  178. 'fqdn': objects.Node.get_node_fqdn(node),
  179. 'status': node.status,
  180. 'role': role,
  181. 'vms_conf': node.vms_conf,
  182. 'fail_if_error': role in self.critical_roles,
  183. # TODO(eli): need to remove, requried for the fake thread only
  184. 'online': node.online,
  185. }
  186. net_serializer = self.get_net_provider_serializer(node.cluster)
  187. node_attrs.update(net_serializer.get_node_attrs(node))
  188. node_attrs.update(net_serializer.network_ranges(node.group_id))
  189. node_attrs.update(self.generate_test_vm_image_data(node))
  190. return node_attrs
  191. def generate_properties_arguments(self, properties_data):
  192. """build a string of properties from a key value hash"""
  193. properties = []
  194. for key, value in six.iteritems(properties_data):
  195. properties.append('--property {key}={value}'.format(
  196. key=key, value=value))
  197. return ' '.join(properties)
  198. def generate_test_vm_image_data(self, node):
  199. # Instantiate all default values in dict.
  200. image_data = {
  201. 'container_format': 'bare',
  202. 'public': 'true',
  203. 'disk_format': 'qcow2',
  204. 'img_name': 'TestVM',
  205. 'img_path': '',
  206. 'os_name': 'cirros',
  207. 'min_ram': 64,
  208. 'glance_properties': '',
  209. 'properties': {},
  210. }
  211. # Generate a right path to image.
  212. c_attrs = node.cluster.attributes
  213. if 'ubuntu' in c_attrs['generated']['cobbler']['profile']:
  214. img_dir = '/usr/share/cirros-testvm/'
  215. else:
  216. img_dir = '/opt/vm/'
  217. image_data['img_path'] = '{0}cirros-x86_64-disk.img'.format(img_dir)
  218. properties_data = {}
  219. # NOTE(aschultz): properties was added as part of N and should be
  220. # used infavor of glance_properties
  221. image_data['glance_properties'] = self.generate_properties_arguments(
  222. properties_data)
  223. image_data['properties'] = properties_data
  224. return {'test_vm_image': image_data}
  225. @classmethod
  226. def get_net_provider_serializer(cls, cluster):
  227. if cluster.net_provider == 'nova_network':
  228. return cls.nova_network_serializer
  229. else:
  230. return cls.neutron_network_serializer
  231. def filter_by_roles(self, nodes, roles):
  232. return filter(
  233. lambda node: node['role'] in roles, nodes)
  234. def set_deployment_priorities(self, nodes):
  235. if self.task_graph is not None:
  236. self.task_graph.add_priorities(nodes)
  237. def set_tasks(self, serialized_nodes):
  238. if self.task_graph is not None:
  239. for node in serialized_nodes:
  240. node['tasks'] = self.task_graph.deploy_task_serialize(node)
  241. def inject_list_of_plugins(self, attributes, cluster):
  242. """Added information about plugins to serialized attributes.
  243. :param attributes: the serialized attributes
  244. :param cluster: the cluster object
  245. """
  246. plugins = objects.ClusterPlugin.get_enabled(cluster.id)
  247. attributes['plugins'] = [
  248. self.serialize_plugin(cluster, p) for p in plugins
  249. ]
  250. @classmethod
  251. def serialize_plugin(cls, cluster, plugin):
  252. """Gets plugin information to include into serialized attributes.
  253. :param cluster: the cluster object
  254. :param plugin: the plugin object
  255. """
  256. return plugin['name']
  257. class DeploymentHASerializer(DeploymentMultinodeSerializer):
  258. """Serializer for HA mode."""
  259. critical_roles = frozenset((
  260. 'primary-controller',
  261. 'primary-mongo',
  262. 'primary-swift-proxy',
  263. 'ceph-osd',
  264. 'controller'
  265. ))
  266. def get_last_controller(self, nodes):
  267. sorted_nodes = sorted(
  268. nodes, key=lambda node: int(node['uid']))
  269. controller_nodes = self.filter_by_roles(
  270. sorted_nodes, ['controller', 'primary-controller'])
  271. last_controller = None
  272. if len(controller_nodes) > 0:
  273. last_controller = controller_nodes[-1]['name']
  274. return {'last_controller': last_controller}
  275. @classmethod
  276. def node_list(cls, nodes):
  277. """Node list."""
  278. node_list = super(
  279. DeploymentHASerializer,
  280. cls
  281. ).node_list(nodes)
  282. for node in node_list:
  283. node['swift_zone'] = node['uid']
  284. return node_list
  285. def get_common_attrs(self, cluster):
  286. """Common attributes for all facts."""
  287. common_attrs = super(
  288. DeploymentHASerializer,
  289. self
  290. ).get_common_attrs(cluster)
  291. common_attrs.update(self.get_assigned_vips(cluster))
  292. common_attrs['mp'] = [
  293. {'point': '1', 'weight': '1'},
  294. {'point': '2', 'weight': '2'}]
  295. last_controller = self.get_last_controller(common_attrs['nodes'])
  296. common_attrs.update(last_controller)
  297. return common_attrs
  298. def get_assigned_vips(self, cluster):
  299. """Assign and get vips for net groups."""
  300. return objects.Cluster.get_network_manager(cluster).\
  301. assign_vips_for_net_groups(cluster)
  302. class DeploymentMultinodeSerializer50(MuranoMetadataSerializerMixin,
  303. DeploymentMultinodeSerializer):
  304. pass
  305. class DeploymentHASerializer50(MuranoMetadataSerializerMixin,
  306. DeploymentHASerializer):
  307. pass
  308. class DeploymentMultinodeSerializer51(DeploymentMultinodeSerializer50):
  309. nova_network_serializer = nova_serializers.NovaNetworkDeploymentSerializer
  310. neutron_network_serializer = \
  311. neutron_serializers.NeutronNetworkDeploymentSerializer51
  312. class DeploymentHASerializer51(DeploymentHASerializer50):
  313. nova_network_serializer = nova_serializers.NovaNetworkDeploymentSerializer
  314. neutron_network_serializer = \
  315. neutron_serializers.NeutronNetworkDeploymentSerializer51
  316. class DeploymentMultinodeSerializer60(DeploymentMultinodeSerializer50):
  317. nova_network_serializer = nova_serializers.NovaNetworkDeploymentSerializer
  318. neutron_network_serializer = \
  319. neutron_serializers.NeutronNetworkDeploymentSerializer60
  320. class DeploymentHASerializer60(DeploymentHASerializer50):
  321. nova_network_serializer = nova_serializers.NovaNetworkDeploymentSerializer
  322. neutron_network_serializer = \
  323. neutron_serializers.NeutronNetworkDeploymentSerializer60
  324. class DeploymentMultinodeSerializer61(DeploymentMultinodeSerializer):
  325. nova_network_serializer = \
  326. nova_serializers.NovaNetworkDeploymentSerializer61
  327. neutron_network_serializer = \
  328. neutron_serializers.NeutronNetworkDeploymentSerializer61
  329. def serialize_node(self, node, role):
  330. base = super(DeploymentMultinodeSerializer61, self)
  331. serialized_node = base.serialize_node(node, role)
  332. serialized_node['user_node_name'] = node.name
  333. return serialized_node
  334. @classmethod
  335. def serialize_node_for_node_list(cls, node, role):
  336. serialized_node = super(
  337. DeploymentMultinodeSerializer61,
  338. cls).serialize_node_for_node_list(node, role)
  339. serialized_node['user_node_name'] = node.name
  340. return serialized_node
  341. class DeploymentHASerializer61(DeploymentHASerializer):
  342. nova_network_serializer = \
  343. nova_serializers.NovaNetworkDeploymentSerializer61
  344. neutron_network_serializer = \
  345. neutron_serializers.NeutronNetworkDeploymentSerializer61
  346. def serialize_node(self, node, role):
  347. base = super(DeploymentHASerializer61, self)
  348. serialized_node = base.serialize_node(node, role)
  349. serialized_node['user_node_name'] = node.name
  350. return serialized_node
  351. @classmethod
  352. def serialize_node_for_node_list(cls, node, role):
  353. serialized_node = super(
  354. DeploymentHASerializer61,
  355. cls).serialize_node_for_node_list(node, role)
  356. serialized_node['user_node_name'] = node.name
  357. return serialized_node
  358. class DeploymentHASerializer70(DeploymentHASerializer61):
  359. # nova_network_serializer is just for compatibility with current BVTs
  360. # and other tests. It can be removed when tests are fixed.
  361. nova_network_serializer = \
  362. nova_serializers.NovaNetworkDeploymentSerializer70
  363. @classmethod
  364. def get_net_provider_serializer(cls, cluster):
  365. if cluster.net_provider == consts.CLUSTER_NET_PROVIDERS.nova_network:
  366. return cls.nova_network_serializer
  367. elif cluster.network_config.configuration_template:
  368. return neutron_serializers.NeutronNetworkTemplateSerializer70
  369. else:
  370. return neutron_serializers.NeutronNetworkDeploymentSerializer70
  371. def get_assigned_vips(self, cluster):
  372. return {}
  373. class DeploymentHASerializer80(DeploymentHASerializer70):
  374. @classmethod
  375. def get_net_provider_serializer(cls, cluster):
  376. if cluster.network_config.configuration_template:
  377. return neutron_serializers.NeutronNetworkTemplateSerializer80
  378. else:
  379. return neutron_serializers.NeutronNetworkDeploymentSerializer80
  380. class DeploymentHASerializer90(DeploymentHASerializer80):
  381. def inject_murano_settings(self, data):
  382. return data
  383. def get_common_attrs(self, cluster):
  384. attrs = super(DeploymentHASerializer90, self).get_common_attrs(cluster)
  385. for node in objects.Cluster.get_nodes_not_for_deletion(cluster):
  386. name = objects.Node.permanent_id(node)
  387. node_attrs = attrs['network_metadata']['nodes'][name]
  388. node_attrs['nova_cpu_pinning_enabled'] = \
  389. objects.NodeAttributes.is_nova_cpu_pinning_enabled(node)
  390. node_attrs['nova_hugepages_enabled'] = (
  391. objects.NodeAttributes.is_nova_hugepages_enabled(node))
  392. return attrs
  393. @classmethod
  394. def get_net_provider_serializer(cls, cluster):
  395. if cluster.network_config.configuration_template:
  396. return neutron_serializers.NeutronNetworkTemplateSerializer90
  397. else:
  398. return neutron_serializers.NeutronNetworkDeploymentSerializer90
  399. def serialize_node(self, node, role):
  400. base = super(DeploymentHASerializer90, self)
  401. serialized_node = base.serialize_node(node, role)
  402. self.serialize_node_attributes(node, serialized_node)
  403. return serialized_node
  404. def serialize_node_attributes(self, node, serialized_node):
  405. self.generate_cpu_pinning(node, serialized_node)
  406. self.generate_node_hugepages(node, serialized_node)
  407. def generate_cpu_pinning(self, node, serialized_node):
  408. if not objects.NodeAttributes.is_cpu_pinning_enabled(node):
  409. return
  410. pinning_info = objects.NodeAttributes.distribute_node_cpus(node)
  411. cpu_pinning = pinning_info['components']
  412. self._generate_nova_cpu_pinning(
  413. serialized_node,
  414. cpu_pinning.pop('nova', [])
  415. )
  416. self._generate_dpdk_cpu_pinning(
  417. serialized_node,
  418. cpu_pinning.pop('ovs_core_mask', []),
  419. cpu_pinning.pop('ovs_pmd_core_mask', [])
  420. )
  421. # Allow user to override CPU distribution using attributes
  422. if 'dpdk' in serialized_node:
  423. serialized_node['dpdk'].update(objects.Node.get_attributes(node)
  424. .get('dpdk', {}))
  425. serialized_node['cpu_pinning'] = cpu_pinning
  426. def generate_node_hugepages(self, node, serialized_node):
  427. if not objects.NodeAttributes.is_hugepages_enabled(node):
  428. return
  429. self._generate_nova_hugepages(node, serialized_node)
  430. self._generate_dpdk_hugepages(node, serialized_node)
  431. self._generate_hugepages_distribution(node, serialized_node)
  432. @staticmethod
  433. def _generate_nova_cpu_pinning(serialized_node, cpus):
  434. if not cpus:
  435. return
  436. serialized_node.setdefault('nova', {})['cpu_pinning'] = cpus
  437. @staticmethod
  438. def _generate_dpdk_cpu_pinning(serialized_node, ovs_core_cpus,
  439. ovs_pmd_core_cpus):
  440. """Translate list of CPU ids to DPDK masks
  441. ovsdpdk application may use pinned CPUs
  442. it takes CPU masks. CPU mask contains information
  443. about pinned CPUs: N-th bit is set to 1 if
  444. appropriate CPU id is pinned for DPDK process
  445. """
  446. if not ovs_core_cpus and not ovs_pmd_core_cpus:
  447. return
  448. ovs_core_mask = 0
  449. ovs_pmd_core_mask = 0
  450. for cpu in ovs_core_cpus:
  451. ovs_core_mask |= (1 << cpu)
  452. for cpu in ovs_pmd_core_cpus:
  453. ovs_pmd_core_mask |= (1 << cpu)
  454. serialized_node.setdefault('dpdk', {}).update({
  455. 'ovs_core_mask': hex(ovs_core_mask),
  456. 'ovs_pmd_core_mask': hex(ovs_pmd_core_mask),
  457. })
  458. @staticmethod
  459. def _generate_nova_hugepages(node, serialized_node):
  460. serialized_node.setdefault('nova', {})['enable_hugepages'] = (
  461. objects.NodeAttributes.is_nova_hugepages_enabled(node))
  462. @staticmethod
  463. def _generate_dpdk_hugepages(node, serialized_node):
  464. serialized_node.setdefault('dpdk', {}).update(
  465. objects.NodeAttributes.dpdk_hugepages_attrs(node))
  466. @classmethod
  467. def _generate_hugepages_distribution(self, node, serialized_node):
  468. hugepages = objects.NodeAttributes.distribute_hugepages(node)
  469. # FIXME(asvechnikov): We should skip our distribution
  470. # due to LP bug #1560532, so we can't configure 1G hugepages
  471. # in runtime. This limitation should gone with kernel 3.16
  472. skip = any((x['size'] == 1024 ** 2) for x in hugepages)
  473. if hugepages and not skip:
  474. serialized_node.setdefault('hugepages', []).extend(
  475. hugepages)
  476. class DeploymentLCMSerializer(DeploymentHASerializer90):
  477. _configs = None
  478. _cluster_info = None
  479. _provision_serializer = None
  480. _priorities = {
  481. consts.OPENSTACK_CONFIG_TYPES.cluster: 0,
  482. consts.OPENSTACK_CONFIG_TYPES.role: 1,
  483. consts.OPENSTACK_CONFIG_TYPES.node: 2,
  484. }
  485. def initialize(self, cluster):
  486. super(DeploymentLCMSerializer, self).initialize(cluster)
  487. self._configs = sorted(
  488. objects.OpenstackConfigCollection.find_configs_for_nodes(
  489. cluster,
  490. cluster.nodes or [],
  491. ),
  492. key=lambda x: self._priorities[x.config_type]
  493. )
  494. self._provision_serializer = ProvisionLCMSerializer()
  495. def finalize(self):
  496. self._configs = None
  497. self._provision_serializer = None
  498. self._cluster_info = None
  499. super(DeploymentLCMSerializer, self).finalize()
  500. def get_common_attrs(self, cluster):
  501. attrs = super(DeploymentLCMSerializer, self).get_common_attrs(
  502. cluster
  503. )
  504. attrs['cluster'] = objects.Cluster.to_dict(
  505. cluster, fields=("id", "name", "fuel_version", "status", "mode")
  506. )
  507. attrs['release'] = objects.Release.to_dict(
  508. cluster.release, fields=('name', 'version', 'operating_system')
  509. )
  510. # the ReleaseSerializer adds this attribute certainly
  511. attrs['release'].pop('is_deployable', None)
  512. provision = attrs.setdefault('provision', {})
  513. utils.dict_update(
  514. provision,
  515. self._provision_serializer.serialize_cluster_info(cluster, attrs)
  516. )
  517. # TODO(bgaifullin) remove using cluster_info
  518. # in serialize_node_for_provision
  519. self._cluster_info = attrs
  520. return attrs
  521. def serialize_customized(self, nodes):
  522. for node in nodes:
  523. data = {}
  524. roles = []
  525. for role_data in node.replaced_deployment_info:
  526. if 'role' in role_data:
  527. # if replaced_deployment_info consists
  528. # of old serialized info, the old info
  529. # have serialized data per role
  530. roles.append(role_data.pop('role'))
  531. utils.dict_update(data, role_data)
  532. if roles:
  533. data['roles'] = roles
  534. self.inject_provision_info(node, data)
  535. yield data
  536. def serialize_nodes(self, nodes):
  537. serialized_nodes = []
  538. for node in nodes:
  539. roles = objects.Node.all_tags(node)
  540. if roles:
  541. serialized_nodes.append(
  542. self.serialize_node(node, roles)
  543. )
  544. # added master node
  545. serialized_nodes.append({
  546. 'uid': consts.MASTER_NODE_UID,
  547. 'roles': [consts.TASK_ROLES.master]
  548. })
  549. return serialized_nodes
  550. def serialize_node(self, node, roles):
  551. # serialize all roles to one config
  552. # Since there is no role depended things except
  553. # OpenStack configs, we can do this
  554. base = super(DeploymentLCMSerializer, self)
  555. serialized_node = base.serialize_node(node, roles[0])
  556. del serialized_node['role']
  557. serialized_node['roles'] = roles
  558. if node.pending_deletion:
  559. serialized_node['deleted'] = True
  560. self.inject_configs(node, serialized_node)
  561. self.inject_provision_info(node, serialized_node)
  562. return serialized_node
  563. @classmethod
  564. def serialize_plugin(cls, cluster, plugin):
  565. os_name = cluster.release.operating_system
  566. adapter = plugins.wrap_plugin(plugin)
  567. result = {
  568. 'name': plugin['name'],
  569. 'scripts': [
  570. {
  571. 'remote_url': adapter.master_scripts_path(cluster),
  572. 'local_path': adapter.slaves_scripts_path
  573. }
  574. ]
  575. }
  576. if not adapter.repo_files(cluster):
  577. return result
  578. # TODO(bgaifullin) move priority to plugin metadata
  579. if os_name == consts.RELEASE_OS.centos:
  580. repo = {
  581. 'type': 'rpm',
  582. 'name': adapter.path_name,
  583. 'uri': adapter.repo_url(cluster),
  584. 'priority': settings.REPO_PRIORITIES['plugins']['centos']
  585. }
  586. elif os_name == consts.RELEASE_OS.ubuntu:
  587. repo = {
  588. 'type': 'deb',
  589. 'name': adapter.path_name,
  590. 'uri': adapter.repo_url(cluster),
  591. 'suite': '/',
  592. 'section': '',
  593. 'priority': settings.REPO_PRIORITIES['plugins']['ubuntu']
  594. }
  595. else:
  596. logger.warning("Unsupported OS: %s.", os_name)
  597. return result
  598. result['repositories'] = [repo]
  599. return result
  600. def inject_configs(self, node, output):
  601. node_config = output.setdefault('configuration', {})
  602. node_config_opts = output.setdefault('configuration_options', {})
  603. for config in self._configs:
  604. # OpenstackConfig.configuration is MutableDict, so we copy
  605. # data for preventing changes in the DB
  606. config_data = config.configuration.copy()
  607. # TODO(akislitsky) refactor CLI and OpenstackConfig object
  608. # to allow serialize arbitrary data. Old configs data should be
  609. # modified to the structure {'configuration': old_configuration}.
  610. # Then new config data will have the structure:
  611. # {'configuration': old_configuration,
  612. # 'configuration_options': ...,
  613. # 'any_key': any_value
  614. # }
  615. # and new structure will be serialized to the node config.
  616. config_data_opts = config_data.pop('configuration_options', {})
  617. if config.config_type == consts.OPENSTACK_CONFIG_TYPES.cluster:
  618. utils.dict_update(node_config, config_data, 1)
  619. utils.dict_update(node_config_opts, config_data_opts, 1)
  620. elif config.config_type == consts.OPENSTACK_CONFIG_TYPES.role:
  621. # (asaprykin): objects.Node.all_roles() has a side effect,
  622. # it replaces "<rolename>" with "primary-<rolename>"
  623. # in case of primary role.
  624. for role in node.all_roles:
  625. if NameMatchingPolicy.create(config.node_role).match(role):
  626. utils.dict_update(node_config, config_data, 1)
  627. utils.dict_update(node_config_opts,
  628. config_data_opts, 1)
  629. elif config.config_type == consts.OPENSTACK_CONFIG_TYPES.node:
  630. if config.node_id == node.id:
  631. utils.dict_update(node_config, config_data, 1)
  632. utils.dict_update(node_config_opts, config_data_opts, 1)
  633. def inject_provision_info(self, node, data):
  634. # TODO(bgaifullin) serialize_node_info should be reworked
  635. if not self._cluster_info:
  636. self._cluster_info = self.get_common_attrs(node.cluster)
  637. if node.replaced_provisioning_info:
  638. info = node.replaced_provisioning_info
  639. else:
  640. info = self._provision_serializer.serialize_node_info(
  641. self._cluster_info, node
  642. )
  643. utils.dict_update(data.setdefault('provision', {}), info)
  644. @classmethod
  645. def serialize_node_for_node_list(cls, node, role):
  646. serialized_node = super(
  647. DeploymentLCMSerializer,
  648. cls).serialize_node_for_node_list(node, role)
  649. for section_name, section_attributes in six.iteritems(
  650. plugins.manager.PluginManager.
  651. get_plugin_node_attributes(node)):
  652. section_attributes.pop('metadata', None)
  653. serialized_node[section_name] = {
  654. k: v.get('value') for k, v in six.iteritems(section_attributes)
  655. }
  656. return serialized_node
  657. class DeploymentLCMSerializer110(DeploymentLCMSerializer):
  658. @classmethod
  659. def get_net_provider_serializer(cls, cluster):
  660. if cluster.network_config.configuration_template:
  661. return neutron_serializers.NeutronNetworkTemplateSerializer110
  662. else:
  663. return neutron_serializers.NeutronNetworkDeploymentSerializer110
  664. def get_serializer_for_cluster(cluster):
  665. """Returns a serializer depends on a given `cluster`.
  666. :param cluster: cluster to process
  667. :returns: a serializer for a given cluster
  668. """
  669. serializers_map = {
  670. '5.0': {
  671. 'multinode': DeploymentMultinodeSerializer50,
  672. 'ha': DeploymentHASerializer50,
  673. },
  674. '5.1': {
  675. 'multinode': DeploymentMultinodeSerializer51,
  676. 'ha': DeploymentHASerializer51,
  677. },
  678. '6.0': {
  679. 'multinode': DeploymentMultinodeSerializer60,
  680. 'ha': DeploymentHASerializer60,
  681. },
  682. '6.1': {
  683. 'multinode': DeploymentMultinodeSerializer61,
  684. 'ha': DeploymentHASerializer61,
  685. },
  686. '7.0': {
  687. # Multinode is not supported anymore
  688. 'ha': DeploymentHASerializer70,
  689. },
  690. '8.0': {
  691. 'ha': DeploymentHASerializer80,
  692. },
  693. '9.0': {
  694. 'ha': DeploymentHASerializer90,
  695. }
  696. }
  697. env_mode = 'ha' if cluster.is_ha_mode else 'multinode'
  698. for version, serializers in six.iteritems(serializers_map):
  699. if cluster.release.environment_version.startswith(version):
  700. return serializers[env_mode]
  701. # return latest serializer by default
  702. latest_version = max(serializers_map.keys(), key=StrictVersion)
  703. return serializers_map[latest_version][env_mode]
  704. def _invoke_serializer(serializer, cluster, nodes,
  705. ignore_customized, skip_extensions):
  706. if not skip_extensions:
  707. extensions.fire_callback_on_before_deployment_serialization(
  708. cluster, cluster.nodes, ignore_customized
  709. )
  710. objects.Cluster.set_primary_tags(cluster, nodes)
  711. # commit the transaction immediately so that the updates
  712. # made to nodes don't lock other updates to these nodes
  713. # until this, possibly very long, transation ends.
  714. db().commit()
  715. return serializer.serialize(
  716. cluster, nodes,
  717. ignore_customized=ignore_customized, skip_extensions=skip_extensions
  718. )
  719. def serialize(orchestrator_graph, cluster, nodes,
  720. ignore_customized=False, skip_extensions=False):
  721. """Serialization depends on deployment mode."""
  722. serialized = _invoke_serializer(
  723. get_serializer_for_cluster(cluster)(orchestrator_graph),
  724. cluster, nodes, ignore_customized, skip_extensions
  725. )
  726. return serialized
  727. def serialize_for_lcm(cluster, nodes,
  728. ignore_customized=False, skip_extensions=False):
  729. serializers_map = {
  730. 'default': DeploymentLCMSerializer,
  731. '11.0': DeploymentLCMSerializer110,
  732. }
  733. serializer_lcm = serializers_map['default']
  734. for version, serializer in six.iteritems(serializers_map):
  735. if cluster.release.environment_version.startswith(version):
  736. serializer_lcm = serializer
  737. break
  738. return _invoke_serializer(
  739. serializer_lcm(), cluster, nodes,
  740. ignore_customized, skip_extensions
  741. )
  742. def deployment_info_to_legacy(deployment_info):
  743. common_attrs = deployment_info['common']
  744. nodes = [utils.dict_merge(common_attrs, n)
  745. for n in deployment_info['nodes']]
  746. return nodes