Fuel UI
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

deployment_serializers.py 35KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980
  1. # -*- coding: utf-8 -*-
  2. # Copyright 2013 Mirantis, Inc.
  3. #
  4. # Licensed under the Apache License, Version 2.0 (the "License"); you may
  5. # not use this file except in compliance with the License. You may obtain
  6. # a copy of the License at
  7. #
  8. # http://www.apache.org/licenses/LICENSE-2.0
  9. #
  10. # Unless required by applicable law or agreed to in writing, software
  11. # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  12. # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  13. # License for the specific language governing permissions and limitations
  14. # under the License.
  15. """Deployment serializers for orchestrator"""
  16. from copy import deepcopy
  17. from distutils.version import StrictVersion
  18. import six
  19. from nailgun import consts
  20. from nailgun.db import db
  21. from nailgun import extensions
  22. from nailgun.logger import logger
  23. from nailgun import objects
  24. from nailgun import plugins
  25. from nailgun.settings import settings
  26. from nailgun import utils
  27. from nailgun.utils.resolvers import NameMatchingPolicy
  28. from nailgun.utils.resolvers import TagResolver
  29. from nailgun.orchestrator.base_serializers import MuranoMetadataSerializerMixin
  30. from nailgun.orchestrator.base_serializers import \
  31. VmwareDeploymentSerializerMixin
  32. from nailgun.orchestrator.provisioning_serializers import \
  33. ProvisionLCMSerializer
  34. from nailgun.extensions.network_manager.serializers import neutron_serializers
  35. from nailgun.extensions.network_manager.serializers import nova_serializers
  36. class DeploymentMultinodeSerializer(object):
  37. nova_network_serializer = nova_serializers.NovaNetworkDeploymentSerializer
  38. neutron_network_serializer = \
  39. neutron_serializers.NeutronNetworkDeploymentSerializer
  40. critical_roles = frozenset(('controller', 'ceph-osd', 'primary-mongo'))
  41. def __init__(self, tasks_graph=None):
  42. self.task_graph = tasks_graph
  43. self.all_nodes = None
  44. self.resolver = None
  45. self.initialized = None
  46. def initialize(self, cluster):
  47. self.all_nodes = objects.Cluster.get_nodes_not_for_deletion(cluster)
  48. self.resolver = TagResolver(self.all_nodes)
  49. self.initialized = cluster.id
  50. def finalize(self):
  51. self.all_nodes = None
  52. self.resolver = None
  53. self.initialized = None
  54. def _ensure_initialized_for(self, cluster):
  55. # TODO(bgaifullin) need to move initialize into __init__
  56. if self.initialized != cluster.id:
  57. self.initialize(cluster)
  58. def serialize(self, cluster, nodes,
  59. ignore_customized=False, skip_extensions=False):
  60. """Method generates facts which are passed to puppet."""
  61. try:
  62. self.initialize(cluster)
  63. common_attrs = self.get_common_attrs(cluster)
  64. if not ignore_customized and cluster.replaced_deployment_info:
  65. # patch common attributes with custom deployment info
  66. utils.dict_update(
  67. common_attrs, cluster.replaced_deployment_info
  68. )
  69. if not skip_extensions:
  70. extensions.\
  71. fire_callback_on_cluster_serialization_for_deployment(
  72. cluster, common_attrs
  73. )
  74. serialized_nodes = []
  75. origin_nodes = []
  76. customized_nodes = []
  77. if ignore_customized:
  78. origin_nodes = nodes
  79. else:
  80. for node in nodes:
  81. if node.replaced_deployment_info:
  82. customized_nodes.append(node)
  83. else:
  84. origin_nodes.append(node)
  85. serialized_nodes.extend(
  86. self.serialize_generated(origin_nodes, skip_extensions)
  87. )
  88. serialized_nodes.extend(
  89. self.serialize_customized(customized_nodes)
  90. )
  91. # NOTE(dshulyak) tasks should not be preserved from replaced
  92. # deployment info, there is different mechanism to control
  93. # changes in tasks introduced during granular deployment,
  94. # and that mech should be used
  95. self.set_tasks(serialized_nodes)
  96. deployment_info = {'common': common_attrs,
  97. 'nodes': serialized_nodes}
  98. finally:
  99. self.finalize()
  100. return deployment_info
  101. def serialize_generated(self, nodes, skip_extensions):
  102. serialized_nodes = self.serialize_nodes(nodes)
  103. nodes_map = {n.uid: n for n in nodes}
  104. self.set_deployment_priorities(serialized_nodes)
  105. for node_data in serialized_nodes:
  106. # the serialized nodes may contain fake nodes like master node
  107. # which does not have related db object. it shall be excluded.
  108. if not skip_extensions and node_data['uid'] in nodes_map:
  109. extensions.fire_callback_on_node_serialization_for_deployment(
  110. nodes_map[node_data['uid']], node_data
  111. )
  112. yield node_data
  113. def serialize_customized(self, nodes):
  114. for node in nodes:
  115. for role_data in node.replaced_deployment_info:
  116. yield role_data
  117. def get_common_attrs(self, cluster):
  118. """Cluster attributes."""
  119. # tests call this method directly.
  120. # and we need this workaround to avoid refactoring a lot of tests.
  121. self._ensure_initialized_for(cluster)
  122. attrs = objects.Cluster.get_attributes(cluster)
  123. attrs = objects.Attributes.merged_attrs_values(attrs)
  124. attrs['deployment_mode'] = cluster.mode
  125. attrs['deployment_id'] = cluster.id
  126. attrs['openstack_version'] = cluster.release.version
  127. attrs['fuel_version'] = cluster.fuel_version
  128. attrs['nodes'] = self.node_list(self.all_nodes)
  129. # Adding params to workloads_collector
  130. if 'workloads_collector' not in attrs:
  131. attrs['workloads_collector'] = {}
  132. attrs['workloads_collector']['create_user'] = \
  133. objects.MasterNodeSettings.must_send_stats()
  134. username = attrs['workloads_collector'].pop('user', None)
  135. attrs['workloads_collector']['username'] = username
  136. if self.resolver.resolve(['cinder']):
  137. attrs['use_cinder'] = True
  138. net_serializer = self.get_net_provider_serializer(cluster)
  139. net_common_attrs = net_serializer.get_common_attrs(cluster, attrs)
  140. utils.dict_update(attrs, net_common_attrs)
  141. self.inject_list_of_plugins(attrs, cluster)
  142. return attrs
  143. @classmethod
  144. def node_list(cls, nodes):
  145. """Generate nodes list. Represents as "nodes" parameter in facts."""
  146. node_list = []
  147. for node in nodes:
  148. for role in objects.Node.all_tags(node):
  149. node_list.append(cls.serialize_node_for_node_list(node, role))
  150. return node_list
  151. @classmethod
  152. def serialize_node_for_node_list(cls, node, role):
  153. return {
  154. 'uid': node.uid,
  155. 'fqdn': objects.Node.get_node_fqdn(node),
  156. 'name': objects.Node.get_slave_name(node),
  157. 'role': role}
  158. # TODO(apopovych): we have more generical method 'filter_by_roles'
  159. def by_role(self, nodes, role):
  160. return filter(lambda node: node['role'] == role, nodes)
  161. def not_roles(self, nodes, roles):
  162. return filter(lambda node: node['role'] not in roles, nodes)
  163. def serialize_nodes(self, nodes):
  164. """Serialize node for each role.
  165. For example if node has two roles then
  166. in orchestrator will be passed two serialized
  167. nodes.
  168. """
  169. serialized_nodes = []
  170. for node in nodes:
  171. for role in objects.Node.all_tags(node):
  172. serialized_nodes.append(
  173. self.serialize_node(node, role)
  174. )
  175. return serialized_nodes
  176. def serialize_node(self, node, role):
  177. """Serialize node, then it will be merged with common attributes."""
  178. node_attrs = {
  179. # Yes, uid is really should be a string
  180. 'uid': node.uid,
  181. 'fqdn': objects.Node.get_node_fqdn(node),
  182. 'status': node.status,
  183. 'role': role,
  184. 'vms_conf': node.vms_conf,
  185. 'fail_if_error': role in self.critical_roles,
  186. # TODO(eli): need to remove, requried for the fake thread only
  187. 'online': node.online,
  188. }
  189. net_serializer = self.get_net_provider_serializer(node.cluster)
  190. node_attrs.update(net_serializer.get_node_attrs(node))
  191. node_attrs.update(net_serializer.network_ranges(node.group_id))
  192. node_attrs.update(self.generate_test_vm_image_data(node))
  193. return node_attrs
  194. def generate_properties_arguments(self, properties_data):
  195. """build a string of properties from a key value hash"""
  196. properties = []
  197. for key, value in six.iteritems(properties_data):
  198. properties.append('--property {key}={value}'.format(
  199. key=key, value=value))
  200. return ' '.join(properties)
  201. def generate_test_vm_image_data(self, node):
  202. # Instantiate all default values in dict.
  203. image_data = {
  204. 'container_format': 'bare',
  205. 'public': 'true',
  206. 'disk_format': 'qcow2',
  207. 'img_name': 'TestVM',
  208. 'img_path': '',
  209. 'os_name': 'cirros',
  210. 'min_ram': 64,
  211. 'glance_properties': '',
  212. 'properties': {},
  213. }
  214. # Generate a right path to image.
  215. c_attrs = node.cluster.attributes
  216. if 'ubuntu' in c_attrs['generated']['cobbler']['profile']:
  217. img_dir = '/usr/share/cirros-testvm/'
  218. else:
  219. img_dir = '/opt/vm/'
  220. image_data['img_path'] = '{0}cirros-x86_64-disk.img'.format(img_dir)
  221. properties_data = {}
  222. # Alternate VMWare specific values.
  223. if c_attrs['editable']['common']['libvirt_type']['value'] == 'vcenter':
  224. image_data.update({
  225. 'disk_format': 'vmdk',
  226. 'img_path': '{0}cirros-i386-disk.vmdk'.format(img_dir),
  227. })
  228. properties_data = {
  229. 'vmware_disktype': 'sparse',
  230. 'vmware_adaptertype': 'lsiLogic',
  231. 'hypervisor_type': 'vmware'
  232. }
  233. # NOTE(aschultz): properties was added as part of N and should be
  234. # used infavor of glance_properties
  235. image_data['glance_properties'] = self.generate_properties_arguments(
  236. properties_data)
  237. image_data['properties'] = properties_data
  238. return {'test_vm_image': image_data}
  239. @classmethod
  240. def get_net_provider_serializer(cls, cluster):
  241. if cluster.net_provider == 'nova_network':
  242. return cls.nova_network_serializer
  243. else:
  244. return cls.neutron_network_serializer
  245. def filter_by_roles(self, nodes, roles):
  246. return filter(
  247. lambda node: node['role'] in roles, nodes)
  248. def set_deployment_priorities(self, nodes):
  249. if self.task_graph is not None:
  250. self.task_graph.add_priorities(nodes)
  251. def set_tasks(self, serialized_nodes):
  252. if self.task_graph is not None:
  253. for node in serialized_nodes:
  254. node['tasks'] = self.task_graph.deploy_task_serialize(node)
  255. def inject_list_of_plugins(self, attributes, cluster):
  256. """Added information about plugins to serialized attributes.
  257. :param attributes: the serialized attributes
  258. :param cluster: the cluster object
  259. """
  260. plugins = objects.ClusterPlugin.get_enabled(cluster.id)
  261. attributes['plugins'] = [
  262. self.serialize_plugin(cluster, p) for p in plugins
  263. ]
  264. @classmethod
  265. def serialize_plugin(cls, cluster, plugin):
  266. """Gets plugin information to include into serialized attributes.
  267. :param cluster: the cluster object
  268. :param plugin: the plugin object
  269. """
  270. return plugin['name']
  271. class DeploymentHASerializer(DeploymentMultinodeSerializer):
  272. """Serializer for HA mode."""
  273. critical_roles = frozenset((
  274. 'primary-controller',
  275. 'primary-mongo',
  276. 'primary-swift-proxy',
  277. 'ceph-osd',
  278. 'controller'
  279. ))
  280. def get_last_controller(self, nodes):
  281. sorted_nodes = sorted(
  282. nodes, key=lambda node: int(node['uid']))
  283. controller_nodes = self.filter_by_roles(
  284. sorted_nodes, ['controller', 'primary-controller'])
  285. last_controller = None
  286. if len(controller_nodes) > 0:
  287. last_controller = controller_nodes[-1]['name']
  288. return {'last_controller': last_controller}
  289. @classmethod
  290. def node_list(cls, nodes):
  291. """Node list."""
  292. node_list = super(
  293. DeploymentHASerializer,
  294. cls
  295. ).node_list(nodes)
  296. for node in node_list:
  297. node['swift_zone'] = node['uid']
  298. return node_list
  299. def get_common_attrs(self, cluster):
  300. """Common attributes for all facts."""
  301. common_attrs = super(
  302. DeploymentHASerializer,
  303. self
  304. ).get_common_attrs(cluster)
  305. common_attrs.update(self.get_assigned_vips(cluster))
  306. common_attrs['mp'] = [
  307. {'point': '1', 'weight': '1'},
  308. {'point': '2', 'weight': '2'}]
  309. last_controller = self.get_last_controller(common_attrs['nodes'])
  310. common_attrs.update(last_controller)
  311. return common_attrs
  312. def get_assigned_vips(self, cluster):
  313. """Assign and get vips for net groups."""
  314. return objects.Cluster.get_network_manager(cluster).\
  315. assign_vips_for_net_groups(cluster)
  316. class DeploymentMultinodeSerializer50(MuranoMetadataSerializerMixin,
  317. DeploymentMultinodeSerializer):
  318. pass
  319. class DeploymentHASerializer50(MuranoMetadataSerializerMixin,
  320. DeploymentHASerializer):
  321. pass
  322. class DeploymentMultinodeSerializer51(DeploymentMultinodeSerializer50):
  323. nova_network_serializer = nova_serializers.NovaNetworkDeploymentSerializer
  324. neutron_network_serializer = \
  325. neutron_serializers.NeutronNetworkDeploymentSerializer51
  326. class DeploymentHASerializer51(DeploymentHASerializer50):
  327. nova_network_serializer = nova_serializers.NovaNetworkDeploymentSerializer
  328. neutron_network_serializer = \
  329. neutron_serializers.NeutronNetworkDeploymentSerializer51
  330. class DeploymentMultinodeSerializer60(DeploymentMultinodeSerializer50):
  331. nova_network_serializer = nova_serializers.NovaNetworkDeploymentSerializer
  332. neutron_network_serializer = \
  333. neutron_serializers.NeutronNetworkDeploymentSerializer60
  334. class DeploymentHASerializer60(DeploymentHASerializer50):
  335. nova_network_serializer = nova_serializers.NovaNetworkDeploymentSerializer
  336. neutron_network_serializer = \
  337. neutron_serializers.NeutronNetworkDeploymentSerializer60
  338. class DeploymentMultinodeSerializer61(DeploymentMultinodeSerializer,
  339. VmwareDeploymentSerializerMixin):
  340. nova_network_serializer = \
  341. nova_serializers.NovaNetworkDeploymentSerializer61
  342. neutron_network_serializer = \
  343. neutron_serializers.NeutronNetworkDeploymentSerializer61
  344. def serialize_node(self, node, role):
  345. base = super(DeploymentMultinodeSerializer61, self)
  346. serialized_node = base.serialize_node(node, role)
  347. serialized_node['user_node_name'] = node.name
  348. serialized_node.update(self.generate_vmware_data(node))
  349. return serialized_node
  350. @classmethod
  351. def serialize_node_for_node_list(cls, node, role):
  352. serialized_node = super(
  353. DeploymentMultinodeSerializer61,
  354. cls).serialize_node_for_node_list(node, role)
  355. serialized_node['user_node_name'] = node.name
  356. return serialized_node
  357. class DeploymentHASerializer61(DeploymentHASerializer,
  358. VmwareDeploymentSerializerMixin):
  359. nova_network_serializer = \
  360. nova_serializers.NovaNetworkDeploymentSerializer61
  361. neutron_network_serializer = \
  362. neutron_serializers.NeutronNetworkDeploymentSerializer61
  363. def serialize_node(self, node, role):
  364. base = super(DeploymentHASerializer61, self)
  365. serialized_node = base.serialize_node(node, role)
  366. serialized_node['user_node_name'] = node.name
  367. serialized_node.update(self.generate_vmware_data(node))
  368. return serialized_node
  369. @classmethod
  370. def serialize_node_for_node_list(cls, node, role):
  371. serialized_node = super(
  372. DeploymentHASerializer61,
  373. cls).serialize_node_for_node_list(node, role)
  374. serialized_node['user_node_name'] = node.name
  375. return serialized_node
  376. # Alternate VMWare specific values.
  377. # FiXME(who): srogov
  378. # This a temporary workaround to keep existing functioanality
  379. # after fully implementation of the multi HV support and astute part
  380. # for multiple images support, it is need to change
  381. # dict image_data['test_vm_image'] to list of dicts
  382. def generate_test_vm_image_data(self, node):
  383. attrs = node.cluster.attributes
  384. image_data = super(
  385. DeploymentHASerializer61,
  386. self).generate_test_vm_image_data(node)
  387. images_data = {}
  388. images_data['test_vm_image'] = []
  389. if attrs.get('editable', {}).get('common', {}). \
  390. get('use_vcenter', {}).get('value') is True:
  391. image_vmdk_data = deepcopy(image_data['test_vm_image'])
  392. img_path = image_vmdk_data['img_path']. \
  393. replace('x86_64-disk.img', 'i386-disk.vmdk')
  394. image_vmdk_data.update({
  395. 'img_name': 'TestVM-VMDK',
  396. 'disk_format': 'vmdk',
  397. 'img_path': img_path,
  398. })
  399. properties_data = {
  400. 'vmware_disktype': 'sparse',
  401. 'vmware_adaptertype': 'lsiLogic',
  402. 'hypervisor_type': 'vmware'
  403. }
  404. glance_properties = self.generate_properties_arguments(
  405. properties_data)
  406. # NOTE(aschultz): properties was added as part of N and should be
  407. # used infavor of glance_properties
  408. image_vmdk_data['glance_properties'] = glance_properties
  409. image_vmdk_data['properties'] = properties_data
  410. images_data['test_vm_image'].append(image_vmdk_data)
  411. images_data['test_vm_image'].append(image_data['test_vm_image'])
  412. else:
  413. images_data = image_data
  414. return images_data
  415. class DeploymentHASerializer70(DeploymentHASerializer61):
  416. # nova_network_serializer is just for compatibility with current BVTs
  417. # and other tests. It can be removed when tests are fixed.
  418. nova_network_serializer = \
  419. nova_serializers.NovaNetworkDeploymentSerializer70
  420. @classmethod
  421. def get_net_provider_serializer(cls, cluster):
  422. if cluster.net_provider == consts.CLUSTER_NET_PROVIDERS.nova_network:
  423. return cls.nova_network_serializer
  424. elif cluster.network_config.configuration_template:
  425. return neutron_serializers.NeutronNetworkTemplateSerializer70
  426. else:
  427. return neutron_serializers.NeutronNetworkDeploymentSerializer70
  428. def get_assigned_vips(self, cluster):
  429. return {}
  430. class DeploymentHASerializer80(DeploymentHASerializer70):
  431. @classmethod
  432. def get_net_provider_serializer(cls, cluster):
  433. if cluster.network_config.configuration_template:
  434. return neutron_serializers.NeutronNetworkTemplateSerializer80
  435. else:
  436. return neutron_serializers.NeutronNetworkDeploymentSerializer80
  437. class DeploymentHASerializer90(DeploymentHASerializer80):
  438. def inject_murano_settings(self, data):
  439. return data
  440. def get_common_attrs(self, cluster):
  441. attrs = super(DeploymentHASerializer90, self).get_common_attrs(cluster)
  442. for node in objects.Cluster.get_nodes_not_for_deletion(cluster):
  443. name = objects.Node.permanent_id(node)
  444. node_attrs = attrs['network_metadata']['nodes'][name]
  445. node_attrs['nova_cpu_pinning_enabled'] = \
  446. objects.NodeAttributes.is_nova_cpu_pinning_enabled(node)
  447. node_attrs['nova_hugepages_enabled'] = (
  448. objects.NodeAttributes.is_nova_hugepages_enabled(node))
  449. return attrs
  450. @classmethod
  451. def get_net_provider_serializer(cls, cluster):
  452. if cluster.network_config.configuration_template:
  453. return neutron_serializers.NeutronNetworkTemplateSerializer90
  454. else:
  455. return neutron_serializers.NeutronNetworkDeploymentSerializer90
  456. def serialize_node(self, node, role):
  457. base = super(DeploymentHASerializer90, self)
  458. serialized_node = base.serialize_node(node, role)
  459. self.serialize_node_attributes(node, serialized_node)
  460. return serialized_node
  461. def serialize_node_attributes(self, node, serialized_node):
  462. self.generate_cpu_pinning(node, serialized_node)
  463. self.generate_node_hugepages(node, serialized_node)
  464. def generate_cpu_pinning(self, node, serialized_node):
  465. if not objects.NodeAttributes.is_cpu_pinning_enabled(node):
  466. return
  467. pinning_info = objects.NodeAttributes.distribute_node_cpus(node)
  468. cpu_pinning = pinning_info['components']
  469. self._generate_nova_cpu_pinning(
  470. serialized_node,
  471. cpu_pinning.pop('nova', [])
  472. )
  473. self._generate_dpdk_cpu_pinning(
  474. serialized_node,
  475. cpu_pinning.pop('ovs_core_mask', []),
  476. cpu_pinning.pop('ovs_pmd_core_mask', [])
  477. )
  478. # Allow user to override CPU distribution using attributes
  479. if 'dpdk' in serialized_node:
  480. serialized_node['dpdk'].update(objects.Node.get_attributes(node)
  481. .get('dpdk', {}))
  482. serialized_node['cpu_pinning'] = cpu_pinning
  483. def generate_node_hugepages(self, node, serialized_node):
  484. if not objects.NodeAttributes.is_hugepages_enabled(node):
  485. return
  486. self._generate_nova_hugepages(node, serialized_node)
  487. self._generate_dpdk_hugepages(node, serialized_node)
  488. self._generate_hugepages_distribution(node, serialized_node)
  489. @staticmethod
  490. def _generate_nova_cpu_pinning(serialized_node, cpus):
  491. if not cpus:
  492. return
  493. serialized_node.setdefault('nova', {})['cpu_pinning'] = cpus
  494. @staticmethod
  495. def _generate_dpdk_cpu_pinning(serialized_node, ovs_core_cpus,
  496. ovs_pmd_core_cpus):
  497. """Translate list of CPU ids to DPDK masks
  498. ovsdpdk application may use pinned CPUs
  499. it takes CPU masks. CPU mask contains information
  500. about pinned CPUs: N-th bit is set to 1 if
  501. appropriate CPU id is pinned for DPDK process
  502. """
  503. if not ovs_core_cpus and not ovs_pmd_core_cpus:
  504. return
  505. ovs_core_mask = 0
  506. ovs_pmd_core_mask = 0
  507. for cpu in ovs_core_cpus:
  508. ovs_core_mask |= (1 << cpu)
  509. for cpu in ovs_pmd_core_cpus:
  510. ovs_pmd_core_mask |= (1 << cpu)
  511. serialized_node.setdefault('dpdk', {}).update({
  512. 'ovs_core_mask': hex(ovs_core_mask),
  513. 'ovs_pmd_core_mask': hex(ovs_pmd_core_mask),
  514. })
  515. @staticmethod
  516. def _generate_nova_hugepages(node, serialized_node):
  517. serialized_node.setdefault('nova', {})['enable_hugepages'] = (
  518. objects.NodeAttributes.is_nova_hugepages_enabled(node))
  519. @staticmethod
  520. def _generate_dpdk_hugepages(node, serialized_node):
  521. serialized_node.setdefault('dpdk', {}).update(
  522. objects.NodeAttributes.dpdk_hugepages_attrs(node))
  523. @classmethod
  524. def _generate_hugepages_distribution(self, node, serialized_node):
  525. hugepages = objects.NodeAttributes.distribute_hugepages(node)
  526. # FIXME(asvechnikov): We should skip our distribution
  527. # due to LP bug #1560532, so we can't configure 1G hugepages
  528. # in runtime. This limitation should gone with kernel 3.16
  529. skip = any((x['size'] == 1024 ** 2) for x in hugepages)
  530. if hugepages and not skip:
  531. serialized_node.setdefault('hugepages', []).extend(
  532. hugepages)
  533. class DeploymentLCMSerializer(DeploymentHASerializer90):
  534. _configs = None
  535. _cluster_info = None
  536. _provision_serializer = None
  537. _priorities = {
  538. consts.OPENSTACK_CONFIG_TYPES.cluster: 0,
  539. consts.OPENSTACK_CONFIG_TYPES.role: 1,
  540. consts.OPENSTACK_CONFIG_TYPES.node: 2,
  541. }
  542. def initialize(self, cluster):
  543. super(DeploymentLCMSerializer, self).initialize(cluster)
  544. self._configs = sorted(
  545. objects.OpenstackConfigCollection.find_configs_for_nodes(
  546. cluster,
  547. cluster.nodes or [],
  548. ),
  549. key=lambda x: self._priorities[x.config_type]
  550. )
  551. self._provision_serializer = ProvisionLCMSerializer()
  552. def finalize(self):
  553. self._configs = None
  554. self._provision_serializer = None
  555. self._cluster_info = None
  556. super(DeploymentLCMSerializer, self).finalize()
  557. def get_common_attrs(self, cluster):
  558. attrs = super(DeploymentLCMSerializer, self).get_common_attrs(
  559. cluster
  560. )
  561. attrs['cluster'] = objects.Cluster.to_dict(
  562. cluster, fields=("id", "name", "fuel_version", "status", "mode")
  563. )
  564. attrs['release'] = objects.Release.to_dict(
  565. cluster.release, fields=('name', 'version', 'operating_system')
  566. )
  567. # the ReleaseSerializer adds this attribute certainly
  568. attrs['release'].pop('is_deployable', None)
  569. provision = attrs.setdefault('provision', {})
  570. utils.dict_update(
  571. provision,
  572. self._provision_serializer.serialize_cluster_info(cluster, attrs)
  573. )
  574. # TODO(bgaifullin) remove using cluster_info
  575. # in serialize_node_for_provision
  576. self._cluster_info = attrs
  577. return attrs
  578. def serialize_customized(self, nodes):
  579. for node in nodes:
  580. data = {}
  581. roles = []
  582. for role_data in node.replaced_deployment_info:
  583. if 'role' in role_data:
  584. # if replaced_deployment_info consists
  585. # of old serialized info, the old info
  586. # have serialized data per role
  587. roles.append(role_data.pop('role'))
  588. utils.dict_update(data, role_data)
  589. if roles:
  590. data['roles'] = roles
  591. self.inject_provision_info(node, data)
  592. yield data
  593. def serialize_nodes(self, nodes):
  594. serialized_nodes = []
  595. for node in nodes:
  596. roles = objects.Node.all_tags(node)
  597. if roles:
  598. serialized_nodes.append(
  599. self.serialize_node(node, roles)
  600. )
  601. # added master node
  602. serialized_nodes.append({
  603. 'uid': consts.MASTER_NODE_UID,
  604. 'roles': [consts.TASK_ROLES.master]
  605. })
  606. return serialized_nodes
  607. def serialize_node(self, node, roles):
  608. # serialize all roles to one config
  609. # Since there is no role depended things except
  610. # OpenStack configs, we can do this
  611. base = super(DeploymentLCMSerializer, self)
  612. serialized_node = base.serialize_node(node, roles[0])
  613. del serialized_node['role']
  614. serialized_node['roles'] = roles
  615. if node.pending_deletion:
  616. serialized_node['deleted'] = True
  617. self.inject_configs(node, serialized_node)
  618. self.inject_provision_info(node, serialized_node)
  619. return serialized_node
  620. @classmethod
  621. def serialize_plugin(cls, cluster, plugin):
  622. os_name = cluster.release.operating_system
  623. adapter = plugins.wrap_plugin(plugin)
  624. result = {
  625. 'name': plugin['name'],
  626. 'scripts': [
  627. {
  628. 'remote_url': adapter.master_scripts_path(cluster),
  629. 'local_path': adapter.slaves_scripts_path
  630. }
  631. ]
  632. }
  633. if not adapter.repo_files(cluster):
  634. return result
  635. # TODO(bgaifullin) move priority to plugin metadata
  636. if os_name == consts.RELEASE_OS.centos:
  637. repo = {
  638. 'type': 'rpm',
  639. 'name': adapter.path_name,
  640. 'uri': adapter.repo_url(cluster),
  641. 'priority': settings.REPO_PRIORITIES['plugins']['centos']
  642. }
  643. elif os_name == consts.RELEASE_OS.ubuntu:
  644. repo = {
  645. 'type': 'deb',
  646. 'name': adapter.path_name,
  647. 'uri': adapter.repo_url(cluster),
  648. 'suite': '/',
  649. 'section': '',
  650. 'priority': settings.REPO_PRIORITIES['plugins']['ubuntu']
  651. }
  652. else:
  653. logger.warning("Unsupported OS: %s.", os_name)
  654. return result
  655. result['repositories'] = [repo]
  656. return result
  657. def inject_configs(self, node, output):
  658. node_config = output.setdefault('configuration', {})
  659. node_config_opts = output.setdefault('configuration_options', {})
  660. for config in self._configs:
  661. # OpenstackConfig.configuration is MutableDict, so we copy
  662. # data for preventing changes in the DB
  663. config_data = config.configuration.copy()
  664. # TODO(akislitsky) refactor CLI and OpenstackConfig object
  665. # to allow serialize arbitrary data. Old configs data should be
  666. # modified to the structure {'configuration': old_configuration}.
  667. # Then new config data will have the structure:
  668. # {'configuration': old_configuration,
  669. # 'configuration_options': ...,
  670. # 'any_key': any_value
  671. # }
  672. # and new structure will be serialized to the node config.
  673. config_data_opts = config_data.pop('configuration_options', {})
  674. if config.config_type == consts.OPENSTACK_CONFIG_TYPES.cluster:
  675. utils.dict_update(node_config, config_data, 1)
  676. utils.dict_update(node_config_opts, config_data_opts, 1)
  677. elif config.config_type == consts.OPENSTACK_CONFIG_TYPES.role:
  678. # (asaprykin): objects.Node.all_roles() has a side effect,
  679. # it replaces "<rolename>" with "primary-<rolename>"
  680. # in case of primary role.
  681. for role in node.all_roles:
  682. if NameMatchingPolicy.create(config.node_role).match(role):
  683. utils.dict_update(node_config, config_data, 1)
  684. utils.dict_update(node_config_opts,
  685. config_data_opts, 1)
  686. elif config.config_type == consts.OPENSTACK_CONFIG_TYPES.node:
  687. if config.node_id == node.id:
  688. utils.dict_update(node_config, config_data, 1)
  689. utils.dict_update(node_config_opts, config_data_opts, 1)
  690. def inject_provision_info(self, node, data):
  691. # TODO(bgaifullin) serialize_node_info should be reworked
  692. if not self._cluster_info:
  693. self._cluster_info = self.get_common_attrs(node.cluster)
  694. if node.replaced_provisioning_info:
  695. info = node.replaced_provisioning_info
  696. else:
  697. info = self._provision_serializer.serialize_node_info(
  698. self._cluster_info, node
  699. )
  700. utils.dict_update(data.setdefault('provision', {}), info)
  701. @classmethod
  702. def serialize_node_for_node_list(cls, node, role):
  703. serialized_node = super(
  704. DeploymentLCMSerializer,
  705. cls).serialize_node_for_node_list(node, role)
  706. for section_name, section_attributes in six.iteritems(
  707. plugins.manager.PluginManager.
  708. get_plugin_node_attributes(node)):
  709. section_attributes.pop('metadata', None)
  710. serialized_node[section_name] = {
  711. k: v.get('value') for k, v in six.iteritems(section_attributes)
  712. }
  713. return serialized_node
  714. class DeploymentLCMSerializer110(DeploymentLCMSerializer):
  715. @classmethod
  716. def get_net_provider_serializer(cls, cluster):
  717. if cluster.network_config.configuration_template:
  718. return neutron_serializers.NeutronNetworkTemplateSerializer110
  719. else:
  720. return neutron_serializers.NeutronNetworkDeploymentSerializer110
  721. def get_serializer_for_cluster(cluster):
  722. """Returns a serializer depends on a given `cluster`.
  723. :param cluster: cluster to process
  724. :returns: a serializer for a given cluster
  725. """
  726. serializers_map = {
  727. '5.0': {
  728. 'multinode': DeploymentMultinodeSerializer50,
  729. 'ha': DeploymentHASerializer50,
  730. },
  731. '5.1': {
  732. 'multinode': DeploymentMultinodeSerializer51,
  733. 'ha': DeploymentHASerializer51,
  734. },
  735. '6.0': {
  736. 'multinode': DeploymentMultinodeSerializer60,
  737. 'ha': DeploymentHASerializer60,
  738. },
  739. '6.1': {
  740. 'multinode': DeploymentMultinodeSerializer61,
  741. 'ha': DeploymentHASerializer61,
  742. },
  743. '7.0': {
  744. # Multinode is not supported anymore
  745. 'ha': DeploymentHASerializer70,
  746. },
  747. '8.0': {
  748. 'ha': DeploymentHASerializer80,
  749. },
  750. '9.0': {
  751. 'ha': DeploymentHASerializer90,
  752. }
  753. }
  754. env_mode = 'ha' if cluster.is_ha_mode else 'multinode'
  755. for version, serializers in six.iteritems(serializers_map):
  756. if cluster.release.environment_version.startswith(version):
  757. return serializers[env_mode]
  758. # return latest serializer by default
  759. latest_version = max(serializers_map.keys(), key=StrictVersion)
  760. return serializers_map[latest_version][env_mode]
  761. def _invoke_serializer(serializer, cluster, nodes,
  762. ignore_customized, skip_extensions):
  763. if not skip_extensions:
  764. extensions.fire_callback_on_before_deployment_serialization(
  765. cluster, cluster.nodes, ignore_customized
  766. )
  767. objects.Cluster.set_primary_tags(cluster, nodes)
  768. # commit the transaction immediately so that the updates
  769. # made to nodes don't lock other updates to these nodes
  770. # until this, possibly very long, transation ends.
  771. db().commit()
  772. return serializer.serialize(
  773. cluster, nodes,
  774. ignore_customized=ignore_customized, skip_extensions=skip_extensions
  775. )
  776. def serialize(orchestrator_graph, cluster, nodes,
  777. ignore_customized=False, skip_extensions=False):
  778. """Serialization depends on deployment mode."""
  779. serialized = _invoke_serializer(
  780. get_serializer_for_cluster(cluster)(orchestrator_graph),
  781. cluster, nodes, ignore_customized, skip_extensions
  782. )
  783. return serialized
  784. def serialize_for_lcm(cluster, nodes,
  785. ignore_customized=False, skip_extensions=False):
  786. serializers_map = {
  787. 'default': DeploymentLCMSerializer,
  788. '11.0': DeploymentLCMSerializer110,
  789. }
  790. serializer_lcm = serializers_map['default']
  791. for version, serializer in six.iteritems(serializers_map):
  792. if cluster.release.environment_version.startswith(version):
  793. serializer_lcm = serializer
  794. break
  795. return _invoke_serializer(
  796. serializer_lcm(), cluster, nodes,
  797. ignore_customized, skip_extensions
  798. )
  799. def deployment_info_to_legacy(deployment_info):
  800. common_attrs = deployment_info['common']
  801. nodes = [utils.dict_merge(common_attrs, n)
  802. for n in deployment_info['nodes']]
  803. return nodes