Fuel UI
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

adapters.py 17KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557
  1. # Copyright 2014 Mirantis, Inc.
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License"); you may
  4. # not use this file except in compliance with the License. You may obtain
  5. # a copy of the License at
  6. #
  7. # http://www.apache.org/licenses/LICENSE-2.0
  8. #
  9. # Unless required by applicable law or agreed to in writing, software
  10. # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  11. # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  12. # License for the specific language governing permissions and limitations
  13. # under the License.
  14. import abc
  15. import copy
  16. from distutils.version import StrictVersion
  17. import glob
  18. import os
  19. from urlparse import urljoin
  20. import six
  21. import loaders
  22. import nailgun
  23. from nailgun import consts
  24. from nailgun import errors
  25. from nailgun.logger import logger
  26. from nailgun.settings import settings
  27. @six.add_metaclass(abc.ABCMeta)
  28. class PluginAdapterBase(object):
  29. """Implements wrapper for plugin db model configuration files logic
  30. 1. Uploading plugin provided attributes
  31. 2. Uploading tasks and deployment tasks
  32. 3. Providing repositories/deployment scripts related info to clients
  33. """
  34. loader_class = loaders.PluginLoaderBase
  35. def __init__(self, plugin):
  36. self.plugin = plugin
  37. self.plugin_path = os.path.join(settings.PLUGINS_PATH, self.path_name)
  38. self.loader = self.loader_class(self.plugin_path)
  39. @property
  40. def attributes_processors(self):
  41. return {
  42. 'attributes_metadata':
  43. lambda data: (data or {}).get('attributes', {}),
  44. 'tasks': lambda data: data or []
  45. }
  46. @abc.abstractmethod
  47. def path_name(self):
  48. """A name which is used to create path to plugin scripts and repo"""
  49. def get_metadata(self):
  50. """Get plugin data tree.
  51. :return: All plugin metadata
  52. :rtype: dict
  53. """
  54. data_tree, report = self.loader.load()
  55. if report.is_failed():
  56. logger.error(report.render())
  57. logger.error('Problem with loading plugin {0}'.format(
  58. self.plugin_path))
  59. return data_tree
  60. for field in data_tree:
  61. if field in self.attributes_processors:
  62. data_tree[field] = \
  63. self.attributes_processors[field](data_tree.get(field))
  64. data_tree = {
  65. k: v for k, v in six.iteritems(data_tree)
  66. if v is not None}
  67. return data_tree
  68. @property
  69. def plugin_release_versions(self):
  70. if not self.plugin.releases:
  71. return set()
  72. return set([rel['version'] for rel in self.plugin.releases])
  73. @property
  74. def title(self):
  75. return self.plugin.title
  76. @property
  77. def name(self):
  78. return self.plugin.name
  79. @property
  80. def full_name(self):
  81. return u'{0}-{1}'.format(self.plugin.name, self.plugin.version)
  82. @property
  83. def slaves_scripts_path(self):
  84. return settings.PLUGINS_SLAVES_SCRIPTS_PATH.format(
  85. plugin_name=self.path_name)
  86. def get_attributes_metadata(self):
  87. return self.plugin.attributes_metadata
  88. @property
  89. def attributes_metadata(self):
  90. return self.get_attributes_metadata()
  91. def _add_defaults_to_task(self, task, roles_metadata):
  92. """Add required fault tolerance and cwd params to tasks.
  93. :param task: task
  94. :type task: dict
  95. :param roles_metadata: node roles metadata
  96. :type roles_metadata: dict
  97. :return: task
  98. :rtype: dict
  99. """
  100. if task.get('parameters'):
  101. task['parameters'].setdefault(
  102. 'cwd', self.slaves_scripts_path)
  103. if task.get('type') == consts.ORCHESTRATOR_TASK_TYPES.group:
  104. try:
  105. task.setdefault(
  106. 'fault_tolerance',
  107. roles_metadata[task['id']]['fault_tolerance']
  108. )
  109. except KeyError:
  110. pass
  111. return task
  112. def get_deployment_graph(self, graph_type=None):
  113. if graph_type is None:
  114. graph_type = consts.DEFAULT_DEPLOYMENT_GRAPH_TYPE
  115. deployment_tasks = []
  116. graph_metadata = {}
  117. graph_instance = nailgun.objects.DeploymentGraph.get_for_model(
  118. self.plugin, graph_type)
  119. roles_metadata = self.plugin.roles_metadata
  120. if graph_instance:
  121. graph_metadata = nailgun.objects.DeploymentGraph.get_metadata(
  122. graph_instance)
  123. for task in nailgun.objects.DeploymentGraph.get_tasks(
  124. graph_instance):
  125. deployment_tasks.append(
  126. self._add_defaults_to_task(task, roles_metadata)
  127. )
  128. graph_metadata['tasks'] = deployment_tasks
  129. return graph_metadata
  130. def get_deployment_tasks(self, graph_type=None):
  131. return self.get_deployment_graph(graph_type)['tasks']
  132. def get_tasks(self):
  133. tasks = self.plugin.tasks
  134. slave_path = self.slaves_scripts_path
  135. for task in tasks:
  136. task['roles'] = task.get('role')
  137. parameters = task.get('parameters')
  138. if parameters is not None:
  139. parameters.setdefault('cwd', slave_path)
  140. return tasks
  141. @property
  142. def tasks(self):
  143. return self.get_tasks()
  144. @property
  145. def volumes_metadata(self):
  146. return self.plugin.volumes_metadata
  147. @property
  148. def components_metadata(self):
  149. return self.plugin.components_metadata
  150. @property
  151. def bond_attributes_metadata(self):
  152. return self.plugin.bond_attributes_metadata
  153. @property
  154. def nic_attributes_metadata(self):
  155. return self.plugin.nic_attributes_metadata
  156. @property
  157. def node_attributes_metadata(self):
  158. return self.plugin.node_attributes_metadata
  159. @property
  160. def releases(self):
  161. return self.plugin.releases
  162. @property
  163. def normalized_roles_metadata(self):
  164. """Block plugin disabling if nodes with plugin-provided roles exist"""
  165. result = {}
  166. for role, meta in six.iteritems(self.plugin.roles_metadata):
  167. condition = "settings:{0}.metadata.enabled == false".format(
  168. self.plugin.name)
  169. meta = copy.copy(meta)
  170. meta['restrictions'] = [condition] + meta.get('restrictions', [])
  171. result[role] = meta
  172. return result
  173. @staticmethod
  174. def _is_release_version_compatible(rel_version, plugin_rel_version):
  175. """Checks if release version is compatible with plugin version.
  176. :param rel_version: Release version
  177. :type rel_version: str
  178. :param plugin_rel_version: Plugin release version
  179. :type plugin_rel_version: str
  180. :return: True if compatible, False if not
  181. :rtype: bool
  182. """
  183. rel_os, rel_fuel = rel_version.split('-')
  184. plugin_os, plugin_rel = plugin_rel_version.split('-')
  185. return rel_os.startswith(plugin_os) and rel_fuel.startswith(plugin_rel)
  186. def validate_compatibility(self, cluster):
  187. """Validates if plugin is compatible with cluster.
  188. - validates operating systems
  189. - modes of clusters (simple or ha)
  190. - release version
  191. :param cluster: A cluster instance
  192. :type cluster: nailgun.db.sqlalchemy.models.cluster.Cluster
  193. :return: True if compatible, False if not
  194. :rtype: bool
  195. """
  196. cluster_os = cluster.release.operating_system.lower()
  197. for release in self.plugin.releases:
  198. if cluster_os != release['os'].lower():
  199. continue
  200. # plugin writer should be able to specify ha in release['mode']
  201. # and know nothing about ha_compact
  202. if not any(
  203. cluster.mode.startswith(mode) for mode in release['mode']
  204. ):
  205. continue
  206. if not self._is_release_version_compatible(
  207. cluster.release.version, release['version']
  208. ):
  209. continue
  210. return True
  211. return False
  212. def get_release_info(self, release):
  213. """Get plugin release information which corresponds to given release.
  214. :returns: release info
  215. :rtype: dict
  216. """
  217. rel_os = release.operating_system.lower()
  218. version = release.version
  219. release_info = filter(
  220. lambda r: (
  221. r['os'] == rel_os and
  222. self._is_release_version_compatible(version, r['version'])),
  223. self.plugin.releases)
  224. return release_info[0]
  225. def repo_files(self, cluster):
  226. release_info = self.get_release_info(cluster.release)
  227. repo_path = os.path.join(
  228. settings.PLUGINS_PATH,
  229. self.path_name,
  230. release_info['repository_path'],
  231. '*')
  232. return glob.glob(repo_path)
  233. def repo_url(self, cluster):
  234. release_info = self.get_release_info(cluster.release)
  235. repo_base = settings.PLUGINS_REPO_URL.format(
  236. master_ip=settings.MASTER_IP,
  237. plugin_name=self.path_name)
  238. return urljoin(
  239. repo_base,
  240. release_info['repository_path']
  241. )
  242. def master_scripts_path(self, cluster):
  243. release_info = self.get_release_info(cluster.release)
  244. # NOTE(eli): we cannot user urljoin here, because it
  245. # works wrong, if protocol is rsync
  246. base_url = settings.PLUGINS_SLAVES_RSYNC.format(
  247. master_ip=settings.MASTER_IP,
  248. plugin_name=self.path_name)
  249. return '{0}{1}'.format(
  250. base_url,
  251. release_info['deployment_scripts_path'])
  252. class PluginAdapterV1(PluginAdapterBase):
  253. """Plugins attributes class for package version 1.0.0"""
  254. loader_class = loaders.PluginLoaderV1
  255. @property
  256. def attributes_processors(self):
  257. ap = super(PluginAdapterV1, self).attributes_processors
  258. ap.update({
  259. 'tasks': self._process_legacy_tasks
  260. })
  261. return ap
  262. @staticmethod
  263. def _process_legacy_tasks(tasks):
  264. if not tasks:
  265. return []
  266. for task in tasks:
  267. role = task['role']
  268. if isinstance(role, list) and 'controller' in role:
  269. role.append('primary-controller')
  270. return tasks
  271. def get_tasks(self):
  272. tasks = self.plugin.tasks
  273. slave_path = self.slaves_scripts_path
  274. for task in tasks:
  275. task['roles'] = task.get('role')
  276. role = task['role']
  277. if isinstance(role, list) \
  278. and ('controller' in role) \
  279. and ('primary-controller' not in role):
  280. role.append('primary-controller')
  281. parameters = task.get('parameters')
  282. if parameters is not None:
  283. parameters.setdefault('cwd', slave_path)
  284. return tasks
  285. @property
  286. def path_name(self):
  287. """Returns a name and full version
  288. e.g. if there is a plugin with name "plugin_name" and version
  289. is "1.0.0", the method returns "plugin_name-1.0.0"
  290. """
  291. return self.full_name
  292. class PluginAdapterV2(PluginAdapterBase):
  293. """Plugins attributes class for package version 2.0.0"""
  294. loader_class = loaders.PluginLoaderV1
  295. @property
  296. def path_name(self):
  297. """Returns a name and major version of the plugin
  298. e.g. if there is a plugin with name "plugin_name" and version
  299. is "1.0.0", the method returns "plugin_name-1.0".
  300. It's different from previous version because in previous
  301. version we did not have plugin updates, in 2.0.0 version
  302. we should expect different plugin path.
  303. See blueprint: https://blueprints.launchpad.net/fuel/+spec
  304. /plugins-security-fixes-delivery
  305. """
  306. return u'{0}-{1}'.format(self.plugin.name, self._major_version)
  307. @property
  308. def _major_version(self):
  309. """Returns major version of plugin's version
  310. e.g. if plugin has 1.2.3 version, the method returns 1.2
  311. """
  312. version_tuple = StrictVersion(self.plugin.version).version
  313. major = '.'.join(map(str, version_tuple[:2]))
  314. return major
  315. class PluginAdapterV3(PluginAdapterV2):
  316. """Plugin wrapper class for package version 3.0.0"""
  317. loader_class = loaders.PluginLoaderV3
  318. def _process_deployment_tasks(self, deployment_tasks):
  319. dg = nailgun.objects.DeploymentGraph.get_for_model(
  320. self.plugin, graph_type=consts.DEFAULT_DEPLOYMENT_GRAPH_TYPE)
  321. if dg:
  322. nailgun.objects.DeploymentGraph.update(
  323. dg, {'tasks': deployment_tasks})
  324. else:
  325. nailgun.objects.DeploymentGraph.create_for_model(
  326. {'tasks': deployment_tasks}, self.plugin)
  327. return deployment_tasks
  328. @property
  329. def attributes_processors(self):
  330. ap = super(PluginAdapterV3, self).attributes_processors
  331. ap.update({
  332. 'deployment_tasks': self._process_deployment_tasks
  333. })
  334. return ap
  335. class PluginAdapterV4(PluginAdapterV3):
  336. """Plugin wrapper class for package version 4.0.0"""
  337. loader_class = loaders.PluginLoaderV4
  338. class PluginAdapterV5(PluginAdapterV4):
  339. """Plugin wrapper class for package version 5.0.0"""
  340. loader_class = loaders.PluginLoaderV5
  341. _release_fields_to_db_fields = {
  342. "attributes": "attributes_metadata",
  343. "networks": "networks_metadata",
  344. "network_roles": "network_roles_metadata",
  345. "volumes": "volumes_metadata",
  346. "roles": "roles_metadata",
  347. "components": "components_metadata",
  348. "vmware_attributes": "vmware_attributes_metadata",
  349. "os": "operating_system"
  350. }
  351. @property
  352. def attributes_processors(self):
  353. ap = super(PluginAdapterV5, self).attributes_processors
  354. ap.update({
  355. 'releases': self._process_releases
  356. })
  357. return ap
  358. def _create_release_from_configuration(self, configuration):
  359. """Create templated release and graphs for given configuration.
  360. :param configuration:
  361. :return:
  362. """
  363. # deployment tasks not supposed for the release description
  364. # but we fix this developer mistake automatically
  365. # apply base template
  366. base_release = configuration.pop('base_release', None)
  367. if base_release:
  368. base_release.update(configuration)
  369. configuration = base_release
  370. configuration['state'] = consts.RELEASE_STATES.available
  371. # remap fields
  372. for alias, name in six.iteritems(self._release_fields_to_db_fields):
  373. value = configuration.pop(alias, None)
  374. if value is not None:
  375. configuration[name] = value
  376. nailgun.objects.Release.create(configuration)
  377. def _process_releases(self, releases_records):
  378. """Split new release records from old-style release-deps records.
  379. :param releases_records: list of plugins and releases data
  380. :type releases_records: list
  381. :return: configurations that are extending existing
  382. :rtype: list
  383. """
  384. extend_releases = []
  385. for release in releases_records:
  386. is_basic_release = release.get('is_release', False)
  387. if is_basic_release:
  388. self._create_release_from_configuration(release)
  389. else:
  390. extend_releases.append(release)
  391. return extend_releases
  392. class PluginAdapterV6(PluginAdapterV5):
  393. """Plugin wrapper class for package version 6.0.0"""
  394. loader_class = loaders.PluginLoaderV6
  395. @property
  396. def _release_fields_to_db_fields(self):
  397. fields = dict(super(PluginAdapterV6, self).
  398. _release_fields_to_db_fields)
  399. fields['tags'] = 'tags_metadata'
  400. return fields
  401. __plugins_mapping = {
  402. '1.0.': PluginAdapterV1,
  403. '2.0.': PluginAdapterV2,
  404. '3.0.': PluginAdapterV3,
  405. '4.0.': PluginAdapterV4,
  406. '5.0.': PluginAdapterV5,
  407. '6.0.': PluginAdapterV6
  408. }
  409. def get_supported_versions():
  410. return list(__plugins_mapping)
  411. def get_adapter_for_package_version(plugin_version):
  412. """Get plugin adapter class for plugin version.
  413. :param plugin_version: plugin version string
  414. :type plugin_version: basestring|str
  415. :return: plugin loader class
  416. :rtype: loaders.PluginLoader|None
  417. """
  418. for plugin_version_head in __plugins_mapping:
  419. if plugin_version.startswith(plugin_version_head):
  420. return __plugins_mapping[plugin_version_head]
  421. def wrap_plugin(plugin):
  422. """Creates plugin object with specific class version
  423. :param plugin: plugin db object
  424. :returns: cluster attribute object
  425. """
  426. package_version = plugin.package_version
  427. attr_class = get_adapter_for_package_version(package_version)
  428. if not attr_class:
  429. supported_versions = ', '.join(get_supported_versions())
  430. raise errors.PackageVersionIsNotCompatible(
  431. 'Plugin id={0} package_version={1} '
  432. 'is not supported by Nailgun, currently '
  433. 'supported versions {2}'.format(
  434. plugin.id, package_version, supported_versions))
  435. return attr_class(plugin)