Fuel UI
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

base.py 22KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746
  1. # -*- coding: utf-8 -*-
  2. # Copyright 2013 Mirantis, Inc.
  3. #
  4. # Licensed under the Apache License, Version 2.0 (the "License"); you may
  5. # not use this file except in compliance with the License. You may obtain
  6. # a copy of the License at
  7. #
  8. # http://www.apache.org/licenses/LICENSE-2.0
  9. #
  10. # Unless required by applicable law or agreed to in writing, software
  11. # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  12. # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  13. # License for the specific language governing permissions and limitations
  14. # under the License.
  15. from datetime import datetime
  16. from decorator import decorator
  17. from oslo_serialization import jsonutils
  18. import six
  19. import traceback
  20. import yaml
  21. from distutils.version import StrictVersion
  22. from sqlalchemy import exc as sa_exc
  23. import web
  24. from nailgun.api.v1.validators.base import BaseDefferedTaskValidator
  25. from nailgun.api.v1.validators.base import BasicValidator
  26. from nailgun.api.v1.validators.orchestrator_graph import \
  27. GraphSolverTasksValidator
  28. from nailgun import consts
  29. from nailgun.db import db
  30. from nailgun import errors
  31. from nailgun.logger import logger
  32. from nailgun import objects
  33. from nailgun.objects.serializers.base import BasicSerializer
  34. from nailgun.orchestrator import orchestrator_graph
  35. from nailgun.settings import settings
  36. from nailgun import transactions
  37. from nailgun import utils
  38. def forbid_client_caching(handler):
  39. if web.ctx.path.startswith("/api"):
  40. web.header('Cache-Control',
  41. 'store, no-cache, must-revalidate,'
  42. ' post-check=0, pre-check=0')
  43. web.header('Pragma', 'no-cache')
  44. dt = datetime.fromtimestamp(0).strftime(
  45. '%a, %d %b %Y %H:%M:%S GMT'
  46. )
  47. web.header('Expires', dt)
  48. return handler()
  49. def load_db_driver(handler):
  50. """Wrap all handlers calls so transaction is handled accordingly
  51. rollback if something wrong or commit changes otherwise. Please note,
  52. only HTTPError should be raised up from this function. All another
  53. possible errors should be handled.
  54. """
  55. try:
  56. # execute handler and commit changes if all is ok
  57. response = handler()
  58. db.commit()
  59. return response
  60. except web.HTTPError:
  61. # a special case: commit changes if http error ends with
  62. # 200, 201, 202, etc
  63. if web.ctx.status.startswith('2'):
  64. db.commit()
  65. else:
  66. db.rollback()
  67. raise
  68. except (sa_exc.IntegrityError, sa_exc.DataError) as exc:
  69. # respond a "400 Bad Request" if database constraints were broken
  70. db.rollback()
  71. raise BaseHandler.http(400, exc.message)
  72. except Exception:
  73. db.rollback()
  74. raise
  75. finally:
  76. db.remove()
  77. class BaseHandler(object):
  78. validator = BasicValidator
  79. serializer = BasicSerializer
  80. fields = []
  81. @classmethod
  82. def render(cls, instance, fields=None):
  83. return cls.serializer.serialize(
  84. instance,
  85. fields=fields or cls.fields
  86. )
  87. @classmethod
  88. def http(cls, status_code, msg="", err_list=None, headers=None):
  89. """Raise an HTTP status code.
  90. Useful for returning status
  91. codes like 401 Unauthorized or 403 Forbidden.
  92. :param status_code: the HTTP status code as an integer
  93. :param msg: the message to send along, as a string
  94. :param err_list: list of fields with errors
  95. :param headers: the headers to send along, as a dictionary
  96. """
  97. class _nocontent(web.HTTPError):
  98. message = 'No Content'
  99. def __init__(self):
  100. super(_nocontent, self).__init__(
  101. status='204 No Content',
  102. data=self.message
  103. )
  104. exc_status_map = {
  105. 200: web.ok,
  106. 201: web.created,
  107. 202: web.accepted,
  108. 204: _nocontent,
  109. 301: web.redirect,
  110. 302: web.found,
  111. 400: web.badrequest,
  112. 401: web.unauthorized,
  113. 403: web.forbidden,
  114. 404: web.notfound,
  115. 405: web.nomethod,
  116. 406: web.notacceptable,
  117. 409: web.conflict,
  118. 410: web.gone,
  119. 415: web.unsupportedmediatype,
  120. 500: web.internalerror,
  121. }
  122. # web.py has a poor exception design: some of them receive
  123. # the `message` argument and some of them not. the only
  124. # solution to set custom message is to assign message directly
  125. # to the `data` attribute. though, that won't work for
  126. # the `internalerror` because it tries to do magic with
  127. # application context without explicit `message` argument.
  128. try:
  129. exc = exc_status_map[status_code](message=msg)
  130. except TypeError:
  131. exc = exc_status_map[status_code]()
  132. exc.data = msg
  133. exc.err_list = err_list or []
  134. exc.status_code = status_code
  135. headers = headers or {}
  136. for key, value in headers.items():
  137. web.header(key, value)
  138. return exc
  139. @classmethod
  140. def checked_data(cls, validate_method=None, **kwargs):
  141. try:
  142. data = kwargs.pop('data', web.data())
  143. method = validate_method or cls.validator.validate
  144. valid_data = method(data, **kwargs)
  145. except (
  146. errors.InvalidInterfacesInfo,
  147. errors.InvalidMetadata
  148. ) as exc:
  149. objects.Notification.create({
  150. "topic": "error",
  151. "message": exc.message
  152. })
  153. raise cls.http(400, exc.message)
  154. except (
  155. errors.NotAllowed
  156. ) as exc:
  157. raise cls.http(403, exc.message)
  158. except (
  159. errors.AlreadyExists
  160. ) as exc:
  161. raise cls.http(409, exc.message)
  162. except (
  163. errors.InvalidData,
  164. errors.NodeOffline,
  165. errors.NoDeploymentTasks,
  166. errors.UnavailableRelease,
  167. errors.CannotCreate,
  168. errors.CannotUpdate,
  169. errors.CannotDelete,
  170. errors.CannotFindExtension,
  171. ) as exc:
  172. raise cls.http(400, exc.message)
  173. except (
  174. errors.ObjectNotFound,
  175. ) as exc:
  176. raise cls.http(404, exc.message)
  177. except Exception as exc:
  178. raise cls.http(500, traceback.format_exc())
  179. return valid_data
  180. def get_object_or_404(self, obj, *args, **kwargs):
  181. """Get object instance by ID
  182. :http: 404 when not found
  183. :returns: object instance
  184. """
  185. log_404 = kwargs.pop("log_404", None)
  186. log_get = kwargs.pop("log_get", None)
  187. uid = kwargs.get("id", (args[0] if args else None))
  188. if uid is None:
  189. if log_404:
  190. getattr(logger, log_404[0])(log_404[1])
  191. raise self.http(404, u'Invalid ID specified')
  192. else:
  193. instance = obj.get_by_uid(uid)
  194. if not instance:
  195. raise self.http(404, u'{0} not found'.format(obj.__name__))
  196. if log_get:
  197. getattr(logger, log_get[0])(log_get[1])
  198. return instance
  199. def get_objects_list_or_404(self, obj, ids):
  200. """Get list of objects
  201. :param obj: model object
  202. :param ids: list of ids
  203. :http: 404 when not found
  204. :returns: list of object instances
  205. """
  206. node_query = obj.filter_by_id_list(None, ids)
  207. objects_count = obj.count(node_query)
  208. if len(set(ids)) != objects_count:
  209. raise self.http(404, '{0} not found'.format(obj.__name__))
  210. return list(node_query)
  211. def raise_task(self, task):
  212. if task.status in [consts.TASK_STATUSES.ready,
  213. consts.TASK_STATUSES.error]:
  214. status = 200
  215. else:
  216. status = 202
  217. raise self.http(status, objects.Task.to_json(task))
  218. @staticmethod
  219. def get_param_as_set(param_name, delimiter=',', default=None):
  220. """Parse array param from web.input()
  221. :param param_name: parameter name in web.input()
  222. :type param_name: str
  223. :param delimiter: delimiter
  224. :type delimiter: str
  225. :returns: list of items
  226. :rtype: set of str or None
  227. """
  228. if param_name in web.input():
  229. param = getattr(web.input(), param_name)
  230. if param == '':
  231. return set()
  232. else:
  233. return set(six.moves.map(
  234. six.text_type.strip,
  235. param.split(delimiter))
  236. )
  237. else:
  238. return default
  239. @staticmethod
  240. def get_requested_mime():
  241. accept = web.ctx.env.get("HTTP_ACCEPT", "application/json")
  242. accept = accept.strip().split(',')[0]
  243. accept = accept.split(';')[0]
  244. return accept
  245. def json_resp(data):
  246. if isinstance(data, (dict, list)) or data is None:
  247. return jsonutils.dumps(data)
  248. else:
  249. return data
  250. @decorator
  251. def handle_errors(func, cls, *args, **kwargs):
  252. try:
  253. return func(cls, *args, **kwargs)
  254. except web.HTTPError as http_error:
  255. if http_error.status_code != 204:
  256. web.header('Content-Type', 'application/json', unique=True)
  257. if http_error.status_code >= 400:
  258. http_error.data = json_resp({
  259. "message": http_error.data,
  260. "errors": http_error.err_list
  261. })
  262. else:
  263. http_error.data = json_resp(http_error.data)
  264. raise
  265. except errors.NailgunException as exc:
  266. logger.exception('NailgunException occured')
  267. http_error = BaseHandler.http(400, exc.message)
  268. web.header('Content-Type', 'text/plain')
  269. raise http_error
  270. # intercepting all errors to avoid huge HTML output
  271. except Exception as exc:
  272. logger.exception('Unexpected exception occured')
  273. http_error = BaseHandler.http(
  274. 500,
  275. (
  276. traceback.format_exc(exc)
  277. if settings.DEVELOPMENT
  278. else 'Unexpected exception, please check logs'
  279. )
  280. )
  281. http_error.data = json_resp(http_error.data)
  282. web.header('Content-Type', 'text/plain')
  283. raise http_error
  284. @decorator
  285. def validate(func, cls, *args, **kwargs):
  286. request_validation_needed = True
  287. resource_type = "single"
  288. if issubclass(
  289. cls.__class__,
  290. CollectionHandler
  291. ) and not func.func_name == "POST":
  292. resource_type = "collection"
  293. if (
  294. func.func_name in ("GET", "DELETE") or
  295. getattr(cls.__class__, 'validator', None) is None or
  296. (resource_type == "single" and not cls.validator.single_schema) or
  297. (resource_type == "collection" and not cls.validator.collection_schema)
  298. ):
  299. request_validation_needed = False
  300. if request_validation_needed:
  301. BaseHandler.checked_data(
  302. cls.validator.validate_request,
  303. resource_type=resource_type
  304. )
  305. return func(cls, *args, **kwargs)
  306. @decorator
  307. def serialize(func, cls, *args, **kwargs):
  308. """Set context-type of response based on Accept header.
  309. This decorator checks Accept header received from client
  310. and returns corresponding wrapper (only JSON is currently
  311. supported). It can be used as is:
  312. @handle_errors
  313. @validate
  314. @serialize
  315. def GET(self):
  316. ...
  317. """
  318. accepted_types = (
  319. "application/json",
  320. "application/x-yaml",
  321. "*/*"
  322. )
  323. accept = cls.get_requested_mime()
  324. if accept not in accepted_types:
  325. raise BaseHandler.http(415)
  326. resp = func(cls, *args, **kwargs)
  327. if accept == 'application/x-yaml':
  328. web.header('Content-Type', 'application/x-yaml', unique=True)
  329. return yaml.dump(resp, default_flow_style=False)
  330. else:
  331. # default is json
  332. web.header('Content-Type', 'application/json', unique=True)
  333. return jsonutils.dumps(resp)
  334. class SingleHandler(BaseHandler):
  335. single = None
  336. validator = BasicValidator
  337. @handle_errors
  338. @serialize
  339. def GET(self, obj_id):
  340. """:returns: JSONized REST object.
  341. :http: * 200 (OK)
  342. * 404 (object not found in db)
  343. """
  344. obj = self.get_object_or_404(self.single, obj_id)
  345. return self.single.to_dict(obj)
  346. @handle_errors
  347. @validate
  348. @serialize
  349. def PUT(self, obj_id):
  350. """:returns: JSONized REST object.
  351. :http: * 200 (OK)
  352. * 404 (object not found in db)
  353. """
  354. obj = self.get_object_or_404(self.single, obj_id)
  355. data = self.checked_data(
  356. self.validator.validate_update,
  357. instance=obj
  358. )
  359. self.single.update(obj, data)
  360. return self.single.to_dict(obj)
  361. @handle_errors
  362. @validate
  363. def DELETE(self, obj_id):
  364. """:returns: Empty string
  365. :http: * 204 (object successfully deleted)
  366. * 404 (object not found in db)
  367. """
  368. obj = self.get_object_or_404(
  369. self.single,
  370. obj_id
  371. )
  372. self.checked_data(
  373. self.validator.validate_delete,
  374. instance=obj
  375. )
  376. self.single.delete(obj)
  377. raise self.http(204)
  378. class CollectionHandler(BaseHandler):
  379. collection = None
  380. validator = BasicValidator
  381. eager = ()
  382. @handle_errors
  383. @validate
  384. @serialize
  385. def GET(self):
  386. """:returns: Collection of JSONized REST objects.
  387. :http: * 200 (OK)
  388. """
  389. q = self.collection.eager(None, self.eager)
  390. return self.collection.to_list(q)
  391. @handle_errors
  392. @validate
  393. def POST(self):
  394. """:returns: JSONized REST object.
  395. :http: * 201 (object successfully created)
  396. * 400 (invalid object data specified)
  397. * 409 (object with such parameters already exists)
  398. """
  399. data = self.checked_data()
  400. try:
  401. new_obj = self.collection.create(data)
  402. except errors.CannotCreate as exc:
  403. raise self.http(400, exc.message)
  404. raise self.http(201, self.collection.single.to_json(new_obj))
  405. class DBSingletonHandler(BaseHandler):
  406. """Manages an object that is supposed to have only one entry in the DB"""
  407. single = None
  408. validator = BasicValidator
  409. not_found_error = "Object not found in the DB"
  410. def get_one_or_404(self):
  411. try:
  412. instance = self.single.get_one(fail_if_not_found=True)
  413. except errors.ObjectNotFound:
  414. raise self.http(404, self.not_found_error)
  415. return instance
  416. @handle_errors
  417. @validate
  418. @serialize
  419. def GET(self):
  420. """Get singleton object from DB
  421. :http: * 200 (OK)
  422. * 404 (Object not found in DB)
  423. """
  424. instance = self.get_one_or_404()
  425. return self.single.to_dict(instance)
  426. @handle_errors
  427. @validate
  428. @serialize
  429. def PUT(self):
  430. """Change object in DB
  431. :http: * 200 (OK)
  432. * 400 (Invalid data)
  433. * 404 (Object not present in DB)
  434. """
  435. data = self.checked_data(self.validator.validate_update)
  436. instance = self.get_one_or_404()
  437. self.single.update(instance, data)
  438. return self.single.to_dict(instance)
  439. @handle_errors
  440. @validate
  441. @serialize
  442. def PATCH(self):
  443. """Update object
  444. :http: * 200 (OK)
  445. * 400 (Invalid data)
  446. * 404 (Object not present in DB)
  447. """
  448. data = self.checked_data(self.validator.validate_update)
  449. instance = self.get_one_or_404()
  450. instance.update(utils.dict_merge(
  451. self.single.serializer.serialize(instance), data
  452. ))
  453. return self.single.to_dict(instance)
  454. class OrchestratorDeploymentTasksHandler(SingleHandler):
  455. """Handler for deployment graph serialization."""
  456. validator = GraphSolverTasksValidator
  457. @handle_errors
  458. @validate
  459. @serialize
  460. def GET(self, obj_id):
  461. """:returns: Deployment tasks
  462. :http: * 200 OK
  463. * 404 (object not found)
  464. """
  465. obj = self.get_object_or_404(self.single, obj_id)
  466. end = web.input(end=None).end
  467. start = web.input(start=None).start
  468. graph_type = web.input(graph_type=None).graph_type or None
  469. # web.py depends on [] to understand that there will be multiple inputs
  470. include = web.input(include=[]).include
  471. # merged (cluster + plugins + release) tasks is returned for cluster
  472. # but the own release tasks is returned for release
  473. tasks = self.single.get_deployment_tasks(obj, graph_type=graph_type)
  474. if end or start:
  475. graph = orchestrator_graph.GraphSolver(tasks)
  476. for t in tasks:
  477. if StrictVersion(t.get('version')) >= \
  478. StrictVersion(consts.TASK_CROSS_DEPENDENCY):
  479. raise self.http(400, (
  480. 'Both "start" and "end" parameters are not allowed '
  481. 'for task-based deployment.'))
  482. try:
  483. return graph.filter_subgraph(
  484. end=end, start=start, include=include).node.values()
  485. except errors.TaskNotFound as e:
  486. raise self.http(400, 'Cannot find task {0} by its '
  487. 'name.'.format(e.task_name))
  488. return tasks
  489. @handle_errors
  490. @validate
  491. @serialize
  492. def PUT(self, obj_id):
  493. """:returns: Deployment tasks
  494. :http: * 200 (OK)
  495. * 400 (invalid data specified)
  496. * 404 (object not found in db)
  497. """
  498. obj = self.get_object_or_404(self.single, obj_id)
  499. graph_type = web.input(graph_type=None).graph_type or None
  500. data = self.checked_data(
  501. self.validator.validate_update,
  502. instance=obj
  503. )
  504. deployment_graph = objects.DeploymentGraph.get_for_model(
  505. obj, graph_type=graph_type)
  506. if deployment_graph:
  507. objects.DeploymentGraph.update(
  508. deployment_graph, {'tasks': data})
  509. else:
  510. deployment_graph = objects.DeploymentGraph.create_for_model(
  511. {'tasks': data}, obj, graph_type=graph_type)
  512. return objects.DeploymentGraph.get_tasks(deployment_graph)
  513. def POST(self, obj_id):
  514. """Creation of metadata disallowed
  515. :http: * 405 (method not supported)
  516. """
  517. raise self.http(405, 'Create not supported for this entity')
  518. def DELETE(self, obj_id):
  519. """Deletion of metadata disallowed
  520. :http: * 405 (method not supported)
  521. """
  522. raise self.http(405, 'Delete not supported for this entity')
  523. class TransactionExecutorHandler(BaseHandler):
  524. def start_transaction(self, cluster, options):
  525. """Starts new transaction.
  526. :param cluster: the cluster object
  527. :param options: the transaction parameters
  528. :return: JSONized task object
  529. """
  530. try:
  531. manager = transactions.TransactionsManager(cluster.id)
  532. self.raise_task(manager.execute(**options))
  533. except errors.ObjectNotFound as e:
  534. raise self.http(404, e.message)
  535. except errors.DeploymentAlreadyStarted as e:
  536. raise self.http(409, e.message)
  537. except errors.InvalidData as e:
  538. raise self.http(400, e.message)
  539. # TODO(enchantner): rewrite more handlers to inherit from this
  540. # and move more common code here, this is deprecated handler
  541. class DeferredTaskHandler(TransactionExecutorHandler):
  542. """Abstract Deferred Task Handler"""
  543. validator = BaseDefferedTaskValidator
  544. single = objects.Task
  545. log_message = u"Starting deferred task on environment '{env_id}'"
  546. log_error = u"Error during execution of deferred task " \
  547. u"on environment '{env_id}': {error}"
  548. task_manager = None
  549. @classmethod
  550. def get_options(cls):
  551. return {}
  552. @classmethod
  553. def get_transaction_options(cls, cluster, options):
  554. """Finds graph for this action."""
  555. return None
  556. @handle_errors
  557. @validate
  558. def PUT(self, cluster_id):
  559. """:returns: JSONized Task object.
  560. :http: * 202 (task successfully executed)
  561. * 400 (invalid object data specified)
  562. * 404 (environment is not found)
  563. * 409 (task with such parameters already exists)
  564. """
  565. cluster = self.get_object_or_404(
  566. objects.Cluster,
  567. cluster_id,
  568. log_404=(
  569. u"warning",
  570. u"Error: there is no cluster "
  571. u"with id '{0}' in DB.".format(cluster_id)
  572. )
  573. )
  574. logger.info(self.log_message.format(env_id=cluster_id))
  575. try:
  576. options = self.get_options()
  577. except ValueError as e:
  578. raise self.http(400, six.text_type(e))
  579. try:
  580. self.validator.validate(cluster)
  581. except errors.NailgunException as e:
  582. raise self.http(400, e.message)
  583. if objects.Release.is_lcm_supported(cluster.release):
  584. # try to get new graph to run transaction manager
  585. try:
  586. transaction_options = self.get_transaction_options(
  587. cluster, options
  588. )
  589. except errors.NailgunException as e:
  590. logger.exception("Failed to get transaction options")
  591. raise self.http(400, msg=six.text_type(e))
  592. if transaction_options:
  593. return self.start_transaction(cluster, transaction_options)
  594. try:
  595. task_manager = self.task_manager(cluster_id=cluster.id)
  596. task = task_manager.execute(**options)
  597. except (
  598. errors.AlreadyExists,
  599. errors.StopAlreadyRunning
  600. ) as exc:
  601. raise self.http(409, exc.message)
  602. except (
  603. errors.DeploymentNotRunning,
  604. errors.NoDeploymentTasks,
  605. errors.WrongNodeStatus,
  606. errors.UnavailableRelease,
  607. errors.CannotBeStopped,
  608. ) as exc:
  609. raise self.http(400, exc.message)
  610. except Exception as exc:
  611. logger.error(
  612. self.log_error.format(
  613. env_id=cluster_id,
  614. error=str(exc)
  615. )
  616. )
  617. # let it be 500
  618. raise
  619. self.raise_task(task)