Fuel UI
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

base.py 19KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550
  1. # -*- coding: utf-8 -*-
  2. # Copyright 2013 Mirantis, Inc.
  3. #
  4. # Licensed under the Apache License, Version 2.0 (the "License"); you may
  5. # not use this file except in compliance with the License. You may obtain
  6. # a copy of the License at
  7. #
  8. # http://www.apache.org/licenses/LICENSE-2.0
  9. #
  10. # Unless required by applicable law or agreed to in writing, software
  11. # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  12. # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  13. # License for the specific language governing permissions and limitations
  14. # under the License.
  15. """
  16. Base classes for objects and collections
  17. """
  18. import collections
  19. import functools
  20. from itertools import ifilter
  21. import operator
  22. from oslo_serialization import jsonutils
  23. import six
  24. from sqlalchemy import and_, not_
  25. from sqlalchemy.orm import joinedload
  26. from sqlalchemy.orm import Query
  27. from nailgun.objects.serializers.base import BasicSerializer
  28. from nailgun.db import db
  29. from nailgun import errors
  30. class NailgunObject(object):
  31. """Base class for objects"""
  32. #: Serializer class for object
  33. serializer = BasicSerializer
  34. #: SQLAlchemy model for object
  35. model = None
  36. @classmethod
  37. def get_by_uid(cls, uid, fail_if_not_found=False, lock_for_update=False):
  38. """Get instance by it's uid (PK in case of SQLAlchemy)
  39. :param uid: uid of object
  40. :param fail_if_not_found: raise an exception if object is not found
  41. :param lock_for_update: lock returned object for update (DB mutex)
  42. :returns: instance of an object (model)
  43. """
  44. q = db().query(cls.model)
  45. if lock_for_update:
  46. # todo(ikutukov): replace to the with_for_update
  47. # http://docs.sqlalchemy.org/en/latest/orm/query.html#sqlalchemy.
  48. # orm.query.Query.with_for_update
  49. q = q.with_lockmode('update')
  50. res = q.get(uid)
  51. if not res and fail_if_not_found:
  52. raise errors.ObjectNotFound(
  53. "Object '{0}' with UID={1} is not found in DB".format(
  54. cls.__name__,
  55. uid
  56. )
  57. )
  58. return res
  59. @classmethod
  60. def create(cls, data):
  61. """Create object instance with specified parameters in DB
  62. :param data: dictionary of key-value pairs as object fields
  63. :returns: instance of an object (model)
  64. """
  65. new_obj = cls.model()
  66. for key, value in six.iteritems(data):
  67. setattr(new_obj, key, value)
  68. db().add(new_obj)
  69. db().flush()
  70. return new_obj
  71. @classmethod
  72. def update(cls, instance, data):
  73. """Update existing instance with specified parameters
  74. :param instance: object (model) instance
  75. :param data: dictionary of key-value pairs as object fields
  76. :returns: instance of an object (model)
  77. """
  78. instance.update(data)
  79. db().add(instance)
  80. db().flush()
  81. return instance
  82. @classmethod
  83. def delete(cls, instance):
  84. """Delete object (model) instance
  85. :param instance: object (model) instance
  86. :returns: None
  87. """
  88. db().delete(instance)
  89. db().flush()
  90. @classmethod
  91. def bulk_delete(cls, instance_ids):
  92. db().query(cls.model).filter(
  93. cls.model.id.in_(instance_ids)).delete(synchronize_session='fetch')
  94. @classmethod
  95. def save(cls, instance=None):
  96. """Save current changes for instance in DB.
  97. Current transaction will be commited
  98. (in case of SQLAlchemy).
  99. :param instance: object (model) instance
  100. :returns: None
  101. """
  102. if instance:
  103. db().add(instance)
  104. db().commit()
  105. @classmethod
  106. def to_dict(cls, instance, fields=None, serializer=None):
  107. """Serialize instance to Python dict
  108. :param instance: object (model) instance
  109. :param fields: exact fields to serialize
  110. :param serializer: the custom serializer
  111. :returns: serialized object (model) as dictionary
  112. """
  113. serializer = serializer or cls.serializer
  114. return serializer.serialize(instance, fields=fields)
  115. @classmethod
  116. def to_json(cls, instance, fields=None, serializer=None):
  117. """Serialize instance to JSON
  118. :param instance: object (model) instance
  119. :param fields: exact fields to serialize
  120. :param serializer: the custom serializer
  121. :returns: serialized object (model) as JSON string
  122. """
  123. return jsonutils.dumps(
  124. cls.to_dict(instance, fields=fields, serializer=serializer)
  125. )
  126. @classmethod
  127. def compare(cls, instance, other, order_by):
  128. for field in order_by:
  129. a = getattr(instance, field['name'])
  130. b = getattr(other, field['name'])
  131. if a < b:
  132. return field['lt']
  133. elif a > b:
  134. return field['gt']
  135. return 0
  136. class NailgunCollection(object):
  137. """Base class for object collections"""
  138. #: Single object class
  139. single = NailgunObject
  140. @classmethod
  141. def content_range(cls, first, last, total, valid):
  142. """Structure to set Content-Range header
  143. Defines structure necessary to implement paged requests.
  144. "total" is needed to let client calculate how many pages are available.
  145. "valid" is used to indicate that the requested page is valid
  146. (contains data) or not (outside of data range).
  147. Used in NailgunCollection.scope()
  148. :param first: first element (row) returned
  149. :param last: last element (row) returned
  150. :param total: total number of elements/rows (before pagination)
  151. :param valid: whether the pagination is within data range or not
  152. :returns: ContentRange object (collections.namedtuple) with 4 fields
  153. """
  154. rng = collections.namedtuple('ContentRange',
  155. ['first', 'last', 'total', 'valid'])
  156. rng.first = first
  157. rng.last = last
  158. rng.total = total
  159. rng.valid = valid
  160. return rng
  161. @classmethod
  162. def _is_iterable(cls, obj):
  163. return isinstance(
  164. obj,
  165. collections.Iterable
  166. )
  167. @classmethod
  168. def _is_query(cls, obj):
  169. return isinstance(
  170. obj,
  171. Query
  172. )
  173. @classmethod
  174. def all(cls):
  175. """Get all instances of this object (model)
  176. :returns: iterable (SQLAlchemy query)
  177. """
  178. return db().query(cls.single.model)
  179. @classmethod
  180. def scope(cls, pagination=None, filter_by=None):
  181. """Return a query to collection's objects and ContentRange object
  182. Return a filtered and paged query, according to the provided pagination
  183. (see api.v1.handlers.base.Pagination)
  184. Also return ContentRange - object with index of first element, last
  185. element and total count of elements in query(after filtering), and
  186. a 'valid' parameter to indicate that the paging scope (limit + offset)
  187. is valid or not (resulted in no data while there was data to provide)
  188. :param pagination: Pagination object
  189. :param filter_by: dict to filter objects {field1: value1, ...}
  190. :returns: SQLAlchemy query and ContentRange object
  191. """
  192. query = cls.all()
  193. content_range = None
  194. if filter_by:
  195. query = query.filter_by(**filter_by)
  196. query_full = query
  197. if pagination:
  198. if pagination.limit > 0 or pagination.limit is None:
  199. if pagination.order_by:
  200. query = cls.order_by(query, pagination.order_by)
  201. if pagination.offset:
  202. query = query.offset(pagination.offset)
  203. if pagination.limit > 0:
  204. query = query.limit(pagination.limit)
  205. else:
  206. # making an empty result
  207. query = query.filter(False)
  208. if pagination.offset or pagination.limit is not None:
  209. total = query_full.count()
  210. selected = query.count() if pagination.limit != 0 else 0
  211. # first element index=1
  212. first = pagination.offset + 1 if pagination.offset else 1
  213. if selected == 0 or pagination.limit == 0:
  214. # no data, report first and last as 0
  215. first = last = 0
  216. elif pagination.limit > 0:
  217. last = min(first + pagination.limit - 1, total)
  218. else:
  219. last = total
  220. valid = selected > 0 or pagination.limit == 0 or total == 0
  221. content_range = cls.content_range(first, last, total, valid)
  222. return query, content_range
  223. @classmethod
  224. def _query_order_by(cls, query, order_by):
  225. """Adds order by clause into SQLAlchemy query
  226. :param query: SQLAlchemy query
  227. :param order_by: tuple of model fields names for ORDER BY criterion
  228. to SQLAlchemy query. If name starts with '-' desc ordering applies,
  229. else asc.
  230. """
  231. order_args = []
  232. for field_name in order_by:
  233. if field_name.startswith('-'):
  234. field_name = field_name.lstrip('-')
  235. ordering = 'desc'
  236. else:
  237. ordering = 'asc'
  238. field = getattr(cls.single.model, field_name)
  239. o_func = getattr(field, ordering)
  240. order_args.append(o_func())
  241. query = query.order_by(*order_args)
  242. return query
  243. @classmethod
  244. def _iterable_order_by(cls, iterable, order_by):
  245. """Sort iterable by field names in order_by
  246. :param iterable: model objects collection
  247. :param order_by: tuple of model fields names for sorting.
  248. If name starts with '-' desc ordering applies, else asc.
  249. """
  250. order_by_fields = []
  251. for field_name in order_by:
  252. if field_name.startswith('-'):
  253. order_by_fields.append({'name': field_name.lstrip('-'),
  254. 'lt': 1, 'gt': -1})
  255. else:
  256. order_by_fields.append({'name': field_name,
  257. 'lt': -1, 'gt': 1})
  258. # 'cmp' argument for 'sorted' function is removed in python3.
  259. # Next code should work fine for both python2 and python3
  260. key = functools.cmp_to_key(functools.partial(cls.single.compare,
  261. order_by=order_by_fields))
  262. return sorted(iterable, key=key)
  263. @classmethod
  264. def get_iterable(cls, iterable, require=True):
  265. """Return either iterable or cls.all() when possible
  266. :param iterable: model objects collection
  267. :returns: original iterable or an SQLAlchemy query
  268. """
  269. if iterable is not None:
  270. if cls._is_iterable(iterable) or cls._is_query(iterable):
  271. return iterable
  272. else:
  273. raise TypeError("'%s' object is not iterable" % type(iterable))
  274. elif cls.single.model:
  275. return cls.all()
  276. elif require:
  277. raise ValueError('iterable not provided and single.model not set')
  278. @classmethod
  279. def order_by(cls, iterable, order_by):
  280. """Order given iterable by specified order_by.
  281. :param iterable: model objects collection
  282. :param order_by: tuple of model fields names or single field name for
  283. ORDER BY criterion to SQLAlchemy query. If name starts with '-'
  284. desc ordering applies, else asc.
  285. :type order_by: tuple of strings or string
  286. :returns: ordered iterable (SQLAlchemy query)
  287. """
  288. if not iterable or not order_by:
  289. return iterable
  290. use_iterable = cls.get_iterable(iterable)
  291. if not isinstance(order_by, (list, tuple)):
  292. order_by = (order_by,)
  293. if cls._is_query(use_iterable):
  294. return cls._query_order_by(use_iterable, order_by)
  295. elif cls._is_iterable(use_iterable):
  296. return cls._iterable_order_by(use_iterable, order_by)
  297. @classmethod
  298. def filter_by(cls, iterable, **kwargs):
  299. """Filter given iterable by specified kwargs.
  300. In case if iterable=None filters all object instances
  301. :param iterable: iterable (SQLAlchemy query)
  302. :param order_by: tuple of model fields names for ORDER BY criterion
  303. to SQLAlchemy query. If name starts with '-' desc ordering applies,
  304. else asc.
  305. :returns: filtered iterable (SQLAlchemy query)
  306. """
  307. use_iterable = cls.get_iterable(iterable)
  308. if cls._is_query(use_iterable):
  309. return use_iterable.filter_by(**kwargs)
  310. elif cls._is_iterable(use_iterable):
  311. return ifilter(
  312. lambda i: all(
  313. (getattr(i, k) == v for k, v in six.iteritems(kwargs))
  314. ),
  315. use_iterable
  316. )
  317. else:
  318. raise TypeError("First argument should be iterable")
  319. @classmethod
  320. def filter_by_not(cls, iterable, **kwargs):
  321. """Filter given iterable by specified kwargs with negation.
  322. In case if `iterable` is `None` filters all object instances.
  323. :param iterable: iterable (SQLAlchemy query)
  324. :returns: filtered iterable (SQLAlchemy query)
  325. """
  326. use_iterable = cls.get_iterable(iterable)
  327. if cls._is_query(use_iterable):
  328. conditions = []
  329. for key, value in six.iteritems(kwargs):
  330. conditions.append(
  331. getattr(cls.single.model, key) == value
  332. )
  333. return use_iterable.filter(not_(and_(*conditions)))
  334. elif cls._is_iterable(use_iterable):
  335. return ifilter(
  336. lambda i: not all(
  337. (getattr(i, k) == v for k, v in six.iteritems(kwargs))
  338. ),
  339. use_iterable
  340. )
  341. @classmethod
  342. def lock_for_update(cls, iterable):
  343. """Use SELECT FOR UPDATE on a given iterable (query).
  344. In case if iterable=None returns all object instances
  345. :param iterable: iterable (SQLAlchemy query)
  346. :returns: filtered iterable (SQLAlchemy query)
  347. """
  348. use_iterable = cls.get_iterable(iterable)
  349. if cls._is_query(use_iterable):
  350. return use_iterable.with_lockmode('update')
  351. elif cls._is_iterable(use_iterable):
  352. # we can't lock abstract iterable, so returning as is
  353. # for compatibility
  354. return use_iterable
  355. @classmethod
  356. def filter_by_list(cls, iterable, field_name, list_of_values,
  357. order_by=()):
  358. """Filter given iterable by list of list_of_values.
  359. In case if iterable=None filters all object instances
  360. :param iterable: iterable (SQLAlchemy query)
  361. :param field_name: filtering field name
  362. :param list_of_values: list of values for objects filtration
  363. :returns: filtered iterable (SQLAlchemy query)
  364. """
  365. field_getter = operator.attrgetter(field_name)
  366. use_iterable = cls.get_iterable(iterable)
  367. if cls._is_query(use_iterable):
  368. result = use_iterable.filter(
  369. field_getter(cls.single.model).in_(list_of_values)
  370. )
  371. result = cls.order_by(result, order_by)
  372. return result
  373. elif cls._is_iterable(use_iterable):
  374. return ifilter(
  375. lambda i: field_getter(i) in list_of_values,
  376. use_iterable
  377. )
  378. @classmethod
  379. def filter_by_id_list(cls, iterable, uid_list):
  380. """Filter given iterable by list of uids.
  381. In case if iterable=None filters all object instances
  382. :param iterable: iterable (SQLAlchemy query)
  383. :param uid_list: list of uids for objects
  384. :returns: filtered iterable (SQLAlchemy query)
  385. """
  386. return cls.filter_by_list(
  387. iterable,
  388. 'id',
  389. uid_list,
  390. )
  391. @classmethod
  392. def eager_base(cls, iterable, options):
  393. """Eager load linked object instances (SQLAlchemy FKs).
  394. In case if iterable=None applies to all object instances
  395. :param iterable: iterable (SQLAlchemy query)
  396. :param options: list of sqlalchemy eagerload types
  397. :returns: iterable (SQLAlchemy query)
  398. """
  399. use_iterable = cls.get_iterable(iterable)
  400. if options:
  401. return use_iterable.options(*options)
  402. return use_iterable
  403. @classmethod
  404. def eager(cls, iterable, fields):
  405. """Eager load linked object instances (SQLAlchemy FKs).
  406. By default joinedload will be applied to every field.
  407. If you want to use custom eagerload method - use eager_base
  408. In case if iterable=None applies to all object instances
  409. :param iterable: iterable (SQLAlchemy query)
  410. :param fields: list of links (model FKs) to eagerload
  411. :returns: iterable (SQLAlchemy query)
  412. """
  413. options = [joinedload(field) for field in fields]
  414. return cls.eager_base(iterable, options)
  415. @classmethod
  416. def count(cls, iterable=None):
  417. use_iterable = cls.get_iterable(iterable)
  418. if cls._is_query(use_iterable):
  419. return use_iterable.count()
  420. elif cls._is_iterable(use_iterable):
  421. return len(list(iterable))
  422. @classmethod
  423. def to_list(cls, iterable=None, fields=None, serializer=None):
  424. """Serialize iterable to list of dicts
  425. In case if iterable=None serializes all object instances
  426. :param iterable: iterable (SQLAlchemy query)
  427. :param fields: exact fields to serialize
  428. :param serializer: the custom serializer
  429. :returns: collection of objects as a list of dicts
  430. """
  431. use_iterable = cls.get_iterable(iterable)
  432. return [
  433. cls.single.to_dict(o, fields=fields, serializer=serializer)
  434. for o in use_iterable
  435. ]
  436. @classmethod
  437. def to_json(cls, iterable=None, fields=None):
  438. """Serialize iterable to JSON
  439. In case if iterable=None serializes all object instances
  440. :param iterable: iterable (SQLAlchemy query)
  441. :param fields: exact fields to serialize
  442. :returns: collection of objects as a JSON string
  443. """
  444. return jsonutils.dumps(
  445. cls.to_list(
  446. fields=fields,
  447. iterable=iterable
  448. )
  449. )
  450. @classmethod
  451. def create(cls, data):
  452. """Create object instance with specified parameters in DB
  453. :param data: dictionary of key-value pairs as object fields
  454. :returns: instance of an object (model)
  455. """
  456. return cls.single.create(data)
  457. @classmethod
  458. def options(cls, iterable, *args):
  459. """Apply the given list of mapper options.
  460. In case if iterable=None applies to all object instances
  461. :param iterable: iterable (SQLAlchemy query)
  462. :param options: list of sqlalchemy mapper options
  463. :returns: iterable (SQLAlchemy query)
  464. """
  465. use_iterable = cls.get_iterable(iterable)
  466. if args:
  467. return use_iterable.options(*args)
  468. return use_iterable