Global requirements for OpenStack
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

check.py 11KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314
  1. # Copyright (C) 2011 OpenStack, LLC.
  2. # Copyright (c) 2013 Hewlett-Packard Development Company, L.P.
  3. # Copyright (c) 2013 OpenStack Foundation
  4. #
  5. # Licensed under the Apache License, Version 2.0 (the "License");
  6. # you may not use this file except in compliance with the License.
  7. # You may obtain a copy of the License at
  8. #
  9. # http://www.apache.org/licenses/LICENSE-2.0
  10. #
  11. # Unless required by applicable law or agreed to in writing, software
  12. # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  13. # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  14. # License for the specific language governing permissions and limitations
  15. # under the License.
  16. import collections
  17. from openstack_requirements import project
  18. from openstack_requirements import requirement
  19. from packaging import markers
  20. from packaging import specifiers
  21. class RequirementsList(object):
  22. def __init__(self, name, project):
  23. self.name = name
  24. self.reqs_by_file = {}
  25. self.project = project
  26. self.failed = False
  27. @property
  28. def reqs(self):
  29. return {k: v for d in self.reqs_by_file.values()
  30. for k, v in d.items()}
  31. def extract_reqs(self, content, strict):
  32. reqs = collections.defaultdict(set)
  33. parsed = requirement.parse(content)
  34. for name, entries in parsed.items():
  35. if not name:
  36. # Comments and other unprocessed lines
  37. continue
  38. list_reqs = [r for (r, line) in entries]
  39. # Strip the comments out before checking if there are duplicates
  40. list_reqs_stripped = [r._replace(comment='') for r in list_reqs]
  41. if strict and len(list_reqs_stripped) != len(set(
  42. list_reqs_stripped)):
  43. print("Requirements file has duplicate entries "
  44. "for package %s : %r." % (name, list_reqs))
  45. self.failed = True
  46. reqs[name].update(list_reqs)
  47. return reqs
  48. def process(self, strict=True):
  49. """Convert the project into ready to use data.
  50. - an iterable of requirement sets to check
  51. - each set has the following rules:
  52. - each has a list of Requirements objects
  53. - duplicates are not permitted within that list
  54. """
  55. print("Checking %(name)s" % {'name': self.name})
  56. # First, parse.
  57. for fname, content in self.project.get('requirements', {}).items():
  58. print("Processing %(fname)s" % {'fname': fname})
  59. if strict and not content.endswith('\n'):
  60. print("Requirements file %s does not "
  61. "end with a newline." % fname)
  62. self.reqs_by_file[fname] = self.extract_reqs(content, strict)
  63. for name, content in project.extras(self.project).items():
  64. print("Processing .[%(extra)s]" % {'extra': name})
  65. self.reqs_by_file[name] = self.extract_reqs(content, strict)
  66. def _get_exclusions(req):
  67. return set(
  68. spec
  69. for spec in req.specifiers.split(',')
  70. if '!=' in spec or '<' in spec
  71. )
  72. def _is_requirement_in_global_reqs(req, global_reqs):
  73. req_exclusions = _get_exclusions(req)
  74. for req2 in global_reqs:
  75. matching = True
  76. for aname in ['package', 'location', 'markers']:
  77. rval = getattr(req, aname)
  78. r2val = getattr(req2, aname)
  79. if rval != r2val:
  80. print('{} {!r}: {!r} does not match {!r}'.format(
  81. req, aname, rval, r2val))
  82. matching = False
  83. if not matching:
  84. continue
  85. # This matches the right package and other properties, so
  86. # ensure that any exclusions are a subset of the global
  87. # set.
  88. global_exclusions = _get_exclusions(req2)
  89. if req_exclusions.issubset(global_exclusions):
  90. return True
  91. else:
  92. print(
  93. "Requirement for package {} "
  94. "has an exclusion not found in the "
  95. "global list: {} vs. {}".format(
  96. req.package, req_exclusions, global_exclusions)
  97. )
  98. return False
  99. return False
  100. def get_global_reqs(content):
  101. """Return global_reqs structure.
  102. Parse content and return dict mapping names to sets of Requirement
  103. objects."
  104. """
  105. global_reqs = {}
  106. parsed = requirement.parse(content)
  107. for k, entries in parsed.items():
  108. # Discard the lines: we don't need them.
  109. global_reqs[k] = set(r for (r, line) in entries)
  110. return global_reqs
  111. def _validate_one(name, reqs, blacklist, global_reqs):
  112. "Returns True if there is a failure."
  113. if name in blacklist:
  114. # Blacklisted items are not synced and are managed
  115. # by project teams as they see fit, so no further
  116. # testing is needed.
  117. return False
  118. if name not in global_reqs:
  119. print("Requirement %s not in openstack/requirements" %
  120. str(reqs))
  121. return True
  122. counts = {}
  123. for req in reqs:
  124. if req.extras:
  125. for extra in req.extras:
  126. counts[extra] = counts.get(extra, 0) + 1
  127. else:
  128. counts[''] = counts.get('', 0) + 1
  129. if not _is_requirement_in_global_reqs(
  130. req, global_reqs[name]):
  131. print("Requirement for package %s: %s does "
  132. "not match openstack/requirements value : %s" % (
  133. name, str(req), str(global_reqs[name])))
  134. return True
  135. for extra, count in counts.items():
  136. if count != len(global_reqs[name]):
  137. print("Package %s%s requirement does not match "
  138. "number of lines (%d) in "
  139. "openstack/requirements" % (
  140. name,
  141. ('[%s]' % extra) if extra else '',
  142. len(global_reqs[name])))
  143. return True
  144. return False
  145. def validate(head_reqs, blacklist, global_reqs):
  146. failed = False
  147. # iterate through the changing entries and see if they match the global
  148. # equivalents we want enforced
  149. for fname, freqs in head_reqs.reqs_by_file.items():
  150. print("Validating %(fname)s" % {'fname': fname})
  151. for name, reqs in freqs.items():
  152. failed = (
  153. _validate_one(
  154. name,
  155. reqs,
  156. blacklist,
  157. global_reqs,
  158. )
  159. or failed
  160. )
  161. return failed
  162. def _find_constraint(req, constraints):
  163. """Return the constraint matching the markers for req.
  164. Given a requirement, find the constraint with matching markers.
  165. If none match, find a constraint without any markers at all.
  166. Otherwise return None.
  167. """
  168. if req.markers:
  169. req_markers = markers.Marker(req.markers)
  170. for constraint_setting, _ in constraints:
  171. if constraint_setting.markers == req.markers:
  172. return constraint_setting
  173. if not constraint_setting.markers:
  174. # There is no point in performing the complex
  175. # comparison for a constraint that has no markers, so
  176. # we skip it here. If we find no closer match then the
  177. # loop at the end of the function will look for a
  178. # constraint without a marker and use that.
  179. continue
  180. # NOTE(dhellmann): This is a very naive attempt to check
  181. # marker compatibility that relies on internal
  182. # implementation details of the packaging library. The
  183. # best way to ensure the constraint and requirements match
  184. # is to use the same marker string in the corresponding
  185. # lines.
  186. c_markers = markers.Marker(constraint_setting.markers)
  187. env = {
  188. str(var): str(val)
  189. for var, op, val in c_markers._markers # WARNING: internals
  190. }
  191. if req_markers.evaluate(env):
  192. return constraint_setting
  193. # Try looking for a constraint without any markers.
  194. for constraint_setting, _ in constraints:
  195. if not constraint_setting.markers:
  196. return constraint_setting
  197. return None
  198. def validate_lower_constraints(req_list, constraints, blacklist):
  199. """Return True if there is an error.
  200. :param reqs: RequirementsList for the head of the branch
  201. :param constraints: Parsed lower-constraints.txt or None
  202. """
  203. if constraints is None:
  204. return False
  205. parsed_constraints = requirement.parse(constraints)
  206. failed = False
  207. for fname, freqs in req_list.reqs_by_file.items():
  208. if fname == 'doc/requirements.txt':
  209. # Skip things that are not needed for unit or functional
  210. # tests.
  211. continue
  212. print("Validating lower constraints of {}".format(fname))
  213. for name, reqs in freqs.items():
  214. if name in blacklist:
  215. continue
  216. if name not in parsed_constraints:
  217. print('Package {!r} is used in {} '
  218. 'but not in lower-constraints.txt'.format(
  219. name, fname))
  220. failed = True
  221. continue
  222. for req in reqs:
  223. spec = specifiers.SpecifierSet(req.specifiers)
  224. # FIXME(dhellmann): This will only find constraints
  225. # where the markers match the requirements list
  226. # exactly, so we can't do things like use different
  227. # constrained versions for different versions of
  228. # python 3 if the requirement range is expressed as
  229. # python_version>3.0. We can support different
  230. # versions if there is a different requirement
  231. # specification for each version of python. I don't
  232. # really know how smart we want this to be, because
  233. # I'm not sure we want to support extremely
  234. # complicated dependency sets.
  235. constraint_setting = _find_constraint(
  236. req,
  237. parsed_constraints[name],
  238. )
  239. if not constraint_setting:
  240. print('Unable to find constraint for {} '
  241. 'matching {!r} or without any markers.'.format(
  242. name, req.markers))
  243. failed = True
  244. continue
  245. version = constraint_setting.specifiers.lstrip('=')
  246. if not spec.contains(version):
  247. print('Package {!r} is constrained to {} '
  248. 'which is incompatible with the settings {} '
  249. 'from {}.'.format(
  250. name, version, req, fname))
  251. failed = True
  252. min = [
  253. s
  254. for s in req.specifiers.split(',')
  255. if '>' in s
  256. ]
  257. if not min:
  258. # No minimum specified. Ignore this and let some
  259. # other validation trap the error.
  260. continue
  261. expected = min[0].lstrip('>=')
  262. if version != expected:
  263. print('Package {!r} is constrained to {} '
  264. 'which does not match '
  265. 'the minimum version specifier {} in {}'.format(
  266. name, version, expected, fname))
  267. failed = True
  268. return failed