Global requirements for OpenStack
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

check.py 12KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319
  1. # Copyright (C) 2011 OpenStack, LLC.
  2. # Copyright (c) 2013 Hewlett-Packard Development Company, L.P.
  3. # Copyright (c) 2013 OpenStack Foundation
  4. #
  5. # Licensed under the Apache License, Version 2.0 (the "License");
  6. # you may not use this file except in compliance with the License.
  7. # You may obtain a copy of the License at
  8. #
  9. # http://www.apache.org/licenses/LICENSE-2.0
  10. #
  11. # Unless required by applicable law or agreed to in writing, software
  12. # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  13. # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  14. # License for the specific language governing permissions and limitations
  15. # under the License.
  16. import collections
  17. from openstack_requirements import project
  18. from openstack_requirements import requirement
  19. from packaging import markers
  20. from packaging import specifiers
  21. class RequirementsList(object):
  22. def __init__(self, name, project):
  23. self.name = name
  24. self.reqs_by_file = {}
  25. self.project = project
  26. self.failed = False
  27. @property
  28. def reqs(self):
  29. return {k: v for d in self.reqs_by_file.values()
  30. for k, v in d.items()}
  31. def extract_reqs(self, content, strict):
  32. reqs = collections.defaultdict(set)
  33. parsed = requirement.parse(content)
  34. for name, entries in parsed.items():
  35. if not name:
  36. # Comments and other unprocessed lines
  37. continue
  38. list_reqs = [r for (r, line) in entries]
  39. # Strip the comments out before checking if there are duplicates
  40. list_reqs_stripped = [r._replace(comment='') for r in list_reqs]
  41. if strict and len(list_reqs_stripped) != len(set(
  42. list_reqs_stripped)):
  43. print("Requirements file has duplicate entries "
  44. "for package %s : %r." % (name, list_reqs))
  45. self.failed = True
  46. reqs[name].update(list_reqs)
  47. return reqs
  48. def process(self, strict=True):
  49. """Convert the project into ready to use data.
  50. - an iterable of requirement sets to check
  51. - each set has the following rules:
  52. - each has a list of Requirements objects
  53. - duplicates are not permitted within that list
  54. """
  55. print("Checking %(name)s" % {'name': self.name})
  56. # First, parse.
  57. for fname, content in self.project.get('requirements', {}).items():
  58. print("Processing %(fname)s" % {'fname': fname})
  59. if strict and not content.endswith('\n'):
  60. print("Requirements file %s does not "
  61. "end with a newline." % fname)
  62. self.reqs_by_file[fname] = self.extract_reqs(content, strict)
  63. for name, content in project.extras(self.project).items():
  64. print("Processing .[%(extra)s]" % {'extra': name})
  65. self.reqs_by_file[name] = self.extract_reqs(content, strict)
  66. def _get_exclusions(req):
  67. return set(
  68. spec
  69. for spec in req.specifiers.split(',')
  70. if '!=' in spec or '<' in spec
  71. )
  72. def _is_requirement_in_global_reqs(req, global_reqs):
  73. req_exclusions = _get_exclusions(req)
  74. for req2 in global_reqs:
  75. matching = True
  76. for aname in ['package', 'location', 'markers']:
  77. rval = getattr(req, aname)
  78. r2val = getattr(req2, aname)
  79. if rval != r2val:
  80. print('{} {!r}: {!r} does not match {!r}'.format(
  81. req, aname, rval, r2val))
  82. matching = False
  83. if not matching:
  84. continue
  85. # This matches the right package and other properties, so
  86. # ensure that any exclusions are a subset of the global
  87. # set.
  88. global_exclusions = _get_exclusions(req2)
  89. if req_exclusions.issubset(global_exclusions):
  90. return True
  91. else:
  92. print(
  93. "Requirement for package {} "
  94. "has an exclusion not found in the "
  95. "global list: {} vs. {}".format(
  96. req.package, req_exclusions, global_exclusions)
  97. )
  98. return False
  99. return False
  100. def get_global_reqs(content):
  101. """Return global_reqs structure.
  102. Parse content and return dict mapping names to sets of Requirement
  103. objects."
  104. """
  105. global_reqs = {}
  106. parsed = requirement.parse(content)
  107. for k, entries in parsed.items():
  108. # Discard the lines: we don't need them.
  109. global_reqs[k] = set(r for (r, line) in entries)
  110. return global_reqs
  111. def _validate_one(name, reqs, blacklist, global_reqs):
  112. "Returns True if there is a failure."
  113. if name in blacklist:
  114. # Blacklisted items are not synced and are managed
  115. # by project teams as they see fit, so no further
  116. # testing is needed.
  117. return False
  118. if name not in global_reqs:
  119. print("Requirement %s not in openstack/requirements" %
  120. str(reqs))
  121. return True
  122. counts = {}
  123. for req in reqs:
  124. if req.extras:
  125. for extra in req.extras:
  126. counts[extra] = counts.get(extra, 0) + 1
  127. else:
  128. counts[''] = counts.get('', 0) + 1
  129. if not _is_requirement_in_global_reqs(
  130. req, global_reqs[name]):
  131. print("Requirement for package %s: %s does "
  132. "not match openstack/requirements value : %s" % (
  133. name, str(req), str(global_reqs[name])))
  134. return True
  135. # check for minimum being defined
  136. min = [s for s in req.specifiers.split(',') if '>' in s]
  137. if not min:
  138. print("Requirement for package %s has no lower bound" % name)
  139. return True
  140. for extra, count in counts.items():
  141. if count != len(global_reqs[name]):
  142. print("Package %s%s requirement does not match "
  143. "number of lines (%d) in "
  144. "openstack/requirements" % (
  145. name,
  146. ('[%s]' % extra) if extra else '',
  147. len(global_reqs[name])))
  148. return True
  149. return False
  150. def validate(head_reqs, blacklist, global_reqs):
  151. failed = False
  152. # iterate through the changing entries and see if they match the global
  153. # equivalents we want enforced
  154. for fname, freqs in head_reqs.reqs_by_file.items():
  155. print("Validating %(fname)s" % {'fname': fname})
  156. for name, reqs in freqs.items():
  157. failed = (
  158. _validate_one(
  159. name,
  160. reqs,
  161. blacklist,
  162. global_reqs,
  163. )
  164. or failed
  165. )
  166. return failed
  167. def _find_constraint(req, constraints):
  168. """Return the constraint matching the markers for req.
  169. Given a requirement, find the constraint with matching markers.
  170. If none match, find a constraint without any markers at all.
  171. Otherwise return None.
  172. """
  173. if req.markers:
  174. req_markers = markers.Marker(req.markers)
  175. for constraint_setting, _ in constraints:
  176. if constraint_setting.markers == req.markers:
  177. return constraint_setting
  178. if not constraint_setting.markers:
  179. # There is no point in performing the complex
  180. # comparison for a constraint that has no markers, so
  181. # we skip it here. If we find no closer match then the
  182. # loop at the end of the function will look for a
  183. # constraint without a marker and use that.
  184. continue
  185. # NOTE(dhellmann): This is a very naive attempt to check
  186. # marker compatibility that relies on internal
  187. # implementation details of the packaging library. The
  188. # best way to ensure the constraint and requirements match
  189. # is to use the same marker string in the corresponding
  190. # lines.
  191. c_markers = markers.Marker(constraint_setting.markers)
  192. env = {
  193. str(var): str(val)
  194. for var, op, val in c_markers._markers # WARNING: internals
  195. }
  196. if req_markers.evaluate(env):
  197. return constraint_setting
  198. # Try looking for a constraint without any markers.
  199. for constraint_setting, _ in constraints:
  200. if not constraint_setting.markers:
  201. return constraint_setting
  202. return None
  203. def validate_lower_constraints(req_list, constraints, blacklist):
  204. """Return True if there is an error.
  205. :param reqs: RequirementsList for the head of the branch
  206. :param constraints: Parsed lower-constraints.txt or None
  207. """
  208. if constraints is None:
  209. return False
  210. parsed_constraints = requirement.parse(constraints)
  211. failed = False
  212. for fname, freqs in req_list.reqs_by_file.items():
  213. if fname == 'doc/requirements.txt':
  214. # Skip things that are not needed for unit or functional
  215. # tests.
  216. continue
  217. print("Validating lower constraints of {}".format(fname))
  218. for name, reqs in freqs.items():
  219. if name in blacklist:
  220. continue
  221. if name not in parsed_constraints:
  222. print('Package {!r} is used in {} '
  223. 'but not in lower-constraints.txt'.format(
  224. name, fname))
  225. failed = True
  226. continue
  227. for req in reqs:
  228. spec = specifiers.SpecifierSet(req.specifiers)
  229. # FIXME(dhellmann): This will only find constraints
  230. # where the markers match the requirements list
  231. # exactly, so we can't do things like use different
  232. # constrained versions for different versions of
  233. # python 3 if the requirement range is expressed as
  234. # python_version>3.0. We can support different
  235. # versions if there is a different requirement
  236. # specification for each version of python. I don't
  237. # really know how smart we want this to be, because
  238. # I'm not sure we want to support extremely
  239. # complicated dependency sets.
  240. constraint_setting = _find_constraint(
  241. req,
  242. parsed_constraints[name],
  243. )
  244. if not constraint_setting:
  245. print('Unable to find constraint for {} '
  246. 'matching {!r} or without any markers.'.format(
  247. name, req.markers))
  248. failed = True
  249. continue
  250. version = constraint_setting.specifiers.lstrip('=')
  251. if not spec.contains(version):
  252. print('Package {!r} is constrained to {} '
  253. 'which is incompatible with the settings {} '
  254. 'from {}.'.format(
  255. name, version, req, fname))
  256. failed = True
  257. min = [
  258. s
  259. for s in req.specifiers.split(',')
  260. if '>' in s
  261. ]
  262. if not min:
  263. # No minimum specified. Ignore this and let some
  264. # other validation trap the error.
  265. continue
  266. expected = min[0].lstrip('>=')
  267. if version != expected:
  268. print('Package {!r} is constrained to {} '
  269. 'which does not match '
  270. 'the minimum version specifier {} in {}'.format(
  271. name, version, expected, fname))
  272. failed = True
  273. return failed