Image building tools for OpenStack
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

element_dependencies.py 12KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353
  1. # Copyright 2013 Hewlett-Packard Development Company, L.P.
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License"); you may
  4. # not use this file except in compliance with the License. You may obtain
  5. # a copy of the License at
  6. #
  7. # http://www.apache.org/licenses/LICENSE-2.0
  8. #
  9. # Unless required by applicable law or agreed to in writing, software
  10. # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  11. # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  12. # License for the specific language governing permissions and limitations
  13. # under the License.
  14. from __future__ import print_function
  15. import argparse
  16. import collections
  17. import errno
  18. import logging
  19. import os
  20. import sys
  21. import yaml
  22. import diskimage_builder.logging_config
  23. logger = logging.getLogger(__name__)
  24. class MissingElementException(Exception):
  25. pass
  26. class AlreadyProvidedException(Exception):
  27. pass
  28. class MissingOSException(Exception):
  29. pass
  30. class InvalidElementDir(Exception):
  31. pass
  32. class Element(object):
  33. """An element"""
  34. def _get_element_set(self, path):
  35. """Get element set from element-[deps|provides] file
  36. Arguments:
  37. :param path: path to element description
  38. :return: the set of elements in the file, or a blank set if
  39. the file is not found.
  40. """
  41. try:
  42. with open(path) as f:
  43. lines = (line.strip() for line in f)
  44. # Strip blanks, but do we want to strip comment lines
  45. # too? No use case at the moment, and comments might
  46. # break other things that poke at the element-* files.
  47. lines = (line for line in lines if line)
  48. return set(lines)
  49. except IOError as e:
  50. if e.errno == errno.ENOENT:
  51. return set([])
  52. else:
  53. raise
  54. def _make_rdeps(self, all_elements):
  55. """Make a list of reverse dependencies (who depends on us).
  56. Only valid after _find_all_elements()
  57. Arguments:
  58. :param all_elements: dict as returned by _find_all_elements()
  59. :return: nothing, but elements will have r_depends var
  60. """
  61. # note; deliberatly left out of __init__ so that accidental
  62. # access without init raises error
  63. self.r_depends = []
  64. for name, element in all_elements.items():
  65. if self.name in element.depends:
  66. self.r_depends.append(element.name)
  67. def __init__(self, name, path):
  68. """A new element
  69. :param name: The element name
  70. :param path: Full path to element. element-deps and
  71. element-provides files will be parsed
  72. """
  73. self.name = name
  74. self.path = path
  75. # read the provides & depends files for this element into a
  76. # set; if the element has them.
  77. self.provides = self._get_element_set(
  78. os.path.join(path, 'element-provides'))
  79. self.depends = self._get_element_set(
  80. os.path.join(path, 'element-deps'))
  81. logger.debug("New element : %s", str(self))
  82. def __eq__(self, other):
  83. return self.name == other.name
  84. def __repr__(self):
  85. return self.name
  86. def __str__(self):
  87. return '%s p:<%s> d:<%s>' % (self.name,
  88. ','.join(self.provides),
  89. ','.join(self.depends))
  90. def _get_elements_dir():
  91. if not os.environ.get('ELEMENTS_PATH'):
  92. raise Exception("$ELEMENTS_PATH must be set.")
  93. return os.environ['ELEMENTS_PATH']
  94. def _expand_element_dependencies(user_elements, all_elements):
  95. """Expand user requested elements using element-deps files.
  96. Arguments:
  97. :param user_elements: iterable enumerating the elements a user requested
  98. :param all_elements: Element object dictionary from find_all_elements
  99. :return: a set containing the names of user_elements and all
  100. dependent elements including any transitive dependencies.
  101. """
  102. final_elements = set(user_elements)
  103. check_queue = collections.deque(user_elements)
  104. provided = set()
  105. provided_by = collections.defaultdict(list)
  106. while check_queue:
  107. # bug #1303911 - run through the provided elements first to avoid
  108. # adding unwanted dependencies and looking for virtual elements
  109. element = check_queue.popleft()
  110. if element in provided:
  111. continue
  112. elif element not in all_elements:
  113. raise MissingElementException("Element '%s' not found" % element)
  114. element_obj = all_elements[element]
  115. element_deps = element_obj.depends
  116. element_provides = element_obj.provides
  117. # Check that we are not providing an element which has already
  118. # been provided by someone else, and additionally save which
  119. # elements provide another element
  120. for provide in element_provides:
  121. if provide in provided:
  122. raise AlreadyProvidedException(
  123. "%s: already provided by %s" %
  124. (provide, provided_by[provide]))
  125. provided_by[provide].append(element)
  126. provided.update(element_provides)
  127. check_queue.extend(element_deps - (final_elements | provided))
  128. final_elements.update(element_deps)
  129. conflicts = set(user_elements) & provided
  130. if conflicts:
  131. logger.error(
  132. "The following elements are already provided by another element")
  133. for element in conflicts:
  134. logger.error("%s : already provided by %s",
  135. element, provided_by[element])
  136. raise AlreadyProvidedException()
  137. if "operating-system" not in provided:
  138. raise MissingOSException("Please include an operating system element")
  139. out = final_elements - provided
  140. return([all_elements[element] for element in out])
  141. def _find_all_elements(paths=None):
  142. """Build a dictionary Element() objects
  143. Walk ELEMENTS_PATH and find all elements. Make an Element object
  144. for each element we wish to consider. Note we process overrides
  145. such that elements specified earlier in the ELEMENTS_PATH override
  146. those seen later.
  147. :param paths: A list of paths to find elements in. If None will
  148. use ELEMENTS_PATH from environment
  149. :return: a dictionary of all elements
  150. """
  151. all_elements = {}
  152. # note we process the later entries *first*, so that earlier
  153. # entries will override later ones. i.e. with
  154. # ELEMENTS_PATH=path1:path2:path3
  155. # we want the elements in "path1" to override "path3"
  156. if not paths:
  157. paths = list(reversed(_get_elements_dir().split(':')))
  158. else:
  159. paths = list(reversed(paths.split(':')))
  160. logger.debug("ELEMENTS_PATH is: %s", ":".join(paths))
  161. for path in paths:
  162. if not os.path.isdir(path):
  163. raise InvalidElementDir("ELEMENTS_PATH entry '%s' "
  164. "is not a directory " % path)
  165. # In words : make a list of directories in "path". Since an
  166. # element is a directory, this is our list of elements.
  167. elements = [os.path.realpath(os.path.join(path, f))
  168. for f in os.listdir(path)
  169. if os.path.isdir(os.path.join(path, f))]
  170. for element in elements:
  171. # the element name is the last part of the full path in
  172. # element (these are all directories, we know that from
  173. # above)
  174. name = os.path.basename(element)
  175. new_element = Element(name, element)
  176. if name in all_elements:
  177. logger.warning("Element <%s> overrides <%s>",
  178. new_element.path, all_elements[name].path)
  179. all_elements[name] = new_element
  180. # Now we have all the elements, make a call on each element to
  181. # store it's reverse dependencies
  182. for name, element in all_elements.items():
  183. element._make_rdeps(all_elements)
  184. return all_elements
  185. def _get_elements(elements, paths=None):
  186. """Return the canonical list of Element objects
  187. This function returns Element objects. For exernal calls, use
  188. get_elements which returns a simple tuple & list.
  189. :param elements: user specified list of elements
  190. :param paths: element paths, default to environment
  191. """
  192. all_elements = _find_all_elements(paths)
  193. return _expand_element_dependencies(elements, all_elements)
  194. def get_elements(elements, paths=None):
  195. """Return the canonical list of elements with their dependencies
  196. .. note::
  197. You probably do not want to use this! Elements that require
  198. access to the list of all other elements should generally use
  199. the environment variables exported by disk-image-create below.
  200. :param elements: user specified elements
  201. :param paths: Alternative ELEMENTS_PATH; default is to use from env
  202. :return: A de-duplicated list of tuples [(element, path),
  203. (element, path) ...] with all elements and their
  204. dependents, including any transitive dependencies.
  205. """
  206. elements = _get_elements(elements, paths)
  207. return [(element.name, element.path) for element in elements]
  208. def expand_dependencies(user_elements, element_dirs):
  209. """Deprecated method for expanding element dependencies.
  210. .. warning::
  211. DO NOT USE THIS FUNCTION. For compatibility reasons, this
  212. function does not provide paths to the returned elements. This
  213. means the caller must process override rules if two elements
  214. with the same name appear in element_dirs
  215. :param user_elements: iterable enumerating the elements a user requested
  216. :param element_dirs: The ELEMENTS_PATH to process
  217. :return: a set containing user_elements and all dependent
  218. elements including any transitive dependencies.
  219. """
  220. logger.warning("expand_dependencies() deprecated, use get_elements")
  221. elements = _get_elements(user_elements, element_dirs)
  222. return set([element.name for element in elements])
  223. def _output_env_vars(elements):
  224. """Output eval-able bash strings for IMAGE_ELEMENT vars
  225. :param elements: list of Element objects to represent
  226. """
  227. # first the "legacy" environment variable that just lists the
  228. # elements
  229. print("export IMAGE_ELEMENT='%s'" %
  230. ' '.join([element.name for element in elements]))
  231. # Then YAML
  232. output = {}
  233. for element in elements:
  234. output[element.name] = element.path
  235. print("export IMAGE_ELEMENT_YAML='%s'" % yaml.safe_dump(output))
  236. # Then bash array. Unfortunately, bash can't export array
  237. # variables. So we take a compromise and produce an exported
  238. # function that outputs the string to re-create the array.
  239. # You can then simply do
  240. # eval declare -A element_array=$(get_image_element_array)
  241. # and you have it.
  242. output = ""
  243. for element in elements:
  244. output += '[%s]=%s ' % (element.name, element.path)
  245. print("function get_image_element_array {\n"
  246. " echo \"%s\"\n"
  247. "};\n"
  248. "export -f get_image_element_array;" % output)
  249. def main():
  250. diskimage_builder.logging_config.setup()
  251. parser = argparse.ArgumentParser()
  252. parser.add_argument('elements', nargs='+',
  253. help='display dependencies of the given elements')
  254. parser.add_argument('--env', '-e', action='store_true',
  255. default=False,
  256. help=('Output eval-able bash strings for '
  257. 'IMAGE_ELEMENT variables'))
  258. args = parser.parse_args(sys.argv[1:])
  259. elements = _get_elements(args.elements)
  260. if args.env:
  261. _output_env_vars(elements)
  262. else:
  263. # deprecated compatibility output; doesn't include paths.
  264. print(' '.join([element.name for element in elements]))
  265. return 0
  266. if __name__ == "__main__":
  267. main()