Manage a pool of nodes for a distributed test infrastructure
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

config.py 9.4KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283
  1. #!/usr/bin/env python
  2. # Copyright (C) 2011-2013 OpenStack Foundation
  3. #
  4. # Licensed under the Apache License, Version 2.0 (the "License");
  5. # you may not use this file except in compliance with the License.
  6. # You may obtain a copy of the License at
  7. #
  8. # http://www.apache.org/licenses/LICENSE-2.0
  9. #
  10. # Unless required by applicable law or agreed to in writing, software
  11. # distributed under the License is distributed on an "AS IS" BASIS,
  12. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  13. # implied.
  14. #
  15. # See the License for the specific language governing permissions and
  16. # limitations under the License.
  17. import math
  18. import time
  19. import yaml
  20. from nodepool import zk
  21. from nodepool.driver import ConfigValue
  22. from nodepool.driver import Drivers
  23. class Config(ConfigValue):
  24. '''
  25. Class representing the nodepool configuration.
  26. This class implements methods to read each of the top-level configuration
  27. items found in the YAML config file, and set attributes accordingly.
  28. '''
  29. def __init__(self):
  30. self.diskimages = {}
  31. self.labels = {}
  32. self.providers = {}
  33. self.provider_managers = {}
  34. self.zookeeper_servers = {}
  35. self.elementsdir = None
  36. self.imagesdir = None
  37. self.build_log_dir = None
  38. self.build_log_retention = None
  39. self.max_hold_age = None
  40. self.webapp = None
  41. def __eq__(self, other):
  42. if isinstance(other, Config):
  43. return (self.diskimages == other.diskimages and
  44. self.labels == other.labels and
  45. self.providers == other.providers and
  46. self.provider_managers == other.provider_managers and
  47. self.zookeeper_servers == other.zookeeper_servers and
  48. self.elementsdir == other.elementsdir and
  49. self.imagesdir == other.imagesdir and
  50. self.build_log_dir == other.build_log_dir and
  51. self.build_log_retention == other.build_log_retention and
  52. self.max_hold_age == other.max_hold_age and
  53. self.webapp == other.webapp)
  54. return False
  55. def setElementsDir(self, value):
  56. self.elementsdir = value
  57. def setImagesDir(self, value):
  58. self.imagesdir = value
  59. def setBuildLog(self, directory, retention):
  60. if retention is None:
  61. retention = 7
  62. self.build_log_dir = directory
  63. self.build_log_retention = retention
  64. def setMaxHoldAge(self, value):
  65. if value is None or value <= 0:
  66. value = math.inf
  67. self.max_hold_age = value
  68. def setWebApp(self, webapp_cfg):
  69. if webapp_cfg is None:
  70. webapp_cfg = {}
  71. self.webapp = {
  72. 'port': webapp_cfg.get('port', 8005),
  73. 'listen_address': webapp_cfg.get('listen_address', '0.0.0.0')
  74. }
  75. def setZooKeeperServers(self, zk_cfg):
  76. if not zk_cfg:
  77. return
  78. for server in zk_cfg:
  79. z = zk.ZooKeeperConnectionConfig(server['host'],
  80. server.get('port', 2181),
  81. server.get('chroot', None))
  82. name = z.host + '_' + str(z.port)
  83. self.zookeeper_servers[name] = z
  84. def setDiskImages(self, diskimages_cfg):
  85. if not diskimages_cfg:
  86. return
  87. for diskimage in diskimages_cfg:
  88. d = DiskImage()
  89. d.name = diskimage['name']
  90. if 'elements' in diskimage:
  91. d.elements = u' '.join(diskimage['elements'])
  92. else:
  93. d.elements = ''
  94. # must be a string, as it's passed as env-var to
  95. # d-i-b, but might be untyped in the yaml and
  96. # interpreted as a number (e.g. "21" for fedora)
  97. d.release = str(diskimage.get('release', ''))
  98. d.rebuild_age = int(diskimage.get('rebuild-age', 86400))
  99. d.env_vars = diskimage.get('env-vars', {})
  100. if not isinstance(d.env_vars, dict):
  101. d.env_vars = {}
  102. d.image_types = set(diskimage.get('formats', []))
  103. d.pause = bool(diskimage.get('pause', False))
  104. d.username = diskimage.get('username', 'zuul')
  105. d.build_timeout = diskimage.get('build-timeout', (8 * 60 * 60))
  106. self.diskimages[d.name] = d
  107. def setSecureDiskimageEnv(self, diskimages, secure_config_path):
  108. for diskimage in diskimages:
  109. if diskimage['name'] not in self.diskimages:
  110. raise Exception('%s: unknown diskimage %s' %
  111. (secure_config_path, diskimage['name']))
  112. self.diskimages[diskimage['name']].env_vars.update(
  113. diskimage['env-vars'])
  114. def setLabels(self, labels_cfg):
  115. if not labels_cfg:
  116. return
  117. for label in labels_cfg:
  118. l = Label()
  119. l.name = label['name']
  120. l.max_ready_age = label.get('max-ready-age', 0)
  121. l.min_ready = label.get('min-ready', 0)
  122. l.pools = []
  123. self.labels[l.name] = l
  124. def setProviders(self, providers_cfg):
  125. if not providers_cfg:
  126. return
  127. for provider in providers_cfg:
  128. p = get_provider_config(provider)
  129. p.load(self)
  130. self.providers[p.name] = p
  131. class Label(ConfigValue):
  132. def __init__(self):
  133. self.name = None
  134. self.max_ready_age = None
  135. self.min_ready = None
  136. self.pools = None
  137. def __eq__(self, other):
  138. if isinstance(other, Label):
  139. return (self.name == other.name and
  140. self.max_ready_age == other.max_ready_age and
  141. self.min_ready == other.min_ready and
  142. self.pools == other.pools)
  143. return False
  144. def __repr__(self):
  145. return "<Label %s>" % self.name
  146. class DiskImage(ConfigValue):
  147. def __init__(self):
  148. self.name = None
  149. self.elements = None
  150. self.release = None
  151. self.rebuild_age = None
  152. self.env_vars = None
  153. self.image_types = None
  154. self.pause = False
  155. self.username = None
  156. self.build_timeout = None
  157. def __eq__(self, other):
  158. if isinstance(other, DiskImage):
  159. return (other.name == self.name and
  160. other.elements == self.elements and
  161. other.release == self.release and
  162. other.rebuild_age == self.rebuild_age and
  163. other.env_vars == self.env_vars and
  164. other.image_types == self.image_types and
  165. other.pause == self.pause and
  166. other.username == self.username and
  167. other.build_timeout == self.build_timeout)
  168. return False
  169. def __repr__(self):
  170. return "<DiskImage %s>" % self.name
  171. def as_list(item):
  172. if not item:
  173. return []
  174. if isinstance(item, list):
  175. return item
  176. return [item]
  177. def get_provider_config(provider):
  178. provider.setdefault('driver', 'openstack')
  179. # Ensure legacy configuration still works when using fake cloud
  180. if provider.get('name', '').startswith('fake'):
  181. provider['driver'] = 'fake'
  182. driver = Drivers.get(provider['driver'])
  183. return driver.getProviderConfig(provider)
  184. def openConfig(path):
  185. retry = 3
  186. # Since some nodepool code attempts to dynamically re-read its config
  187. # file, we need to handle the race that happens if an outside entity
  188. # edits it (causing it to temporarily not exist) at the same time we
  189. # attempt to reload it.
  190. while True:
  191. try:
  192. config = yaml.load(open(path))
  193. break
  194. except IOError as e:
  195. if e.errno == 2:
  196. retry = retry - 1
  197. time.sleep(.5)
  198. else:
  199. raise e
  200. if retry == 0:
  201. raise e
  202. return config
  203. def loadConfig(config_path):
  204. config = openConfig(config_path)
  205. # Call driver config reset now to clean global hooks like openstacksdk
  206. for driver in Drivers.drivers.values():
  207. driver.reset()
  208. newconfig = Config()
  209. newconfig.setElementsDir(config.get('elements-dir'))
  210. newconfig.setImagesDir(config.get('images-dir'))
  211. newconfig.setBuildLog(config.get('build-log-dir'),
  212. config.get('build-log-retention'))
  213. newconfig.setMaxHoldAge(config.get('max-hold-age'))
  214. newconfig.setWebApp(config.get('webapp'))
  215. newconfig.setZooKeeperServers(config.get('zookeeper-servers'))
  216. newconfig.setDiskImages(config.get('diskimages'))
  217. newconfig.setLabels(config.get('labels'))
  218. newconfig.setProviders(config.get('providers'))
  219. # Ensure at least qcow2 is set to be passed to qemu-img.
  220. # Note that this needs to be after setting the providers as they
  221. # add their supported image types to the diskimages.
  222. for diskimage in newconfig.diskimages.values():
  223. if not diskimage.image_types:
  224. diskimage.image_types.add('qcow2')
  225. return newconfig
  226. def loadSecureConfig(config, secure_config_path):
  227. secure = openConfig(secure_config_path)
  228. if not secure: # empty file
  229. return
  230. # Eliminate any servers defined in the normal config
  231. if secure.get('zookeeper-servers', []):
  232. config.zookeeper_servers = {}
  233. # TODO(Shrews): Support ZooKeeper auth
  234. config.setZooKeeperServers(secure.get('zookeeper-servers'))
  235. config.setSecureDiskimageEnv(
  236. secure.get('diskimages', []), secure_config_path)