Manage a pool of nodes for a distributed test infrastructure
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

config.py 15KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411
  1. # Copyright (C) 2011-2013 OpenStack Foundation
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License");
  4. # you may not use this file except in compliance with the License.
  5. # You may obtain a copy of the License at
  6. #
  7. # http://www.apache.org/licenses/LICENSE-2.0
  8. #
  9. # Unless required by applicable law or agreed to in writing, software
  10. # distributed under the License is distributed on an "AS IS" BASIS,
  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  12. # implied.
  13. #
  14. # See the License for the specific language governing permissions and
  15. # limitations under the License.
  16. import math
  17. import voluptuous as v
  18. from nodepool.driver import ProviderConfig
  19. from nodepool.driver import ConfigValue
  20. from nodepool.driver import ConfigPool
  21. class ProviderDiskImage(ConfigValue):
  22. def __init__(self):
  23. self.name = None
  24. self.pause = False
  25. self.config_drive = None
  26. self.connection_type = None
  27. self.connection_port = None
  28. self.meta = None
  29. def __eq__(self, other):
  30. if isinstance(other, ProviderDiskImage):
  31. return (self.name == other.name and
  32. self.pause == other.pause and
  33. self.config_drive == other.config_drive and
  34. self.connection_type == other.connection_type and
  35. self.connection_port == other.connection_port and
  36. self.meta == other.meta)
  37. return False
  38. def __repr__(self):
  39. return "<ProviderDiskImage %s>" % self.name
  40. class ProviderCloudImage(ConfigValue):
  41. def __init__(self):
  42. self.name = None
  43. self.config_drive = None
  44. self.image_id = None
  45. self.image_name = None
  46. self.username = None
  47. self.connection_type = None
  48. self.connection_port = None
  49. def __eq__(self, other):
  50. if isinstance(other, ProviderCloudImage):
  51. return (self.name == other.name and
  52. self.config_drive == other.config_drive and
  53. self.image_id == other.image_id and
  54. self.image_name == other.image_name and
  55. self.username == other.username and
  56. self.connection_type == other.connection_type and
  57. self.connection_port == other.connection_port)
  58. return False
  59. def __repr__(self):
  60. return "<ProviderCloudImage %s>" % self.name
  61. @property
  62. def external_name(self):
  63. '''Human readable version of external.'''
  64. return self.image_id or self.image_name or self.name
  65. class ProviderLabel(ConfigValue):
  66. def __init__(self):
  67. self.name = None
  68. self.diskimage = None
  69. self.cloud_image = None
  70. self.min_ram = None
  71. self.flavor_name = None
  72. self.key_name = None
  73. self.console_log = False
  74. self.boot_from_volume = False
  75. self.volume_size = None
  76. self.instance_properties = None
  77. # The ProviderPool object that owns this label.
  78. self.pool = None
  79. def __eq__(self, other):
  80. if isinstance(other, ProviderLabel):
  81. # NOTE(Shrews): We intentionally do not compare 'pool' here
  82. # since this causes recursive checks with ProviderPool.
  83. return (other.diskimage == self.diskimage and
  84. other.cloud_image == self.cloud_image and
  85. other.min_ram == self.min_ram and
  86. other.flavor_name == self.flavor_name and
  87. other.key_name == self.key_name and
  88. other.name == self.name and
  89. other.console_log == self.console_log and
  90. other.boot_from_volume == self.boot_from_volume and
  91. other.volume_size == self.volume_size and
  92. other.instance_properties == self.instance_properties)
  93. return False
  94. def __repr__(self):
  95. return "<ProviderLabel %s>" % self.name
  96. class ProviderPool(ConfigPool):
  97. def __init__(self):
  98. self.name = None
  99. self.max_cores = None
  100. self.max_ram = None
  101. self.ignore_provider_quota = False
  102. self.azs = None
  103. self.networks = None
  104. self.security_groups = None
  105. self.auto_floating_ip = True
  106. self.host_key_checking = True
  107. self.labels = None
  108. # The OpenStackProviderConfig object that owns this pool.
  109. self.provider = None
  110. # Initialize base class attributes
  111. super().__init__()
  112. def __eq__(self, other):
  113. if isinstance(other, ProviderPool):
  114. # NOTE(Shrews): We intentionally do not compare 'provider' here
  115. # since this causes recursive checks with OpenStackProviderConfig.
  116. return (super().__eq__(other) and
  117. other.name == self.name and
  118. other.max_cores == self.max_cores and
  119. other.max_ram == self.max_ram and
  120. other.ignore_provider_quota == (
  121. self.ignore_provider_quota) and
  122. other.azs == self.azs and
  123. other.networks == self.networks and
  124. other.security_groups == self.security_groups and
  125. other.auto_floating_ip == self.auto_floating_ip and
  126. other.host_key_checking == self.host_key_checking and
  127. other.labels == self.labels)
  128. return False
  129. def __repr__(self):
  130. return "<ProviderPool %s>" % self.name
  131. def load(self, pool_config, full_config, provider):
  132. '''
  133. Load pool configuration options.
  134. :param dict pool_config: A single pool config section from which we
  135. will load the values.
  136. :param dict full_config: The full nodepool config.
  137. :param OpenStackProviderConfig: The calling provider object.
  138. '''
  139. super().load(pool_config)
  140. self.provider = provider
  141. self.name = pool_config['name']
  142. self.max_cores = pool_config.get('max-cores', math.inf)
  143. self.max_ram = pool_config.get('max-ram', math.inf)
  144. self.ignore_provider_quota = pool_config.get('ignore-provider-quota',
  145. False)
  146. self.azs = pool_config.get('availability-zones')
  147. self.networks = pool_config.get('networks', [])
  148. self.security_groups = pool_config.get('security-groups', [])
  149. self.auto_floating_ip = bool(pool_config.get('auto-floating-ip', True))
  150. self.host_key_checking = bool(pool_config.get('host-key-checking',
  151. True))
  152. for label in pool_config.get('labels', []):
  153. pl = ProviderLabel()
  154. pl.name = label['name']
  155. pl.pool = self
  156. self.labels[pl.name] = pl
  157. diskimage = label.get('diskimage', None)
  158. if diskimage:
  159. pl.diskimage = full_config.diskimages[diskimage]
  160. else:
  161. pl.diskimage = None
  162. cloud_image_name = label.get('cloud-image', None)
  163. if cloud_image_name:
  164. cloud_image = provider.cloud_images.get(cloud_image_name, None)
  165. if not cloud_image:
  166. raise ValueError(
  167. "cloud-image %s does not exist in provider %s"
  168. " but is referenced in label %s" %
  169. (cloud_image_name, self.name, pl.name))
  170. else:
  171. cloud_image = None
  172. pl.cloud_image = cloud_image
  173. pl.min_ram = label.get('min-ram', 0)
  174. pl.flavor_name = label.get('flavor-name', None)
  175. pl.key_name = label.get('key-name')
  176. pl.console_log = label.get('console-log', False)
  177. pl.boot_from_volume = bool(label.get('boot-from-volume',
  178. False))
  179. pl.volume_size = label.get('volume-size', 50)
  180. pl.instance_properties = label.get('instance-properties',
  181. None)
  182. top_label = full_config.labels[pl.name]
  183. top_label.pools.append(self)
  184. class OpenStackProviderConfig(ProviderConfig):
  185. def __init__(self, driver, provider):
  186. self.driver_object = driver
  187. self.__pools = {}
  188. self.cloud_config = None
  189. self.image_type = None
  190. self.rate = None
  191. self.boot_timeout = None
  192. self.launch_timeout = None
  193. self.clean_floating_ips = None
  194. self.diskimages = {}
  195. self.cloud_images = {}
  196. self.hostname_format = None
  197. self.image_name_format = None
  198. super().__init__(provider)
  199. def __eq__(self, other):
  200. if isinstance(other, OpenStackProviderConfig):
  201. return (super().__eq__(other) and
  202. other.cloud_config == self.cloud_config and
  203. other.pools == self.pools and
  204. other.image_type == self.image_type and
  205. other.rate == self.rate and
  206. other.boot_timeout == self.boot_timeout and
  207. other.launch_timeout == self.launch_timeout and
  208. other.clean_floating_ips == self.clean_floating_ips and
  209. other.diskimages == self.diskimages and
  210. other.cloud_images == self.cloud_images)
  211. return False
  212. def _cloudKwargs(self):
  213. cloud_kwargs = {}
  214. for arg in ['region-name', 'cloud']:
  215. if arg in self.provider:
  216. cloud_kwargs[arg] = self.provider[arg]
  217. return cloud_kwargs
  218. @property
  219. def pools(self):
  220. return self.__pools
  221. @property
  222. def manage_images(self):
  223. return True
  224. def load(self, config):
  225. cloud_kwargs = self._cloudKwargs()
  226. openstack_config = self.driver_object.openstack_config
  227. self.cloud_config = openstack_config.get_one(**cloud_kwargs)
  228. self.image_type = self.cloud_config.config['image_format']
  229. self.region_name = self.provider.get('region-name')
  230. self.rate = float(self.provider.get('rate', 1.0))
  231. self.boot_timeout = self.provider.get('boot-timeout', 60)
  232. self.launch_timeout = self.provider.get('launch-timeout', 3600)
  233. self.launch_retries = self.provider.get('launch-retries', 3)
  234. self.clean_floating_ips = self.provider.get('clean-floating-ips')
  235. self.hostname_format = self.provider.get(
  236. 'hostname-format',
  237. '{label.name}-{provider.name}-{node.id}'
  238. )
  239. self.image_name_format = self.provider.get(
  240. 'image-name-format',
  241. '{image_name}-{timestamp}'
  242. )
  243. default_port_mapping = {
  244. 'ssh': 22,
  245. 'winrm': 5986,
  246. }
  247. for image in self.provider.get('diskimages', []):
  248. i = ProviderDiskImage()
  249. i.name = image['name']
  250. self.diskimages[i.name] = i
  251. diskimage = config.diskimages[i.name]
  252. diskimage.image_types.add(self.image_type)
  253. i.pause = bool(image.get('pause', False))
  254. i.config_drive = image.get('config-drive', None)
  255. i.connection_type = image.get('connection-type', 'ssh')
  256. i.connection_port = image.get(
  257. 'connection-port',
  258. default_port_mapping.get(i.connection_type, 22))
  259. # This dict is expanded and used as custom properties when
  260. # the image is uploaded.
  261. i.meta = image.get('meta', {})
  262. # 5 elements, and no key or value can be > 255 chars
  263. # per Nova API rules
  264. if i.meta:
  265. if len(i.meta) > 5 or \
  266. any([len(k) > 255 or len(v) > 255
  267. for k, v in i.meta.items()]):
  268. # soft-fail
  269. # self.log.error("Invalid metadata for %s; ignored"
  270. # % i.name)
  271. i.meta = {}
  272. for image in self.provider.get('cloud-images', []):
  273. i = ProviderCloudImage()
  274. i.name = image['name']
  275. i.config_drive = image.get('config-drive', None)
  276. i.image_id = image.get('image-id', None)
  277. i.image_name = image.get('image-name', None)
  278. i.username = image.get('username', None)
  279. i.connection_type = image.get('connection-type', 'ssh')
  280. i.connection_port = image.get(
  281. 'connection-port',
  282. default_port_mapping.get(i.connection_type, 22))
  283. self.cloud_images[i.name] = i
  284. for pool in self.provider.get('pools', []):
  285. pp = ProviderPool()
  286. pp.load(pool, config, self)
  287. self.pools[pp.name] = pp
  288. def getSchema(self):
  289. provider_diskimage = {
  290. 'name': str,
  291. 'pause': bool,
  292. 'meta': dict,
  293. 'config-drive': bool,
  294. 'connection-type': str,
  295. 'connection-port': int,
  296. }
  297. provider_cloud_images = {
  298. 'name': str,
  299. 'config-drive': bool,
  300. 'connection-type': str,
  301. 'connection-port': int,
  302. v.Exclusive('image-id', 'cloud-image-name-or-id'): str,
  303. v.Exclusive('image-name', 'cloud-image-name-or-id'): str,
  304. 'username': str,
  305. }
  306. pool_label_main = {
  307. v.Required('name'): str,
  308. v.Exclusive('diskimage', 'label-image'): str,
  309. v.Exclusive('cloud-image', 'label-image'): str,
  310. 'min-ram': int,
  311. 'flavor-name': str,
  312. 'key-name': str,
  313. 'console-log': bool,
  314. 'boot-from-volume': bool,
  315. 'volume-size': int,
  316. 'instance-properties': dict,
  317. }
  318. label_min_ram = v.Schema({v.Required('min-ram'): int}, extra=True)
  319. label_flavor_name = v.Schema({v.Required('flavor-name'): str},
  320. extra=True)
  321. label_diskimage = v.Schema({v.Required('diskimage'): str}, extra=True)
  322. label_cloud_image = v.Schema({v.Required('cloud-image'): str},
  323. extra=True)
  324. pool_label = v.All(pool_label_main,
  325. v.Any(label_min_ram, label_flavor_name),
  326. v.Any(label_diskimage, label_cloud_image))
  327. pool = ConfigPool.getCommonSchemaDict()
  328. pool.update({
  329. 'name': str,
  330. 'networks': [str],
  331. 'auto-floating-ip': bool,
  332. 'host-key-checking': bool,
  333. 'ignore-provider-quota': bool,
  334. 'max-cores': int,
  335. 'max-ram': int,
  336. 'labels': [pool_label],
  337. 'availability-zones': [str],
  338. 'security-groups': [str]
  339. })
  340. schema = ProviderConfig.getCommonSchemaDict()
  341. schema.update({
  342. 'region-name': str,
  343. v.Required('cloud'): str,
  344. 'boot-timeout': int,
  345. 'launch-timeout': int,
  346. 'launch-retries': int,
  347. 'nodepool-id': str,
  348. 'rate': v.Coerce(float),
  349. 'hostname-format': str,
  350. 'image-name-format': str,
  351. 'clean-floating-ips': bool,
  352. 'pools': [pool],
  353. 'diskimages': [provider_diskimage],
  354. 'cloud-images': [provider_cloud_images],
  355. })
  356. return v.Schema(schema)
  357. def getSupportedLabels(self, pool_name=None):
  358. labels = set()
  359. for pool in self.pools.values():
  360. if not pool_name or (pool.name == pool_name):
  361. labels.update(pool.labels.keys())
  362. return labels