Manage a pool of nodes for a distributed test infrastructure
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

config.py 15KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415
  1. # Copyright (C) 2011-2013 OpenStack Foundation
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License");
  4. # you may not use this file except in compliance with the License.
  5. # You may obtain a copy of the License at
  6. #
  7. # http://www.apache.org/licenses/LICENSE-2.0
  8. #
  9. # Unless required by applicable law or agreed to in writing, software
  10. # distributed under the License is distributed on an "AS IS" BASIS,
  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  12. # implied.
  13. #
  14. # See the License for the specific language governing permissions and
  15. # limitations under the License.
  16. import math
  17. import voluptuous as v
  18. from nodepool.driver import ProviderConfig
  19. from nodepool.driver import ConfigValue
  20. from nodepool.driver import ConfigPool
  21. class ProviderDiskImage(ConfigValue):
  22. def __init__(self):
  23. self.name = None
  24. self.pause = False
  25. self.config_drive = None
  26. self.connection_type = None
  27. self.connection_port = None
  28. self.meta = None
  29. def __eq__(self, other):
  30. if isinstance(other, ProviderDiskImage):
  31. return (self.name == other.name and
  32. self.pause == other.pause and
  33. self.config_drive == other.config_drive and
  34. self.connection_type == other.connection_type and
  35. self.connection_port == other.connection_port and
  36. self.meta == other.meta)
  37. return False
  38. def __repr__(self):
  39. return "<ProviderDiskImage %s>" % self.name
  40. class ProviderCloudImage(ConfigValue):
  41. def __init__(self):
  42. self.name = None
  43. self.config_drive = None
  44. self.image_id = None
  45. self.image_name = None
  46. self.username = None
  47. self.connection_type = None
  48. self.connection_port = None
  49. def __eq__(self, other):
  50. if isinstance(other, ProviderCloudImage):
  51. return (self.name == other.name and
  52. self.config_drive == other.config_drive and
  53. self.image_id == other.image_id and
  54. self.image_name == other.image_name and
  55. self.username == other.username and
  56. self.connection_type == other.connection_type and
  57. self.connection_port == other.connection_port)
  58. return False
  59. def __repr__(self):
  60. return "<ProviderCloudImage %s>" % self.name
  61. @property
  62. def external_name(self):
  63. '''Human readable version of external.'''
  64. return self.image_id or self.image_name or self.name
  65. class ProviderLabel(ConfigValue):
  66. def __init__(self):
  67. self.name = None
  68. self.diskimage = None
  69. self.cloud_image = None
  70. self.min_ram = None
  71. self.flavor_name = None
  72. self.key_name = None
  73. self.console_log = False
  74. self.boot_from_volume = False
  75. self.volume_size = None
  76. self.instance_properties = None
  77. self.userdata = None
  78. # The ProviderPool object that owns this label.
  79. self.pool = None
  80. def __eq__(self, other):
  81. if isinstance(other, ProviderLabel):
  82. # NOTE(Shrews): We intentionally do not compare 'pool' here
  83. # since this causes recursive checks with ProviderPool.
  84. return (other.diskimage == self.diskimage and
  85. other.cloud_image == self.cloud_image and
  86. other.min_ram == self.min_ram and
  87. other.flavor_name == self.flavor_name and
  88. other.key_name == self.key_name and
  89. other.name == self.name and
  90. other.console_log == self.console_log and
  91. other.boot_from_volume == self.boot_from_volume and
  92. other.volume_size == self.volume_size and
  93. other.instance_properties == self.instance_properties and
  94. other.userdata == self.userdata)
  95. return False
  96. def __repr__(self):
  97. return "<ProviderLabel %s>" % self.name
  98. class ProviderPool(ConfigPool):
  99. def __init__(self):
  100. self.name = None
  101. self.max_cores = None
  102. self.max_ram = None
  103. self.ignore_provider_quota = False
  104. self.azs = None
  105. self.networks = None
  106. self.security_groups = None
  107. self.auto_floating_ip = True
  108. self.host_key_checking = True
  109. self.labels = None
  110. # The OpenStackProviderConfig object that owns this pool.
  111. self.provider = None
  112. # Initialize base class attributes
  113. super().__init__()
  114. def __eq__(self, other):
  115. if isinstance(other, ProviderPool):
  116. # NOTE(Shrews): We intentionally do not compare 'provider' here
  117. # since this causes recursive checks with OpenStackProviderConfig.
  118. return (super().__eq__(other) and
  119. other.name == self.name and
  120. other.max_cores == self.max_cores and
  121. other.max_ram == self.max_ram and
  122. other.ignore_provider_quota == (
  123. self.ignore_provider_quota) and
  124. other.azs == self.azs and
  125. other.networks == self.networks and
  126. other.security_groups == self.security_groups and
  127. other.auto_floating_ip == self.auto_floating_ip and
  128. other.host_key_checking == self.host_key_checking and
  129. other.labels == self.labels)
  130. return False
  131. def __repr__(self):
  132. return "<ProviderPool %s>" % self.name
  133. def load(self, pool_config, full_config, provider):
  134. '''
  135. Load pool configuration options.
  136. :param dict pool_config: A single pool config section from which we
  137. will load the values.
  138. :param dict full_config: The full nodepool config.
  139. :param OpenStackProviderConfig: The calling provider object.
  140. '''
  141. super().load(pool_config)
  142. self.provider = provider
  143. self.name = pool_config['name']
  144. self.max_cores = pool_config.get('max-cores', math.inf)
  145. self.max_ram = pool_config.get('max-ram', math.inf)
  146. self.ignore_provider_quota = pool_config.get('ignore-provider-quota',
  147. False)
  148. self.azs = pool_config.get('availability-zones')
  149. self.networks = pool_config.get('networks', [])
  150. self.security_groups = pool_config.get('security-groups', [])
  151. self.auto_floating_ip = bool(pool_config.get('auto-floating-ip', True))
  152. self.host_key_checking = bool(pool_config.get('host-key-checking',
  153. True))
  154. for label in pool_config.get('labels', []):
  155. pl = ProviderLabel()
  156. pl.name = label['name']
  157. pl.pool = self
  158. self.labels[pl.name] = pl
  159. diskimage = label.get('diskimage', None)
  160. if diskimage:
  161. pl.diskimage = full_config.diskimages[diskimage]
  162. else:
  163. pl.diskimage = None
  164. cloud_image_name = label.get('cloud-image', None)
  165. if cloud_image_name:
  166. cloud_image = provider.cloud_images.get(cloud_image_name, None)
  167. if not cloud_image:
  168. raise ValueError(
  169. "cloud-image %s does not exist in provider %s"
  170. " but is referenced in label %s" %
  171. (cloud_image_name, self.name, pl.name))
  172. else:
  173. cloud_image = None
  174. pl.cloud_image = cloud_image
  175. pl.min_ram = label.get('min-ram', 0)
  176. pl.flavor_name = label.get('flavor-name', None)
  177. pl.key_name = label.get('key-name')
  178. pl.console_log = label.get('console-log', False)
  179. pl.boot_from_volume = bool(label.get('boot-from-volume',
  180. False))
  181. pl.volume_size = label.get('volume-size', 50)
  182. pl.instance_properties = label.get('instance-properties',
  183. None)
  184. pl.userdata = label.get('userdata', None)
  185. top_label = full_config.labels[pl.name]
  186. top_label.pools.append(self)
  187. class OpenStackProviderConfig(ProviderConfig):
  188. def __init__(self, driver, provider):
  189. self.driver_object = driver
  190. self.__pools = {}
  191. self.cloud_config = None
  192. self.image_type = None
  193. self.rate = None
  194. self.boot_timeout = None
  195. self.launch_timeout = None
  196. self.clean_floating_ips = None
  197. self.diskimages = {}
  198. self.cloud_images = {}
  199. self.hostname_format = None
  200. self.image_name_format = None
  201. super().__init__(provider)
  202. def __eq__(self, other):
  203. if isinstance(other, OpenStackProviderConfig):
  204. return (super().__eq__(other) and
  205. other.cloud_config == self.cloud_config and
  206. other.pools == self.pools and
  207. other.image_type == self.image_type and
  208. other.rate == self.rate and
  209. other.boot_timeout == self.boot_timeout and
  210. other.launch_timeout == self.launch_timeout and
  211. other.clean_floating_ips == self.clean_floating_ips and
  212. other.diskimages == self.diskimages and
  213. other.cloud_images == self.cloud_images)
  214. return False
  215. def _cloudKwargs(self):
  216. cloud_kwargs = {}
  217. for arg in ['region-name', 'cloud']:
  218. if arg in self.provider:
  219. cloud_kwargs[arg] = self.provider[arg]
  220. return cloud_kwargs
  221. @property
  222. def pools(self):
  223. return self.__pools
  224. @property
  225. def manage_images(self):
  226. return True
  227. def load(self, config):
  228. cloud_kwargs = self._cloudKwargs()
  229. openstack_config = self.driver_object.openstack_config
  230. self.cloud_config = openstack_config.get_one(**cloud_kwargs)
  231. self.image_type = self.cloud_config.config['image_format']
  232. self.region_name = self.provider.get('region-name')
  233. self.rate = float(self.provider.get('rate', 1.0))
  234. self.boot_timeout = self.provider.get('boot-timeout', 60)
  235. self.launch_timeout = self.provider.get('launch-timeout', 3600)
  236. self.launch_retries = self.provider.get('launch-retries', 3)
  237. self.clean_floating_ips = self.provider.get('clean-floating-ips')
  238. self.hostname_format = self.provider.get(
  239. 'hostname-format',
  240. '{label.name}-{provider.name}-{node.id}'
  241. )
  242. self.image_name_format = self.provider.get(
  243. 'image-name-format',
  244. '{image_name}-{timestamp}'
  245. )
  246. default_port_mapping = {
  247. 'ssh': 22,
  248. 'winrm': 5986,
  249. }
  250. for image in self.provider.get('diskimages', []):
  251. i = ProviderDiskImage()
  252. i.name = image['name']
  253. self.diskimages[i.name] = i
  254. diskimage = config.diskimages[i.name]
  255. diskimage.image_types.add(self.image_type)
  256. i.pause = bool(image.get('pause', False))
  257. i.config_drive = image.get('config-drive', None)
  258. i.connection_type = image.get('connection-type', 'ssh')
  259. i.connection_port = image.get(
  260. 'connection-port',
  261. default_port_mapping.get(i.connection_type, 22))
  262. # This dict is expanded and used as custom properties when
  263. # the image is uploaded.
  264. i.meta = image.get('meta', {})
  265. # 5 elements, and no key or value can be > 255 chars
  266. # per Nova API rules
  267. if i.meta:
  268. if len(i.meta) > 5 or \
  269. any([len(k) > 255 or len(v) > 255
  270. for k, v in i.meta.items()]):
  271. # soft-fail
  272. # self.log.error("Invalid metadata for %s; ignored"
  273. # % i.name)
  274. i.meta = {}
  275. for image in self.provider.get('cloud-images', []):
  276. i = ProviderCloudImage()
  277. i.name = image['name']
  278. i.config_drive = image.get('config-drive', None)
  279. i.image_id = image.get('image-id', None)
  280. i.image_name = image.get('image-name', None)
  281. i.username = image.get('username', None)
  282. i.connection_type = image.get('connection-type', 'ssh')
  283. i.connection_port = image.get(
  284. 'connection-port',
  285. default_port_mapping.get(i.connection_type, 22))
  286. self.cloud_images[i.name] = i
  287. for pool in self.provider.get('pools', []):
  288. pp = ProviderPool()
  289. pp.load(pool, config, self)
  290. self.pools[pp.name] = pp
  291. def getSchema(self):
  292. provider_diskimage = {
  293. 'name': str,
  294. 'pause': bool,
  295. 'meta': dict,
  296. 'config-drive': bool,
  297. 'connection-type': str,
  298. 'connection-port': int,
  299. }
  300. provider_cloud_images = {
  301. 'name': str,
  302. 'config-drive': bool,
  303. 'connection-type': str,
  304. 'connection-port': int,
  305. v.Exclusive('image-id', 'cloud-image-name-or-id'): str,
  306. v.Exclusive('image-name', 'cloud-image-name-or-id'): str,
  307. 'username': str,
  308. }
  309. pool_label_main = {
  310. v.Required('name'): str,
  311. v.Exclusive('diskimage', 'label-image'): str,
  312. v.Exclusive('cloud-image', 'label-image'): str,
  313. 'min-ram': int,
  314. 'flavor-name': str,
  315. 'key-name': str,
  316. 'console-log': bool,
  317. 'boot-from-volume': bool,
  318. 'volume-size': int,
  319. 'instance-properties': dict,
  320. 'userdata': str,
  321. }
  322. label_min_ram = v.Schema({v.Required('min-ram'): int}, extra=True)
  323. label_flavor_name = v.Schema({v.Required('flavor-name'): str},
  324. extra=True)
  325. label_diskimage = v.Schema({v.Required('diskimage'): str}, extra=True)
  326. label_cloud_image = v.Schema({v.Required('cloud-image'): str},
  327. extra=True)
  328. pool_label = v.All(pool_label_main,
  329. v.Any(label_min_ram, label_flavor_name),
  330. v.Any(label_diskimage, label_cloud_image))
  331. pool = ConfigPool.getCommonSchemaDict()
  332. pool.update({
  333. 'name': str,
  334. 'networks': [str],
  335. 'auto-floating-ip': bool,
  336. 'host-key-checking': bool,
  337. 'ignore-provider-quota': bool,
  338. 'max-cores': int,
  339. 'max-ram': int,
  340. 'labels': [pool_label],
  341. 'availability-zones': [str],
  342. 'security-groups': [str]
  343. })
  344. schema = ProviderConfig.getCommonSchemaDict()
  345. schema.update({
  346. 'region-name': str,
  347. v.Required('cloud'): str,
  348. 'boot-timeout': int,
  349. 'launch-timeout': int,
  350. 'launch-retries': int,
  351. 'nodepool-id': str,
  352. 'rate': v.Coerce(float),
  353. 'hostname-format': str,
  354. 'image-name-format': str,
  355. 'clean-floating-ips': bool,
  356. 'pools': [pool],
  357. 'diskimages': [provider_diskimage],
  358. 'cloud-images': [provider_cloud_images],
  359. })
  360. return v.Schema(schema)
  361. def getSupportedLabels(self, pool_name=None):
  362. labels = set()
  363. for pool in self.pools.values():
  364. if not pool_name or (pool.name == pool_name):
  365. labels.update(pool.labels.keys())
  366. return labels