Fuel plugin for Mellanox support
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

mellanox_settings.py 22KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543
  1. #!/usr/bin/python
  2. # Copyright 2016 Mellanox Technologies, Ltd
  3. #
  4. # Licensed under the Apache License, Version 2.0 (the "License");
  5. # you may not use this file except in compliance with the License.
  6. # You may obtain a copy of the License at
  7. #
  8. # http://www.apache.org/licenses/LICENSE-2.0
  9. #
  10. # Unless required by applicable law or agreed to in writing, software
  11. # distributed under the License is distributed on an "AS IS" BASIS,
  12. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  13. # implied.
  14. # See the License for the specific language governing permissions and
  15. # limitations under the License.
  16. import os
  17. import sys
  18. import subprocess
  19. import yaml
  20. import glob
  21. import logging
  22. import traceback
  23. MAX_NUM_VFS = 16
  24. MLNX_SECTION = 'mellanox-plugin'
  25. SETTINGS_FILE = '/etc/astute.yaml'
  26. PLUGIN_OVERRIDE_FILE = '/etc/hiera/override/plugins.yaml'
  27. MLNX_DRIVERS_LIST = { 'ConnectX-3': {'eth_driver' : 'mlx4_en', 'ib_driver' : 'eth_ipoib'},
  28. 'ConnectX-4': {'eth_driver' : 'mlx5_core', 'ib_driver' : 'eth_ipoib'},
  29. 'ConnectX-5': {'eth_driver' : 'mlx5_core', 'ib_driver' : 'eth_ipoib'}}
  30. MLNX_DRIVERS = set([MLNX_DRIVERS_LIST[card][net]
  31. for card in MLNX_DRIVERS_LIST
  32. for net in MLNX_DRIVERS_LIST[card]])
  33. ETH_DRIVERS = set([MLNX_DRIVERS_LIST[card][net]
  34. for card in MLNX_DRIVERS_LIST
  35. for net in MLNX_DRIVERS_LIST[card]
  36. if net == 'eth_driver'])
  37. IB_DRIVERS = MLNX_DRIVERS - ETH_DRIVERS
  38. ISER_IFC_NAME = 'mlnx_iser0'
  39. LOG_FILE = '/var/log/mellanox-plugin.log'
  40. class MellanoxSettingsException(Exception):
  41. pass
  42. class MellanoxSettings(object):
  43. data = None
  44. mlnx_interfaces_section = None
  45. @classmethod
  46. def get_mlnx_section(cls):
  47. if cls.data is None:
  48. raise MellanoxSettingsException("No YAML file loaded")
  49. if MLNX_SECTION not in cls.data:
  50. raise MellanoxSettingsException(
  51. "Couldn't find section '{0}'".format(MLNX_SECTION)
  52. )
  53. return cls.data[MLNX_SECTION]
  54. @classmethod
  55. def get_bridge_for_network(cls, network):
  56. network_to_bridge = {
  57. 'private': 'prv',
  58. 'management': 'mgmt',
  59. 'storage': 'storage',
  60. }
  61. return 'br-{0}'.format(network_to_bridge[network])
  62. @classmethod
  63. def get_interface_by_network(cls, network):
  64. if network not in ('management', 'storage', 'private'):
  65. raise MellanoxSettingsException("Unknown network: {0}".format(network))
  66. mlnx_interfaces_section = cls.mlnx_interfaces_section
  67. ifc = mlnx_interfaces_section[network]['interface']
  68. return ifc
  69. @classmethod
  70. def get_card_type(cls, driver):
  71. for card in MLNX_DRIVERS_LIST.keys():
  72. if driver in MLNX_DRIVERS_LIST[card].values():
  73. network_driver_type = MLNX_DRIVERS_LIST[card].keys()[MLNX_DRIVERS_LIST[card].values()\
  74. .index(driver)]
  75. return card
  76. @classmethod
  77. def get_card_type_by_interface_name(cls, ifc_name):
  78. card_type = os.popen('mst status -v | grep {0} | grep -o ConnectX[0-9]*'
  79. .format(ifc_name)).readlines()
  80. if len(card_type) == 1:
  81. card_type = card_type[0].replace('ConnectX', 'ConnectX-').strip()
  82. return card_type
  83. else:
  84. logging.error('No driver found for interface {0}'.format(ifc_name))
  85. exit(1)
  86. @classmethod
  87. def add_cx_card(cls):
  88. mlnx_interfaces = cls.mlnx_interfaces_section
  89. drivers = list()
  90. interfaces = list()
  91. mlnx = cls.get_mlnx_section()
  92. for network_type, ifc_dict in mlnx_interfaces.iteritems():
  93. if 'driver' in ifc_dict and network_type in ['private','management','storage']:
  94. # The bond interfaces extend the original list,
  95. # otherwise, the interface is appended to the list.
  96. if(type(ifc_dict['driver']) is list):
  97. drivers.extend(ifc_dict['driver'])
  98. else:
  99. drivers.append(ifc_dict['driver'])
  100. if(type(ifc_dict['interface']) is list):
  101. interfaces.extend(ifc_dict['interface'])
  102. else:
  103. interfaces.append(ifc_dict['interface'])
  104. drivers_set = list(set(drivers))
  105. interfaces_set = list(set(interfaces))
  106. if (len(drivers_set) > 1):
  107. logging.error("Multiple ConnectX adapters was found in this environment.")
  108. raise MellanoxSettingsException(
  109. "Multiple ConnectX adapters was found in this environment."
  110. )
  111. else:
  112. current_driver = drivers_set[0]
  113. mellanox_interface = interfaces_set[0]
  114. if current_driver in ETH_DRIVERS:
  115. mlnx['network_type'] = 'ethernet'
  116. mlnx['cx_card'] = cls.get_card_type_by_interface_name(mellanox_interface)
  117. elif current_driver in IB_DRIVERS:
  118. mlnx['network_type'] = 'infiniband'
  119. ibdev = os.popen('ibdev2netdev').readlines()
  120. if not ibdev:
  121. mlnx['cx_card'] = 'none'
  122. logging.error('Failed executing ibdev2netdev')
  123. return 0
  124. if ('bonds' in cls.data and mellanox_interface.startswith('bond')):
  125. mellanox_interface = cls.data['bonds'][mellanox_interface]['interfaces'][0]
  126. mlnx['cx_card'] = cls.get_card_type_by_interface_name(mellanox_interface)
  127. network_info_msg = 'Detected Network Type is: {0} '.format(mlnx['network_type'])
  128. card_info_msg = 'Detected Card Type is: {0} '.format(mlnx['cx_card'])
  129. logging.info(network_info_msg)
  130. logging.info(card_info_msg)
  131. @classmethod
  132. def add_driver(cls):
  133. interfaces = cls.get_interfaces_section()
  134. mlnx = cls.get_mlnx_section()
  135. drivers = cls.get_physical_interfaces()
  136. if len(drivers) > 1:
  137. raise MellanoxSettingsException(
  138. "Found mismatching Mellanox drivers on different interfaces: "
  139. "{0}".format(mlnx_drivers)
  140. )
  141. if len(drivers) == 0:
  142. raise MellanoxSettingsException(
  143. "\nNo Network role was assigned to Mellanox interfaces. "
  144. "\nPlease go to nodes tab in Fuel UI and reset your network "
  145. "roles in interfaces screen. aborting. "
  146. )
  147. mlnx['driver'] = drivers[0]
  148. @classmethod
  149. def add_physical_port(cls):
  150. interfaces = cls.get_interfaces_section()
  151. mlnx = cls.get_mlnx_section()
  152. private_ifc = cls.get_interface_by_network('private')
  153. if mlnx['driver'] == MLNX_DRIVERS_LIST[mlnx['cx_card']]['ib_driver']:
  154. if 'bus_info' not in interfaces[private_ifc]['vendor_specific']:
  155. raise MellanoxSettingsException(
  156. "Couldn't find 'bus_info' for interface "
  157. "{0}".format(private_ifc)
  158. )
  159. mlnx['physical_port'] = interfaces[private_ifc]['vendor_specific']['bus_info']
  160. elif mlnx['driver'] == MLNX_DRIVERS_LIST[mlnx['cx_card']]['eth_driver']:
  161. # If only iSER
  162. if not cls.is_sriov_enabled() and cls.is_iser_enabled():
  163. mlnx = cls.get_mlnx_section()
  164. storage_ifc = cls.get_interface_by_network('storage')
  165. mlnx['physical_port'] = storage_ifc
  166. # If SR-IOV
  167. else:
  168. mlnx['physical_port'] = private_ifc
  169. @classmethod
  170. def add_storage_vlan(cls):
  171. mlnx = cls.get_mlnx_section()
  172. mlnx_interfaces_section = cls.mlnx_interfaces_section
  173. vlan = mlnx_interfaces_section['storage']['vlan']
  174. # Set storage vlan in mlnx section if vlan is used with iser
  175. if vlan:
  176. try:
  177. mlnx['storage_vlan'] = int(vlan)
  178. except ValueError:
  179. raise MellanoxSettingsException(
  180. "Failed reading vlan for br-storage"
  181. )
  182. if mlnx['driver'] == MLNX_DRIVERS_LIST[mlnx['cx_card']]['ib_driver']:
  183. pkey = format((int(vlan) ^ 0x8000),'04x')
  184. mlnx['storage_pkey'] = pkey
  185. @classmethod
  186. def add_storage_parent(cls):
  187. mlnx = cls.get_mlnx_section()
  188. storage_ifc = cls.get_interface_by_network('storage')
  189. mlnx['storage_parent'] = storage_ifc
  190. @classmethod
  191. def add_iser_interface_name(cls):
  192. mlnx = cls.get_mlnx_section()
  193. storage_ifc = cls.get_interface_by_network('storage')
  194. if mlnx['driver'] == MLNX_DRIVERS_LIST[mlnx['cx_card']]['eth_driver']:
  195. mlnx['iser_ifc_name'] = ISER_IFC_NAME
  196. elif mlnx['driver'] == MLNX_DRIVERS_LIST[mlnx['cx_card']]['ib_driver']:
  197. interfaces = cls.get_interfaces_section()
  198. mlnx['iser_ifc_name'] = interfaces[storage_ifc]['vendor_specific']['bus_info']
  199. else:
  200. raise MellanoxSettingsException("Could not find 'driver' in "
  201. "{0} section".format(MLNX_SECTION))
  202. @classmethod
  203. def set_storage_networking_scheme(cls):
  204. endpoints = cls.get_endpoints_section()
  205. interfaces = cls.get_interfaces_section()
  206. transformations = cls.data['network_scheme']['transformations']
  207. mlnx = cls.get_mlnx_section()
  208. for transformation in transformations:
  209. if ('bridges' in transformation) and ('br-storage' in transformation['bridges']):
  210. transformations.remove(transformation)
  211. elif ('name' in transformation) and ('br-storage' == transformation['name']) \
  212. and ('action' in transformation) and ('add-br' == transformation['action']):
  213. transformations.remove(transformation)
  214. # Handle iSER interface with and w/o vlan tagging
  215. storage_vlan = mlnx.get('storage_vlan')
  216. storage_parent = cls.get_interface_by_network('storage')
  217. if storage_vlan and mlnx['driver'] == MLNX_DRIVERS_LIST[mlnx['cx_card']]['eth_driver']: # Use VLAN dev
  218. vlan_name = "{0}.{1}".format(ISER_IFC_NAME, storage_vlan)
  219. # Set storage rule to iSER interface vlan interface
  220. cls.data['network_scheme']['roles']['storage'] = vlan_name
  221. # Set iSER interface vlan interface
  222. transformations.append({
  223. 'action': 'add-port',
  224. 'name': vlan_name,
  225. 'vlan_id': int(storage_vlan),
  226. 'vlan_dev': ISER_IFC_NAME,
  227. 'mtu': '1500'
  228. })
  229. endpoints[vlan_name] = (
  230. endpoints.pop('br-storage', {})
  231. )
  232. else:
  233. vlan_name = mlnx['iser_ifc_name']
  234. # Commented until fixing bug LP #1450420
  235. # Meanwhile using a workaround of configuring ib0
  236. # and changing to its child in post deployment
  237. #if storage_vlan: # IB child
  238. # vlan_name = mlnx['iser_ifc_name'] = \
  239. # "{0}.{1}".format(mlnx['iser_ifc_name'],
  240. # mlnx['storage_pkey'])
  241. # Set storage rule to iSER port
  242. cls.data['network_scheme']['roles']['storage'] = \
  243. mlnx['iser_ifc_name']
  244. # Set iSER endpoint with br-storage parameters
  245. endpoints[mlnx['iser_ifc_name']] = (
  246. endpoints.pop('br-storage', {})
  247. )
  248. interfaces[mlnx['iser_ifc_name']] = {}
  249. # Set role
  250. for role,bridge in cls.data['network_scheme']['roles'].iteritems():
  251. if bridge == 'br-storage':
  252. cls.data['network_scheme']['roles'][role] = vlan_name
  253. # Clean
  254. if storage_vlan: \
  255. storage_parent = "{0}.{1}".format(storage_parent, storage_vlan)
  256. for transformation in transformations:
  257. if ('name' in transformation) and (transformation['name'] == storage_parent) \
  258. and ('bridge' in transformation) and (transformation['bridge'] == 'br-storage') \
  259. and ('action' in transformation) and (transformation['action'] == 'add-port'):
  260. transformations.remove(transformation)
  261. endpoints['br-storage'] = {'IP' : 'None'}
  262. @classmethod
  263. def get_endpoints_section(cls):
  264. return cls.data['network_scheme']['endpoints']
  265. @classmethod
  266. def get_physical_interfaces(cls):
  267. # the main change will be here because it reads phy_interfaces
  268. mlnx_interfaces = cls.mlnx_interfaces_section
  269. drivers = list()
  270. mlnx = cls.get_mlnx_section()
  271. for network_type, ifc_dict in mlnx_interfaces.iteritems():
  272. if 'driver' in ifc_dict and \
  273. ifc_dict['driver'] in MLNX_DRIVERS_LIST[mlnx['cx_card']].values():
  274. drivers.append(ifc_dict['driver'])
  275. return list(set(drivers))
  276. @classmethod
  277. def get_interfaces_section(cls):
  278. return cls.data['network_scheme']['interfaces']
  279. @classmethod
  280. def is_iser_enabled(cls):
  281. return cls.get_mlnx_section()['iser']
  282. @classmethod
  283. def is_sriov_enabled(cls):
  284. return cls.get_mlnx_section()['sriov']
  285. @classmethod
  286. def is_vxlan_offloading_enabled(cls):
  287. return cls.get_mlnx_section()['vxlan_offloading']
  288. @classmethod
  289. def add_reboot_condition(cls):
  290. # if MAX_NUM_VF > default which is 16, reboot
  291. mlnx = cls.get_mlnx_section()
  292. mst_start = os.popen('mst start;').readlines()
  293. burned_num_vfs_list = list()
  294. devices = os.popen('mst status -v| grep pciconf | grep {0} | awk \'{{print $2}}\' '.format(
  295. mlnx['cx_card'].replace("-",""))).readlines()
  296. if len(devices) > 0:
  297. for dev in devices:
  298. num = os.popen('mlxconfig -d {0} q | grep NUM_OF_VFS | awk \'{{print $2}}\' \
  299. '.format(dev.rsplit()[0])).readlines()
  300. if len(num) > 0:
  301. burned_num_vfs_list.append(num[0].rsplit()[0])
  302. else:
  303. logging.error("Failed to grep NUM_OF_VFS from Mellanox card")
  304. sys.exit(1)
  305. burned_num_vfs_set_list = list(set(burned_num_vfs_list))
  306. for burned_num_vfs in burned_num_vfs_set_list :
  307. if int(burned_num_vfs) < int(mlnx['num_of_vfs']) :
  308. mlnx['reboot_required'] = True
  309. logging.info('reboot_required is true as {0} is < {1}'.format(burned_num_vfs,
  310. mlnx['num_of_vfs']))
  311. break;
  312. else:
  313. logging.error("There are no Mellanox devices with {0} card".format(mlnx['cx_card']))
  314. sys.exit(1)
  315. @classmethod
  316. def update_role_settings(cls):
  317. # detect ConnectX card
  318. cls.add_cx_card()
  319. # realize the driver in use (eth/ib)
  320. cls.add_driver()
  321. # decide the physical function for SR-IOV
  322. cls.add_physical_port()
  323. # set iSER parameters
  324. if cls.is_iser_enabled():
  325. cls.add_storage_parent()
  326. cls.add_storage_vlan()
  327. cls.add_iser_interface_name()
  328. cls.set_storage_networking_scheme()
  329. # fill reboot condition
  330. cls.add_reboot_condition()
  331. @classmethod
  332. def read_from_yaml(cls, settings_file):
  333. try:
  334. fd = open(settings_file, 'r')
  335. except IOError:
  336. raise MellanoxSettingsException("Given YAML file {0} doesn't "
  337. "exist".format(settings_file))
  338. try:
  339. data = yaml.load(fd)
  340. except yaml.YAMLError, exc:
  341. if hasattr(exc, 'problem_mark'):
  342. mark = exc.problem_mark
  343. raise MellanoxSettingsException(
  344. "Faild parsing YAML file {0}: error position "
  345. "({2},{3})".format(mark.line+1, mark.column+1)
  346. )
  347. finally:
  348. fd.close()
  349. cls.data = data
  350. cls.mlnx_interfaces_section = cls.get_mlnx_interfaces_section()
  351. @classmethod
  352. def write_to_yaml(cls, settings_file):
  353. # choose only the edited sections
  354. data = {}
  355. data['network_scheme'] = cls.data['network_scheme']
  356. data[MLNX_SECTION] = cls.data[MLNX_SECTION]
  357. # create containing adir
  358. try:
  359. settings_dir = os.path.dirname(settings_file)
  360. if not os.path.isdir(settings_dir):
  361. os.makedirs(settings_dir)
  362. except OSError:
  363. raise MellanoxSettingsException(
  364. "Failed creating directory: {0}".format(settings_dir)
  365. )
  366. try:
  367. fd = open(settings_file, 'w')
  368. yaml.dump(data, fd, default_flow_style=False)
  369. except IOError:
  370. raise MellanoxSettingsException("Failed writing changes to "
  371. "{0}".format(settings_file))
  372. finally:
  373. if fd:
  374. fd.close()
  375. @classmethod
  376. def update_settings(cls):
  377. # define input yaml file
  378. try:
  379. cls.read_from_yaml(SETTINGS_FILE)
  380. cls.update_role_settings()
  381. cls.write_to_yaml(PLUGIN_OVERRIDE_FILE)
  382. except MellanoxSettingsException, exc:
  383. error_msg = "Couldn't add Mellanox settings to " \
  384. "{0}: {1}\n".format(SETTINGS_FILE, exc)
  385. sys.stderr.write(error_msg)
  386. logging.error(error_msg)
  387. raise MellanoxSettingsException("Failed updating one or more "
  388. "setting files")
  389. @classmethod
  390. def get_mlnx_interfaces_section(cls):
  391. transformations = cls.data['network_scheme']['transformations']
  392. interfaces = cls.data['network_scheme']['interfaces']
  393. dict_of_interfaces = {}
  394. # Map bonds to interfaces
  395. for transformation in transformations:
  396. if transformation['action'] == 'add-bond':
  397. # Init bonds on the first bond
  398. if 'bonds' not in cls.data:
  399. cls.data['bonds'] = {}
  400. # Init bond assumptions
  401. all_drivers_equal = True
  402. first = transformation['interfaces'][0]
  403. driver = interfaces[first]['vendor_specific']['driver']
  404. # Check if all bond drivers are the same
  405. for interface in transformation['interfaces']:
  406. new_driver = \
  407. interfaces[interface]['vendor_specific']['driver']
  408. if new_driver != driver:
  409. all_drivers_equal = False
  410. if all_drivers_equal:
  411. bond_driver = driver
  412. else:
  413. bond_driver = None
  414. cls.data['bonds'][transformation['name']] = \
  415. {'interfaces' : transformation['interfaces'],
  416. 'driver' : bond_driver}
  417. # Map networks to interfaces
  418. for transformation in transformations:
  419. if 'bridge' in transformation.keys() and \
  420. (transformation['action'] == 'add-port' or \
  421. transformation['action'] == 'add-bond'):
  422. if transformation['bridge'] == 'br-fw-admin':
  423. network_type = 'admin'
  424. elif transformation['bridge'] == 'br-ex':
  425. network_type = 'public'
  426. elif transformation['bridge'] == 'br-aux' or \
  427. transformation['bridge'] == 'br-mesh':
  428. network_type = 'private'
  429. elif transformation['bridge'] == 'br-mgmt':
  430. network_type = 'management'
  431. elif transformation['bridge'] == 'br-storage':
  432. network_type = 'storage'
  433. elif transformation['bridge'] == 'br-baremetal':
  434. network_type = 'baremetal'
  435. network_interface = {}
  436. network_interface['bridge'] = transformation['bridge']
  437. # Split to iface name and VLAN
  438. iface_split = transformation['name'].split('.')
  439. if len(iface_split)==1:
  440. iface_split.append(str(1))
  441. interface, vlan = iface_split
  442. network_interface['interface'] = interface
  443. network_interface['vlan'] = vlan
  444. # If bond
  445. if 'bonds' in cls.data and interface in cls.data['bonds']:
  446. network_interface['driver'] = \
  447. cls.data['bonds'][interface]['driver']
  448. if ( network_type == 'private' and cls.is_sriov_enabled() ) or \
  449. ( network_type == 'storage' and cls.is_iser_enabled() ):
  450. # Assign SR-IOV/ISER to the first port only.
  451. # This is a temporary workaround until supporing bond over VFs.
  452. # We sort the array of interfaces in order to get the first
  453. # interface on all nodes.
  454. if_list = cls.data['bonds'][interface]['interfaces']
  455. if_list.sort()
  456. network_interface['interface'] = if_list[0]
  457. else: # Not a bond
  458. network_interface['driver'] = \
  459. interfaces[interface]['vendor_specific']['driver']
  460. dict_of_interfaces[network_type] = network_interface
  461. # Set private network in case private and storage on the same port
  462. if 'private' not in dict_of_interfaces.keys() and \
  463. 'storage' in dict_of_interfaces.keys():
  464. dict_of_interfaces['private'] = dict_of_interfaces['storage']
  465. dict_of_interfaces['private']['bridge'] = 'br-prv'
  466. return dict_of_interfaces
  467. def main():
  468. logging.basicConfig(format='%(asctime)s %(message)s',
  469. level=logging.DEBUG, filename=LOG_FILE)
  470. try:
  471. settings = MellanoxSettings()
  472. settings.update_settings()
  473. except MellanoxSettingsException, exc:
  474. error_msg = "Failed adding Mellanox settings: {0}\n".format(exc)
  475. sys.stderr.write(error_msg)
  476. logging.error(exc)
  477. sys.exit(1)
  478. success_msg = "Done adding Mellanox settings\n"
  479. sys.stdout.write(success_msg)
  480. logging.info(success_msg)
  481. sys.exit(0)
  482. if __name__ == '__main__':
  483. main()