Host network configuration tool
Nelze vybrat více než 25 témat Téma musí začínat písmenem nebo číslem, může obsahovat pomlčky („-“) a může být dlouhé až 35 znaků.

603 řádky
21KB

  1. # -*- coding: utf-8 -*-
  2. # Copyright 2014 Red Hat, Inc.
  3. #
  4. # Licensed under the Apache License, Version 2.0 (the "License"); you may
  5. # not use this file except in compliance with the License. You may obtain
  6. # a copy of the License at
  7. #
  8. # http://www.apache.org/licenses/LICENSE-2.0
  9. #
  10. # Unless required by applicable law or agreed to in writing, software
  11. # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  12. # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  13. # License for the specific language governing permissions and limitations
  14. # under the License.
  15. #
  16. # The sriov_config.py module does the SR-IOV PF configuration.
  17. # It'll be invoked by the sriov_config systemd service for the persistence of
  18. # the SR-IOV configuration across reboots. And os-net-config:utils also invokes
  19. # it for the first time configuration.
  20. # An entry point os-net-config-sriov is added for invocation of this module.
  21. import argparse
  22. import logging
  23. import os
  24. import pyudev
  25. import re
  26. from six.moves import queue as Queue
  27. import sys
  28. import time
  29. import yaml
  30. from oslo_concurrency import processutils
  31. logger = logging.getLogger(__name__)
  32. _SYS_CLASS_NET = '/sys/class/net'
  33. _UDEV_RULE_FILE = '/etc/udev/rules.d/80-persistent-os-net-config.rules'
  34. _UDEV_LEGACY_RULE_FILE = '/etc/udev/rules.d/70-os-net-config-sriov.rules'
  35. _IFUP_LOCAL_FILE = '/sbin/ifup-local'
  36. _RESET_SRIOV_RULES_FILE = '/etc/udev/rules.d/70-tripleo-reset-sriov.rules'
  37. _ALLOCATE_VFS_FILE = '/etc/sysconfig/allocate_vfs'
  38. MAX_RETRIES = 10
  39. PF_FUNC_RE = re.compile(r"\.(\d+)$", 0)
  40. # In order to keep VF representor name consistent specially after the upgrade
  41. # proccess, we should have a udev rule to handle that.
  42. # The udev rule will rename the VF representor as "<sriov_pf_name>_<vf_num>"
  43. _REP_LINK_NAME_FILE = "/etc/udev/rep-link-name.sh"
  44. _REP_LINK_NAME_DATA = '''#!/bin/bash
  45. # This file is autogenerated by os-net-config
  46. set -x
  47. PORT="$1"
  48. echo "NUMBER=${PORT##pf*vf}"
  49. '''
  50. # Create a queue for passing the udev network events
  51. vf_queue = Queue.Queue()
  52. # File to contain the list of SR-IOV PF, VF and their configurations
  53. # Format of the file shall be
  54. # - device_type: pf
  55. # name: <pf name>
  56. # numvfs: <number of VFs>
  57. # promisc: "on"/"off"
  58. # - device_type: vf
  59. # device:
  60. # name: <pf name>
  61. # vfid: <VF id>
  62. # name: <vf name>
  63. # vlan_id: <vlan>
  64. # qos: <qos>
  65. # spoofcheck: "on"/"off"
  66. # trust: "on"/"off"
  67. # state: "auto"/"enable"/"disable"
  68. # macaddr: <mac address>
  69. # promisc: "on"/"off"
  70. _SRIOV_CONFIG_FILE = '/var/lib/os-net-config/sriov_config.yaml'
  71. class SRIOVNumvfsException(ValueError):
  72. pass
  73. def udev_event_handler(action, device):
  74. event = {"action": action, "device": device.sys_path}
  75. logger.info("Received udev event %s for %s"
  76. % (event["action"], event["device"]))
  77. vf_queue.put(event)
  78. def get_file_data(filename):
  79. if not os.path.exists(filename):
  80. return ''
  81. try:
  82. with open(filename, 'r') as f:
  83. return f.read()
  84. except IOError:
  85. logger.error("Error reading file: %s" % filename)
  86. return ''
  87. def _get_sriov_map():
  88. contents = get_file_data(_SRIOV_CONFIG_FILE)
  89. sriov_map = yaml.safe_load(contents) if contents else []
  90. return sriov_map
  91. def get_numvfs(ifname):
  92. try:
  93. sriov_numvfs_path = os.path.join(_SYS_CLASS_NET, ifname,
  94. "device/sriov_numvfs")
  95. with open(sriov_numvfs_path, 'r') as f:
  96. return int(f.read())
  97. except IOError as exc:
  98. msg = ("Unable to read numvfs for %s" % ifname)
  99. raise SRIOVNumvfsException(msg)
  100. def restart_ovs_and_pfs_netdevs():
  101. sriov_map = _get_sriov_map()
  102. processutils.execute('/usr/bin/systemctl', 'restart', 'openvswitch')
  103. for item in sriov_map:
  104. if item['device_type'] == 'pf':
  105. if_down_interface(item['name'])
  106. if_up_interface(item['name'])
  107. def cleanup_puppet_config():
  108. file_contents = ""
  109. if os.path.exists(_RESET_SRIOV_RULES_FILE):
  110. os.remove(_RESET_SRIOV_RULES_FILE)
  111. if os.path.exists(_ALLOCATE_VFS_FILE):
  112. os.remove(_ALLOCATE_VFS_FILE)
  113. if os.path.exists(_IFUP_LOCAL_FILE):
  114. # Remove the invocation of allocate_vfs script generated by puppet
  115. # After the removal of allocate_vfs, if the ifup-local file has just
  116. # "#!/bin/bash" left, then remove the file as well.
  117. with open(_IFUP_LOCAL_FILE) as oldfile:
  118. for line in oldfile:
  119. if "/etc/sysconfig/allocate_vfs" not in line:
  120. file_contents = file_contents + line
  121. if file_contents.strip() == "#!/bin/bash":
  122. os.remove(_IFUP_LOCAL_FILE)
  123. else:
  124. with open(_IFUP_LOCAL_FILE, 'w') as newfile:
  125. newfile.write(file_contents)
  126. def udev_monitor_setup():
  127. # Create a context for pyudev and observe udev events for network
  128. context = pyudev.Context()
  129. monitor = pyudev.Monitor.from_netlink(context)
  130. monitor.filter_by('net')
  131. observer = pyudev.MonitorObserver(monitor, udev_event_handler)
  132. return observer
  133. def udev_monitor_start(observer):
  134. observer.start()
  135. def udev_monitor_stop(observer):
  136. observer.stop()
  137. def configure_sriov_pf(execution_from_cli=False, restart_openvswitch=False):
  138. observer = udev_monitor_setup()
  139. udev_monitor_start(observer)
  140. sriov_map = _get_sriov_map()
  141. MLNX_UNBIND_FILE_PATH = "/sys/bus/pci/drivers/mlx5_core/unbind"
  142. MLNX_VENDOR_ID = "0x15b3"
  143. trigger_udev_rule = False
  144. # Cleanup the previous config by puppet-tripleo
  145. cleanup_puppet_config()
  146. for item in sriov_map:
  147. if item['device_type'] == 'pf':
  148. _pf_interface_up(item)
  149. if item.get('link_mode') == "legacy":
  150. # Add a udev rule to configure the VF's when PF's are
  151. # released by a guest
  152. add_udev_rule_for_legacy_sriov_pf(item['name'],
  153. item['numvfs'])
  154. try:
  155. sriov_numvfs_path = os.path.join(_SYS_CLASS_NET, item['name'],
  156. "device/sriov_numvfs")
  157. curr_numvfs = get_numvfs(item['name'])
  158. if curr_numvfs == item['numvfs']:
  159. logger.info("Numvfs already configured for %s"
  160. % item['name'])
  161. continue
  162. with open(sriov_numvfs_path, 'w') as f:
  163. f.write("%d" % item['numvfs'])
  164. except IOError as exc:
  165. msg = ("Unable to configure pf: %s with numvfs: %d\n%s"
  166. % (item['name'], item['numvfs'], exc))
  167. raise SRIOVNumvfsException(msg)
  168. # Wait for the creation of VFs for each PF
  169. _wait_for_vf_creation(item['name'], item['numvfs'])
  170. # Configure switchdev mode
  171. vendor_id = get_vendor_id(item['name'])
  172. if (item.get('link_mode') == "switchdev" and
  173. vendor_id == MLNX_VENDOR_ID):
  174. vf_pcis_list = get_vf_pcis_list(item['name'])
  175. for vf_pci in vf_pcis_list:
  176. vf_pci_path = "/sys/bus/pci/devices/%s/driver" % vf_pci
  177. if os.path.exists(vf_pci_path):
  178. with open(MLNX_UNBIND_FILE_PATH, 'w') as f:
  179. f.write("%s" % vf_pci)
  180. # Adding a udev rule to make vf-representors unmanaged by
  181. # NetworkManager
  182. add_udev_rule_to_unmanage_vf_representors_by_nm()
  183. # Adding a udev rule to save the sriov_pf name
  184. trigger_udev_rule = add_udev_rule_for_sriov_pf(item['name'])\
  185. or trigger_udev_rule
  186. configure_switchdev(item['name'])
  187. # Adding a udev rule to rename vf-representors
  188. trigger_udev_rule = add_udev_rule_for_vf_representors(
  189. item['name']) or trigger_udev_rule
  190. # Moving the sriov-PFs to switchdev mode will put the netdev
  191. # interfaces in down state.
  192. # In case we are running during initial deployment,
  193. # bring the interfaces up.
  194. # In case we are running as part of the sriov_config service
  195. # after reboot, net config scripts, which run after
  196. # sriov_config service will bring the interfaces up.
  197. if execution_from_cli:
  198. if_up_interface(item['name'])
  199. # Trigger udev rules if there is new rules written
  200. if trigger_udev_rule:
  201. trigger_udev_rules()
  202. udev_monitor_stop(observer)
  203. if restart_openvswitch:
  204. restart_ovs_and_pfs_netdevs()
  205. def _write_numvfs(device_name, numvfs):
  206. sriov_numvfs_path = os.path.join(_SYS_CLASS_NET, device_name,
  207. "device/sriov_numvfs")
  208. curr_numvfs = get_numvfs(device_name)
  209. if curr_numvfs != 0:
  210. logger.info("Numvfs already configured for %s" % device_name)
  211. return
  212. try:
  213. with open(sriov_numvfs_path, 'w') as f:
  214. f.write("%d" % numvfs)
  215. except IOError as exc:
  216. msg = ("Unable to configure pf: %s with numvfs: %d\n%s"
  217. % (device_name, numvfs, exc))
  218. raise SRIOVNumvfsException(msg)
  219. def _wait_for_vf_creation(pf_name, numvfs):
  220. vf_count = 0
  221. vf_list = []
  222. while vf_count < numvfs:
  223. try:
  224. # wait for 5 seconds after every udev event
  225. event = vf_queue.get(True, 5)
  226. vf_name = os.path.basename(event["device"])
  227. pf_path = os.path.normpath(os.path.join(event["device"],
  228. "../../physfn/net"))
  229. if os.path.isdir(pf_path):
  230. pf_nic = os.listdir(pf_path)
  231. if len(pf_nic) == 1 and pf_name == pf_nic[0]:
  232. if vf_name not in vf_list:
  233. vf_list.append(vf_name)
  234. logger.info("VF: %s created for PF: %s"
  235. % (vf_name, pf_name))
  236. vf_count = vf_count + 1
  237. else:
  238. logger.warning("Unable to parse event %s"
  239. % event["device"])
  240. else:
  241. logger.warning("%s is not a directory" % pf_path)
  242. except Queue.Empty:
  243. logger.info("Timeout in the creation of VFs for PF %s" % pf_name)
  244. return
  245. logger.info("Required VFs are created for PF %s" % pf_name)
  246. def _wait_for_uplink_rep_creation(pf_name):
  247. uplink_rep_phys_switch_id_path = "/sys/class/net/%s/phys_switch_id" \
  248. % pf_name
  249. for i in range(MAX_RETRIES):
  250. if get_file_data(uplink_rep_phys_switch_id_path):
  251. logger.info("Uplink representor %s ready", pf_name)
  252. break
  253. time.sleep(1)
  254. else:
  255. raise RuntimeError("Timeout while waiting for uplink representor %s.",
  256. pf_name)
  257. def create_rep_link_name_script():
  258. with open(_REP_LINK_NAME_FILE, "w") as f:
  259. f.write(_REP_LINK_NAME_DATA)
  260. # Make the _REP_LINK_NAME_FILE executable
  261. os.chmod(_REP_LINK_NAME_FILE, 0o755)
  262. def add_udev_rule_for_sriov_pf(pf_name):
  263. pf_pci = get_pf_pci(pf_name)
  264. udev_data_line = 'SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", '\
  265. 'KERNELS=="%s", NAME="%s"' % (pf_pci, pf_name)
  266. return add_udev_rule(udev_data_line, _UDEV_RULE_FILE)
  267. def add_udev_rule_for_legacy_sriov_pf(pf_name, numvfs):
  268. logger.info("adding udev rules for %s" % (pf_name))
  269. udev_line = 'KERNEL=="%s", '\
  270. 'RUN+="/bin/os-net-config-sriov -n %%k:%d"' \
  271. % (pf_name, numvfs)
  272. return add_udev_rule(udev_line, _UDEV_LEGACY_RULE_FILE)
  273. def add_udev_rule_for_vf_representors(pf_name):
  274. phys_switch_id_path = os.path.join(_SYS_CLASS_NET, pf_name,
  275. "phys_switch_id")
  276. phys_switch_id = get_file_data(phys_switch_id_path).strip()
  277. pf_pci = get_pf_pci(pf_name)
  278. pf_fun_num_match = PF_FUNC_RE.search(pf_pci)
  279. if pf_fun_num_match:
  280. pf_fun_num = pf_fun_num_match.group(1)
  281. else:
  282. logger.error("Failed to get function number for %s \n"
  283. "and so failed to create a udev rule for renaming "
  284. "its' vf-represent" % pf_name)
  285. return
  286. udev_data_line = 'SUBSYSTEM=="net", ACTION=="add", ATTR{phys_switch_id}'\
  287. '=="%s", ATTR{phys_port_name}=="pf%svf*", '\
  288. 'IMPORT{program}="%s $attr{phys_port_name}", '\
  289. 'NAME="%s_$env{NUMBER}"' % (phys_switch_id,
  290. pf_fun_num,
  291. _REP_LINK_NAME_FILE,
  292. pf_name)
  293. create_rep_link_name_script()
  294. return add_udev_rule(udev_data_line, _UDEV_RULE_FILE)
  295. def add_udev_rule_to_unmanage_vf_representors_by_nm():
  296. udev_data_line = 'SUBSYSTEM=="net", ACTION=="add", ATTR{phys_switch_id}'\
  297. '!="", ATTR{phys_port_name}=="pf*vf*", '\
  298. 'ENV{NM_UNMANAGED}="1"'
  299. return add_udev_rule(udev_data_line, _UDEV_RULE_FILE)
  300. def add_udev_rule(udev_data, udev_file):
  301. trigger_udev_rule = False
  302. udev_data = udev_data.strip()
  303. if not os.path.exists(udev_file):
  304. with open(udev_file, "w") as f:
  305. data = "# This file is autogenerated by os-net-config\n%s\n"\
  306. % udev_data
  307. f.write(data)
  308. reload_udev_rules()
  309. trigger_udev_rule = True
  310. else:
  311. file_data = get_file_data(udev_file)
  312. udev_lines = file_data.split("\n")
  313. if udev_data not in udev_lines:
  314. with open(udev_file, "a") as f:
  315. f.write(udev_data + "\n")
  316. reload_udev_rules()
  317. trigger_udev_rule = True
  318. return trigger_udev_rule
  319. def reload_udev_rules():
  320. try:
  321. processutils.execute('/usr/sbin/udevadm', 'control', '--reload-rules')
  322. logger.info("udev rules reloaded successfully")
  323. except processutils.ProcessExecutionError:
  324. logger.error("Failed to reload udev rules")
  325. raise
  326. def trigger_udev_rules():
  327. try:
  328. processutils.execute('/usr/sbin/udevadm', 'trigger', '--action=add',
  329. '--attr-match=subsystem=net')
  330. logger.info("udev rules triggered successfully")
  331. except processutils.ProcessExecutionError:
  332. logger.error("Failed to trigger udev rules")
  333. raise
  334. def configure_switchdev(pf_name):
  335. pf_pci = get_pf_pci(pf_name)
  336. pf_device_id = get_pf_device_id(pf_name)
  337. if pf_device_id == "0x1013" or pf_device_id == "0x1015":
  338. try:
  339. processutils.execute('/usr/sbin/devlink', 'dev', 'eswitch', 'set',
  340. 'pci/%s' % pf_pci, 'inline-mode', 'transport')
  341. except processutils.ProcessExecutionError:
  342. logger.error("Failed to set inline-mode to transport")
  343. raise
  344. try:
  345. processutils.execute('/usr/sbin/devlink', 'dev', 'eswitch', 'set',
  346. 'pci/%s' % pf_pci, 'mode', 'switchdev')
  347. except processutils.ProcessExecutionError:
  348. logger.error("Failed to set mode to switchdev")
  349. raise
  350. logger.info("Device pci/%s set to switchdev mode." % pf_pci)
  351. # WA to make sure that the uplink_rep is ready after moving to switchdev,
  352. # as moving to switchdev will remove the sriov_pf and create uplink
  353. # representor, so we need to make sure that uplink representor is ready
  354. # before proceed
  355. _wait_for_uplink_rep_creation(pf_name)
  356. try:
  357. processutils.execute('/usr/sbin/ethtool', '-K', pf_name,
  358. 'hw-tc-offload', 'on')
  359. logger.info("Enabled \"hw-tc-offload\" for PF %s." % pf_name)
  360. except processutils.ProcessExecutionError:
  361. logger.error("Failed to enable hw-tc-offload")
  362. raise
  363. def run_ip_config_cmd(*cmd, **kwargs):
  364. logger.info("Running %s" % ' '.join(cmd))
  365. try:
  366. processutils.execute(*cmd, **kwargs)
  367. except processutils.ProcessExecutionError:
  368. logger.error("Failed to execute %s" % ' '.join(cmd))
  369. raise
  370. def _pf_interface_up(pf_device):
  371. if 'promisc' in pf_device:
  372. run_ip_config_cmd('ip', 'link', 'set', 'dev', pf_device['name'],
  373. 'promisc', pf_device['promisc'])
  374. logger.info("Bringing up PF: %s" % pf_device['name'])
  375. run_ip_config_cmd('ip', 'link', 'set', 'dev', pf_device['name'], 'up')
  376. def get_vendor_id(ifname):
  377. try:
  378. with open(os.path.join(_SYS_CLASS_NET, ifname, "device/vendor"),
  379. 'r') as f:
  380. out = f.read().strip()
  381. return out
  382. except IOError:
  383. return
  384. def get_pf_pci(pf_name):
  385. pf_pci_path = os.path.join(_SYS_CLASS_NET, pf_name, "device/uevent")
  386. pf_info = get_file_data(pf_pci_path)
  387. pf_pci = re.search(r'PCI_SLOT_NAME=(.*)', pf_info, re.MULTILINE).group(1)
  388. return pf_pci
  389. def get_pf_device_id(pf_name):
  390. pf_device_path = os.path.join(_SYS_CLASS_NET, pf_name, "device/device")
  391. pf_device_id = get_file_data(pf_device_path).strip()
  392. return pf_device_id
  393. def get_vf_pcis_list(pf_name):
  394. vf_pcis_list = []
  395. listOfPfFiles = os.listdir(os.path.join(_SYS_CLASS_NET, pf_name,
  396. "device"))
  397. for pf_file in listOfPfFiles:
  398. if pf_file.startswith("virtfn"):
  399. vf_info = get_file_data(os.path.join(_SYS_CLASS_NET, pf_name,
  400. "device", pf_file, "uevent"))
  401. vf_pcis_list.append(re.search(r'PCI_SLOT_NAME=(.*)',
  402. vf_info, re.MULTILINE).group(1))
  403. return vf_pcis_list
  404. def if_down_interface(device):
  405. logger.info("Running /sbin/ifdown %s" % device)
  406. try:
  407. processutils.execute('/sbin/ifdown', device)
  408. except processutils.ProcessExecutionError:
  409. logger.error("Failed to ifdown %s" % device)
  410. raise
  411. def if_up_interface(device):
  412. logger.info("Running /sbin/ifup %s" % device)
  413. try:
  414. processutils.execute('/sbin/ifup', device)
  415. except processutils.ProcessExecutionError:
  416. logger.error("Failed to ifup %s" % device)
  417. raise
  418. def configure_sriov_vf():
  419. sriov_map = _get_sriov_map()
  420. for item in sriov_map:
  421. if item['device_type'] == 'vf':
  422. pf_name = item['device']['name']
  423. vfid = item['device']['vfid']
  424. base_cmd = ('ip', 'link', 'set', 'dev', pf_name, 'vf', str(vfid))
  425. logger.info("Configuring settings for PF: %s VF :%d VF name : %s"
  426. % (pf_name, vfid, item['name']))
  427. if 'macaddr' in item:
  428. cmd = base_cmd + ('mac', item['macaddr'])
  429. run_ip_config_cmd(*cmd)
  430. if 'vlan_id' in item:
  431. vlan_cmd = base_cmd + ('vlan', str(item['vlan_id']))
  432. if 'qos' in item:
  433. vlan_cmd = vlan_cmd + ('qos', str(item['qos']))
  434. run_ip_config_cmd(*vlan_cmd)
  435. if 'spoofcheck' in item:
  436. cmd = base_cmd + ('spoofchk', item['spoofcheck'])
  437. run_ip_config_cmd(*cmd)
  438. if 'state' in item:
  439. cmd = base_cmd + ('state', item['state'])
  440. run_ip_config_cmd(*cmd)
  441. if 'trust' in item:
  442. cmd = base_cmd + ('trust', item['trust'])
  443. run_ip_config_cmd(*cmd)
  444. if 'promisc' in item:
  445. run_ip_config_cmd('ip', 'link', 'set', 'dev', item['name'],
  446. 'promisc', item['promisc'])
  447. def parse_opts(argv):
  448. parser = argparse.ArgumentParser(
  449. description='Configure SR-IOV PF and VF interfaces using a YAML'
  450. ' config file format.')
  451. parser.add_argument(
  452. '-d', '--debug',
  453. dest="debug",
  454. action='store_true',
  455. help="Print debugging output.",
  456. required=False)
  457. parser.add_argument(
  458. '-v', '--verbose',
  459. dest="verbose",
  460. action='store_true',
  461. help="Print verbose output.",
  462. required=False)
  463. parser.add_argument(
  464. '-n', '--numvfs',
  465. dest="numvfs",
  466. action='store',
  467. help="Provide the numvfs for device in the format <device>:<numvfs>",
  468. required=False)
  469. opts = parser.parse_args(argv[1:])
  470. return opts
  471. def configure_logger(verbose=False, debug=False):
  472. LOG_FORMAT = '[%(asctime)s] [%(levelname)s] %(message)s'
  473. DATE_FORMAT = '%Y/%m/%d %I:%M:%S %p'
  474. log_level = logging.WARN
  475. if debug:
  476. log_level = logging.DEBUG
  477. elif verbose:
  478. log_level = logging.INFO
  479. logging.basicConfig(format=LOG_FORMAT, datefmt=DATE_FORMAT,
  480. level=log_level)
  481. def main(argv=sys.argv):
  482. opts = parse_opts(argv)
  483. configure_logger(opts.verbose, opts.debug)
  484. if opts.numvfs:
  485. if re.match("^\w+:\d+$", opts.numvfs):
  486. device_name, numvfs = opts.numvfs.split(':')
  487. _write_numvfs(device_name, int(numvfs))
  488. else:
  489. logging.error("Invalid arguments for --numvfs %s" % opts.numvfs)
  490. return 1
  491. else:
  492. # Configure the PF's
  493. configure_sriov_pf()
  494. # Configure the VFs
  495. configure_sriov_vf()
  496. if __name__ == '__main__':
  497. sys.exit(main(sys.argv))