diff --git a/os_net_config/sriov_config.py b/os_net_config/sriov_config.py index 2f8b3686..263b717d 100644 --- a/os_net_config/sriov_config.py +++ b/os_net_config/sriov_config.py @@ -252,6 +252,24 @@ def udev_monitor_stop(observer): observer.stop() +def is_partitioned_pf(dev_name): + """Check if any nic-partition(VF) is already used + + Given a PF device, returns True if any VFs of this + device are in-use. + """ + sriov_map = _get_sriov_map() + for config in sriov_map: + devtype = config.get('device_type', None) + if devtype == 'vf': + name = config.get('device', {}).get('name') + vf_name = config.get('name') + if dev_name == name: + logger.warning("%s has VF(%s) used by host" % (name, vf_name)) + return True + return False + + def configure_sriov_pf(execution_from_cli=False, restart_openvswitch=False): observer = udev_monitor_setup() udev_monitor_start(observer) @@ -267,8 +285,9 @@ def configure_sriov_pf(execution_from_cli=False, restart_openvswitch=False): if item.get('link_mode') == "legacy": # Add a udev rule to configure the VF's when PF's are # released by a guest - add_udev_rule_for_legacy_sriov_pf(item['name'], - item['numvfs']) + if not is_partitioned_pf(item['name']): + add_udev_rule_for_legacy_sriov_pf(item['name'], + item['numvfs']) set_numvfs(item['name'], item['numvfs']) vendor_id = get_vendor_id(item['name']) if (item.get('link_mode') == "switchdev" and @@ -281,6 +300,7 @@ def configure_sriov_pf(execution_from_cli=False, restart_openvswitch=False): logger.info("Unbinding %s" % vf_pci) with open(MLNX_UNBIND_FILE_PATH, 'w') as f: f.write(vf_pci) + logger.info("%s: Adding udev rules" % item['name']) # Adding a udev rule to make vf-representors unmanaged by # NetworkManager @@ -451,6 +471,7 @@ def configure_switchdev(pf_name): % (pf_pci, exc)) raise logger.info("Device pci/%s set to switchdev mode." % pf_pci) + # WA to make sure that the uplink_rep is ready after moving to switchdev, # as moving to switchdev will remove the sriov_pf and create uplink # representor, so we need to make sure that uplink representor is ready @@ -473,6 +494,7 @@ def configure_smfs_software_steering(pf_name): 'pci/%s' % pf_pci, 'name', 'flow_steering_mode', 'value', 'smfs', 'cmode', 'runtime') logger.info("Device pci/%s is set to smfs steering mode." % pf_pci) + except processutils.ProcessExecutionError as exc: logger.warning("Could not set pci/%s to smfs steering mode: %s" % (pf_pci, exc)) diff --git a/os_net_config/tests/test_sriov_config.py b/os_net_config/tests/test_sriov_config.py index dccbcd61..b8daeddf 100644 --- a/os_net_config/tests/test_sriov_config.py +++ b/os_net_config/tests/test_sriov_config.py @@ -226,6 +226,52 @@ class TestSriovConfig(base.TestCase): f = open(udev_file.name, 'r') self.assertEqual(exp_udev_content, f.read()) + def setUp_pf_stubs(self, vendor_id="0x8086"): + run_cmd = [] + self.setUp_udev_stubs() + + def run_ip_config_cmd_stub(*args, **kwargs): + run_cmd.append(' '.join(args)) + self.stub_out('os_net_config.sriov_config.run_ip_config_cmd', + run_ip_config_cmd_stub) + + def cleanup_puppet_config_stub(): + return + self.stub_out('os_net_config.sriov_config.cleanup_puppet_config', + cleanup_puppet_config_stub) + + def _wait_for_vf_creation_stub(pf_name, numvfs): + self._save_action('_wait_for_vf_creation_stub') + return + self.stub_out('os_net_config.sriov_config._wait_for_vf_creation', + _wait_for_vf_creation_stub) + + def get_vendor_id_stub(ifname): + return vendor_id + self.stub_out('os_net_config.sriov_config.get_vendor_id', + get_vendor_id_stub) + + def configure_switchdev_stub(pf_name): + self._save_action('configure_switchdev') + return + self.stub_out('os_net_config.sriov_config.configure_switchdev', + configure_switchdev_stub) + + self.set_numvfs = sriov_config.set_numvfs + self.get_numvfs = sriov_config.get_numvfs + + def get_numvfs_stub(*args): + self._save_action('get_numvfs') + return self.get_numvfs(*args) + self.stub_out('os_net_config.sriov_config.get_numvfs', + get_numvfs_stub) + + def set_numvfs_stub(*args): + self._save_action('set_numvfs') + return self.set_numvfs(*args) + self.stub_out('os_net_config.sriov_config.set_numvfs', + set_numvfs_stub) + def test_configure_sriov_pf(self): """Test the numvfs setting for SR-IOV PF @@ -273,6 +319,101 @@ class TestSriovConfig(base.TestCase): self.assertEqual(10, sriov_config.get_numvfs('p2p1')) self.assertEqual(12, sriov_config.get_numvfs('p2p2')) + def test_configure_sriov_pf_nicpart(self): + """Test the udev rules created for legacy mode of SR-IOV PF + + In this case, the VF(s) are already bound/attached + """ + + self.setUp_pf_stubs() + + exp_udev_content = '# This file is autogenerated by os-net-config\n'\ + 'KERNEL=="p2p2", RUN+="/bin/os-net-config-sriov -n %k:12"\n' + exp_actions = [ + 'udev_monitor_setup', + 'udev_monitor_start', + 'set_numvfs', + 'get_numvfs', + '_wait_for_vf_creation_stub', + 'get_numvfs', + 'reload_udev_rules', + 'set_numvfs', + 'get_numvfs', + '_wait_for_vf_creation_stub', + 'get_numvfs', + 'udev_monitor_stop', + ] + + pf_config = [{"device_type": "pf", "name": "p2p1", "numvfs": 10, + "promisc": "on", "link_mode": "legacy"}, + {"device_type": "pf", "name": "p2p2", "numvfs": 12, + "promisc": "off", "link_mode": "legacy"}, + {"device": {"name": "p2p1", "vfid": 0}, + "device_type": "vf", "name": "p2p1v0", "max_tx_rate": 0, + "min_tx_rate": 0, "pci_address": "0000:18:0a.0", + "trust": "on", "spoofcheck": "off"}] + + for ifname in ['p2p1', 'p2p2']: + self._write_numvfs(ifname) + + self._action_order = [] + utils.write_yaml_config(sriov_config._SRIOV_CONFIG_FILE, pf_config) + sriov_config.configure_logger(debug=True) + sriov_config.configure_sriov_pf() + self.assertEqual(exp_actions, self._action_order) + f = open(sriov_config._UDEV_LEGACY_RULE_FILE, 'r') + self.assertEqual(exp_udev_content, f.read()) + self.assertEqual(12, sriov_config.get_numvfs('p2p2')) + + def test_configure_sriov_pf_non_nicpart(self): + """Test the udev rules created for legacy mode of SR-IOV PF + + In this case, the nic partitioned VF(s) are not attached + """ + + self.setUp_pf_stubs() + + exp_udev_content = '# This file is autogenerated by os-net-config\n'\ + 'KERNEL=="p2p1", RUN+="/bin/os-net-config-sriov -n %k:10"\n'\ + 'KERNEL=="p2p2", RUN+="/bin/os-net-config-sriov -n %k:12"\n' + exp_actions = [ + 'udev_monitor_setup', + 'udev_monitor_start', + 'reload_udev_rules', + 'set_numvfs', + 'get_numvfs', + '_wait_for_vf_creation_stub', + 'get_numvfs', + 'reload_udev_rules', + 'set_numvfs', + 'get_numvfs', + '_wait_for_vf_creation_stub', + 'get_numvfs', + 'udev_monitor_stop', + ] + + pf_config = [{"device_type": "pf", "name": "p2p1", "numvfs": 10, + "promisc": "on", "link_mode": "legacy"}, + {"device_type": "pf", "name": "p2p2", "numvfs": 12, + "promisc": "off", "link_mode": "legacy"}, + {"device": {"name": "eno3", "vfid": 1}, + "device_type": "vf", "name": "eno3v0", "max_tx_rate": 0, + "min_tx_rate": 0, "pci_address": "0000:18:0e.1", + "trust": "on", "spoofcheck": "off"}] + + for ifname in ['p2p1', 'p2p2']: + self._write_numvfs(ifname) + + self._action_order = [] + utils.write_yaml_config(sriov_config._SRIOV_CONFIG_FILE, pf_config) + sriov_config.configure_logger(debug=True) + sriov_config.configure_sriov_pf() + self.assertEqual(exp_actions, self._action_order) + f = open(sriov_config._UDEV_LEGACY_RULE_FILE, 'r') + self.assertEqual(exp_udev_content, f.read()) + self.assertEqual(10, sriov_config.get_numvfs('p2p1')) + self.assertEqual(12, sriov_config.get_numvfs('p2p2')) + def test_cleanup_puppet_config_deprecation(self): """Test the cleanup of puppet-tripleo generated config file. diff --git a/os_net_config/tests/test_utils.py b/os_net_config/tests/test_utils.py index 3aac8ab8..b21e7d68 100644 --- a/os_net_config/tests/test_utils.py +++ b/os_net_config/tests/test_utils.py @@ -75,6 +75,11 @@ dpdk { } ''' +udev_content = '# This file is autogenerated by os-net-config\n'\ + 'KERNEL=="eth1", RUN+="/bin/os-net-config-sriov -n %k:10"\n' \ + 'KERNEL=="eth2", RUN+="/bin/os-net-config-sriov -n %k:10"\n' +UDEV_FILE = '/tmp/70-os-net-config-sriov.' + class TestUtils(base.TestCase): @@ -83,6 +88,7 @@ class TestUtils(base.TestCase): rand = str(int(random.random() * 100000)) utils._DPDK_MAPPING_FILE = '/tmp/dpdk_mapping_' + rand + '.yaml' sriov_config._SRIOV_CONFIG_FILE = '/tmp/sriov_config_' + rand + '.yaml' + sriov_config._UDEV_LEGACY_RULE_FILE = UDEV_FILE + rand def tearDown(self): super(TestUtils, self).tearDown() @@ -90,6 +96,8 @@ class TestUtils(base.TestCase): os.remove(utils._DPDK_MAPPING_FILE) if os.path.isfile(sriov_config._SRIOV_CONFIG_FILE): os.remove(sriov_config._SRIOV_CONFIG_FILE) + if os.path.isfile(sriov_config._UDEV_LEGACY_RULE_FILE): + os.remove(sriov_config._UDEV_LEGACY_RULE_FILE) def test_ordered_active_nics(self): @@ -204,6 +212,37 @@ class TestUtils(base.TestCase): 'device': {"name": "eth1", "vfid": 2}}] self.assertListEqual(test_sriov_vf_map, sriov_vf_map) + def test_udev_rule_for_sriov_vf(self): + def get_numvfs_stub(pf_name): + return 10 + self.stub_out('os_net_config.sriov_config.get_numvfs', + get_numvfs_stub) + utils.update_sriov_pf_map('eth1', 10, False) + utils.update_sriov_pf_map('eth2', 10, False) + utils.update_sriov_vf_map('eth1', 2, 'eth1_2') + contents = utils.get_file_data(sriov_config._SRIOV_CONFIG_FILE) + sriov_vf_map = yaml.safe_load(contents) if contents else [] + self.assertEqual(3, len(sriov_vf_map)) + + test_sriov_vf_map = {'device': {'name': 'eth1', 'vfid': 2}, + 'device_type': 'vf', 'max_tx_rate': 0, + 'min_tx_rate': 0, 'name': 'eth1_2'} + udev_file = open(sriov_config._UDEV_LEGACY_RULE_FILE, "w") + udev_file.write(udev_content) + udev_file.close() + + # sriov_config.add_udev_rule_for_legacy_sriov_pf("eth1", 10) + for dev in sriov_vf_map: + if(dev.keys() == test_sriov_vf_map.keys()): + self.assertEqual(test_sriov_vf_map, dev) + + expect = '# This file is autogenerated by os-net-config\n'\ + 'KERNEL=="eth2", RUN+="/bin/os-net-config-sriov -n %k:10"\n' + utils.nicpart_udev_rules_check() + f = open(sriov_config._UDEV_LEGACY_RULE_FILE, 'r') + self.assertEqual(expect, f.read()) + f.close() + def test_update_sriov_vf_map_complete_new(self): utils.update_sriov_vf_map('eth1', 2, 'eth1_2', vlan_id=10, qos=5, spoofcheck="on", trust="on", state="enable", diff --git a/os_net_config/utils.py b/os_net_config/utils.py index 150100c6..7dcd41d7 100644 --- a/os_net_config/utils.py +++ b/os_net_config/utils.py @@ -586,6 +586,32 @@ def _get_vf_name_from_map(pf_name, vfid): return item['name'] +def nicpart_udev_rules_check(): + """Clean-up UDEV rules on initial deployment + + After writing sriov_config.yaml file, clean-up + UDEV rule(s) of PF for which VFs are used by host + """ + if not os.path.exists(sriov_config._UDEV_LEGACY_RULE_FILE): + return + + udev = '^KERNEL=="(.*)", RUN.*' + udev_reg = re.compile(udev, 0) + + with open(sriov_config._UDEV_LEGACY_RULE_FILE, "r") as fp: + rules = fp.readlines() + + with open(sriov_config._UDEV_LEGACY_RULE_FILE, "w") as fp: + for line in rules: + match = udev_reg.match(line) + if match: + dev_name = match.group(1) + if not sriov_config.is_partitioned_pf(dev_name): + fp.write(line) + else: + fp.write(line) + + def _configure_sriov_config_service(): """Generate the sriov_config.service @@ -607,6 +633,7 @@ def configure_sriov_pfs(execution_from_cli=False, restart_openvswitch=False): def configure_sriov_vfs(): logger.info("Configuring VFs now") sriov_config.configure_sriov_vf() + nicpart_udev_rules_check() def get_vf_devname(pf_name, vfid):