Improve NeutronFixture and remove unncessary stubbing

With a small improvement on NeutronFixture that allows creating more
than two ports we can let our tests to exercise and therefore cover
most of the neutronv2/api code as well. To do that this path removes a
lot of fake_network.set_stub_network_methods() calls from the tests.
The remaining calls are not that trivial to remove so those are left in
for a later patch.

The numa functional tests uses the libvirt fixture and during the guest
config xml generation it tries to instantiate OVOs from os_vif. To make
this work the libvirt fixture has to make sure that the
os_vif.initialize() is called as that call registers the OVOs.

Change-Id: I1dbccc2be6ba79bf267edac9208c80e187e6256a

nova/tests/fixtures.py

@@ -22,6 +22,7 @@ from contextlib import contextmanager
 import copy
 import logging as std_logging
 import os
+import random
 import warnings
 
 import fixtures
@@ -1121,6 +1122,7 @@ class NeutronFixture(fixtures.Fixture):
         'admin_state_up': True,
         'tenant_id': tenant_id,
         'id': '3cb9bc59-5699-4588-a4b1-b87f96708bc6',
+        'shared': False,
     }
     subnet_1 = {
         'name': 'private-subnet',
@@ -1156,7 +1158,8 @@ class NeutronFixture(fixtures.Fixture):
                 'subnet_id': subnet_1['id']
             }
         ],
-        'tenant_id': tenant_id
+        'tenant_id': tenant_id,
+        'binding:vif_type': 'ovs'
     }
 
     port_2 = {
@@ -1171,7 +1174,8 @@ class NeutronFixture(fixtures.Fixture):
                 'subnet_id': subnet_1['id']
             }
         ],
-        'tenant_id': tenant_id
+        'tenant_id': tenant_id,
+        'binding:vif_type': 'ovs'
     }
 
     nw_info = [{
@@ -1235,10 +1239,6 @@ class NeutronFixture(fixtures.Fixture):
     def setUp(self):
         super(NeutronFixture, self).setUp()
 
-        self.test.stub_out(
-            'nova.network.neutronv2.api.API.'
-            'validate_networks',
-            lambda *args, **kwargs: 1)
         self.test.stub_out(
             'nova.network.neutronv2.api.API.setup_networks_on_host',
             lambda *args, **kwargs: None)
@@ -1302,6 +1302,9 @@ class NeutronFixture(fixtures.Fixture):
         networks = copy.deepcopy(self._networks)
         if 'id' in _params:
             networks = [x for x in networks if x['id'] in _params['id']]
+        if 'shared' in _params:
+            networks = [x for x in networks if x['shared'] ==
+                        _params['shared']]
         return {'networks': networks}
 
     def list_ports(self, retrieve_all=True, **_params):
@@ -1314,8 +1317,23 @@ class NeutronFixture(fixtures.Fixture):
         return copy.deepcopy({'floatingips': self._floatingips})
 
     def create_port(self, body=None):
-        self._ports.append(copy.deepcopy(NeutronFixture.port_2))
+        if self._get_first_id_match(NeutronFixture.port_2['id'],
-        return copy.deepcopy({'port': NeutronFixture.port_2})
+                                    self._ports) is None:
+            # we need the double copy as port_2 is a class variable but
+            # self._ports is an instance variable
+            new_port = copy.deepcopy(NeutronFixture.port_2)
+            self._ports.append(new_port)
+        else:
+            new_port = copy.deepcopy(NeutronFixture.port_2)
+            # we need truly random uuids instead of named sentinels as some
+            # tests needs more than 3 ports
+            new_port.update({
+                'id': str(uuidutils.generate_uuid()),
+                'mac_address': '00:' + ':'.join(
+                    ['%02x' % random.randint(0, 255) for _ in range(5)]),
+            })
+            self._ports.append(new_port)
+        return {'port': copy.deepcopy(new_port)}
 
     def update_port(self, port_id, body=None):
         new_port = self._get_first_id_match(port_id, self._ports)
@@ -1326,6 +1344,10 @@ class NeutronFixture(fixtures.Fixture):
 
         return {'port': new_port}
 
+    def show_quota(self, project_id):
+        # unlimited quota
+        return {'quota': {'port': -1}}
+
 
 class _NoopConductor(object):
     def __getattr__(self, key):

nova/tests/functional/db/test_archive.py

@@ -26,7 +26,6 @@ from nova import context
 from nova.db import api as db
 from nova.db.sqlalchemy import api as sqlalchemy_api
 from nova.tests.functional import test_servers
-from nova.tests.unit import fake_network
 
 
 class TestDatabaseArchive(test_servers.ServersTestBase):
@@ -59,10 +58,6 @@ class TestDatabaseArchive(test_servers.ServersTestBase):
 
         :returns: created server (dict)
         """
-        # TODO(mriedem): We should pull this up into the parent class so we
-        # don't have so much copy/paste in these functional tests.
-        fake_network.set_stub_network_methods(self)
-
         # Create a server
         server = self._build_minimal_create_server_request()
         created_server = self.api.post_server({'server': server})

nova/tests/functional/db/test_request_spec.py

@@ -20,7 +20,6 @@ from nova.objects import request_spec
 from nova import test
 from nova.tests import fixtures
 from nova.tests.functional import integrated_helpers
-from nova.tests.unit import fake_network
 from nova.tests.unit import fake_request_spec
 
 
@@ -98,11 +97,11 @@ class RequestSpecInstanceMigrationTestCase(
     _image_ref_parameter = 'imageRef'
     _flavor_ref_parameter = 'flavorRef'
 
+    USE_NEUTRON = True
+
     def setUp(self):
         super(RequestSpecInstanceMigrationTestCase, self).setUp()
-
         self.context = context.get_admin_context()
-        fake_network.set_stub_network_methods(self)
 
     def _create_instances(self, old=2, total=5):
         request = self._build_minimal_create_server_request()

nova/tests/functional/libvirt/test_numa_servers.py

@@ -27,12 +27,10 @@ from nova import test
 from nova.tests import fixtures as nova_fixtures
 from nova.tests.functional.api import client
 from nova.tests.functional.test_servers import ServersTestBase
-from nova.tests.unit import fake_network
 from nova.tests.unit.virt.libvirt import fake_imagebackend
 from nova.tests.unit.virt.libvirt import fake_libvirt_utils
 from nova.tests.unit.virt.libvirt import fakelibvirt
 
-import os_vif
 
 CONF = cfg.CONF
 LOG = logging.getLogger(__name__)
@@ -332,8 +330,6 @@ class NUMAAffinityNeutronFixture(nova_fixtures.NeutronFixture):
 
 class NUMAServersWithNetworksTest(NUMAServersTestBase):
 
-    USE_NEUTRON = True
-
     def setUp(self):
         # We need to enable neutron in this one
         self.flags(physnets=['foo', 'bar'], group='neutron')
@@ -344,18 +340,15 @@ class NUMAServersWithNetworksTest(NUMAServersTestBase):
 
         super(NUMAServersWithNetworksTest, self).setUp()
 
-        # NOTE(mriedem): Unset the stub methods so we actually run our
+        # The ultimate base class _IntegratedTestBase uses NeutronFixture but
-        # neutronv2/api code and populate the net attributes on the
+        # we need a bit more intelligent neutron for these tests. Applying the
-        # network model.
+        # new fixture here means that we re-stub what the previous neutron
-        fake_network.unset_stub_network_methods(self)
+        # fixture already stubbed.
-
+        self.neutron = self.useFixture(NUMAAffinityNeutronFixture(self))
-        self.neutron_fixture = self.useFixture(
-            NUMAAffinityNeutronFixture(self))
 
         _p = mock.patch('nova.virt.libvirt.host.Host.get_connection')
         self.mock_conn = _p.start()
         self.addCleanup(_p.stop)
-        os_vif.initialize()
 
     def _test_create_server_with_networks(self, flavor_id, networks):
         host_info = fakelibvirt.NUMAHostInfo(cpu_nodes=2, cpu_sockets=1,

nova/tests/functional/libvirt/test_pci_sriov_servers.py

@@ -22,7 +22,6 @@ from nova.objects import fields
 from nova import test
 from nova.tests import fixtures as nova_fixtures
 from nova.tests.functional.test_servers import ServersTestBase
-from nova.tests.unit import fake_network
 from nova.tests.unit.virt.libvirt import fake_libvirt_utils
 from nova.tests.unit.virt.libvirt import fakelibvirt
 
@@ -97,8 +96,6 @@ class SRIOVServersTest(ServersTestBase):
             self.compute = self.start_service('compute', host='test_compute0')
             self.compute_started = True
 
-        fake_network.set_stub_network_methods(self)
-
         # Create server
         good_server = self._build_server(flavor_id)
 

nova/tests/functional/libvirt/test_rt_servers.py

@@ -18,7 +18,6 @@ import mock
 
 from nova.tests.functional.api import client
 from nova.tests.functional.test_servers import ServersTestBase
-from nova.tests.unit import fake_network
 from nova.tests.unit.virt.libvirt import fake_imagebackend
 from nova.tests.unit.virt.libvirt import fake_libvirt_utils
 from nova.tests.unit.virt.libvirt import fakelibvirt
@@ -79,7 +78,6 @@ class RealTimeServersTest(ServersTestBase):
         with mock.patch('nova.virt.libvirt.host.Host.get_connection',
                         return_value=fake_connection):
             self.compute = self.start_service('compute', host='test_compute0')
-            fake_network.set_stub_network_methods(self)
 
             flavor = self._create_flavor(extra_spec={
                 'hw:cpu_realtime': 'yes',

nova/tests/functional/test_instance_actions.py

@@ -14,7 +14,6 @@
 
 from nova.tests.functional.api import client
 from nova.tests.functional import test_servers
-from nova.tests.unit import fake_network
 
 
 class InstanceActionsTestV2(test_servers.ServersTestBase):
@@ -28,9 +27,6 @@ class InstanceActionsTestV2(test_servers.ServersTestBase):
 
         :returns: created server (dict)
         """
-        # TODO(mriedem): We should pull this up into the parent class so we
-        # don't have so much copy/paste in these functional tests.
-        fake_network.set_stub_network_methods(self)
 
         # Create a server
         server = self._build_minimal_create_server_request()

nova/tests/functional/test_legacy_v2_compatible_wrapper.py

@@ -18,7 +18,6 @@ from nova.api.openstack import compute
 from nova.api.openstack import wsgi
 from nova.tests.functional.api import client
 from nova.tests.functional import test_servers
-from nova.tests.unit import fake_network
 
 
 class LegacyV2CompatibleTestBase(test_servers.ServersTestBase):
@@ -50,7 +49,6 @@ class LegacyV2CompatibleTestBase(test_servers.ServersTestBase):
         self.assertNotIn('type', response.body["keypair"])
 
     def test_request_with_pattern_properties_check(self):
-        fake_network.set_stub_network_methods(self)
         server = self._build_minimal_create_server_request()
         post = {'server': server}
         created_server = self.api.post_server(post)
@@ -60,7 +58,6 @@ class LegacyV2CompatibleTestBase(test_servers.ServersTestBase):
         self.assertEqual(response, {'a': 'b'})
 
     def test_request_with_pattern_properties_with_avoid_metadata(self):
-        fake_network.set_stub_network_methods(self)
         server = self._build_minimal_create_server_request()
         post = {'server': server}
         created_server = self.api.post_server(post)

nova/tests/functional/test_multiattach.py

@@ -36,7 +36,6 @@ class TestMultiattachVolumes(integrated_helpers._IntegratedTestBase,
         self.useFixture(nova_fixtures.AllServicesCurrent())
         super(TestMultiattachVolumes, self).setUp()
         self.useFixture(nova_fixtures.CinderFixtureNewAttachFlow(self))
-        self.useFixture(nova_fixtures.NeutronFixture(self))
 
     def test_boot_from_volume_and_attach_to_second_server(self):
         """This scenario creates a server from the multiattach volume, waits

nova/tests/functional/test_server_group.py

@@ -25,7 +25,6 @@ from nova import test
 from nova.tests import fixtures as nova_fixtures
 from nova.tests.functional.api import client
 from nova.tests.functional import integrated_helpers
-from nova.tests.unit import fake_network
 from nova.tests.unit import policy_fixture
 from nova.virt import fake
 
@@ -149,7 +148,6 @@ class ServerGroupTestV21(ServerGroupTestBase):
         fake.set_nodes(['host2'])
         self.addCleanup(fake.restore_nodes)
         self.compute2 = self.start_service('compute', host='host2')
-        fake_network.set_stub_network_methods(self)
 
     def test_get_no_groups(self):
         groups = self.api.get_server_groups()

nova/tests/functional/test_servers.py

@@ -38,7 +38,6 @@ from nova.tests.functional.api import client
 from nova.tests.functional import integrated_helpers
 from nova.tests.unit.api.openstack import fakes
 from nova.tests.unit import fake_block_device
-from nova.tests.unit import fake_network
 from nova.tests.unit import fake_notifier
 import nova.tests.unit.image.fake
 from nova.tests import uuidsentinel as uuids
@@ -70,13 +69,11 @@ class ServersTestBase(integrated_helpers._IntegratedTestBase):
     _return_resv_id_parameter = 'return_reservation_id'
     _min_count_parameter = 'min_count'
 
+    USE_NEUTRON = True
+
     def setUp(self):
         self.computes = {}
         super(ServersTestBase, self).setUp()
-        # The network service is called as part of server creates but no
-        # networks have been populated in the db, so stub the methods.
-        # The networks aren't relevant to what is being tested.
-        fake_network.set_stub_network_methods(self)
         self.conductor = self.start_service(
             'conductor', manager='nova.conductor.manager.ConductorManager')
 
@@ -1046,7 +1043,6 @@ class ServerTestV220(ServersTestBase):
     def setUp(self):
         super(ServerTestV220, self).setUp()
         self.api.microversion = '2.20'
-        fake_network.set_stub_network_methods(self)
         self.ctxt = context.get_admin_context()
 
     def _create_server(self):

nova/tests/unit/test_notifications.py

@@ -54,13 +54,11 @@ class NotificationsTestCase(test.TestCase):
 
         self.stub_out('nova.network.api.API.get_instance_nw_info',
                 fake_get_nw_info)
-        fake_network.set_stub_network_methods(self)
 
         fake_notifier.stub_notifier(self)
         self.addCleanup(fake_notifier.reset)
 
-        self.flags(network_manager='nova.network.manager.FlatManager',
+        self.flags(host='testhost')
-                   host='testhost')
         self.flags(notify_on_state_change="vm_and_task_state",
                    group='notifications')
 

nova/tests/unit/virt/libvirt/fakelibvirt.py

@@ -1573,3 +1573,10 @@ class FakeLibvirtFixture(fixtures.Fixture):
             self.useFixture(fixtures.MonkeyPatch(
                 'nova.virt.libvirt.vif.LibvirtGenericVIFDriver._plug_os_vif',
                 lambda *a, **kw: None))
+
+        # os_vif.initialize is typically done in nova-compute startup
+        # even if we are not planning to plug anything with os_vif in the test
+        # we still need the object model initialized to be able to generate
+        # guest config xml properly
+        import os_vif
+        os_vif.initialize()