Use quota fields as per services enabled

There are few quotas such as fixed-ips, floating-ips, sec-group etc..
which can be managed by neutron as well as nova-network based on the service
enablement.

Use neutron to manage these quota fields if neutron is enabled or else
use nova.
Added consts.py file to keep track of all the constants required for KB.

Change-Id: I7fed28fdc27b5f090e12baf2a25edbc8592f2df1

kingbird/common/consts.py

@@ -0,0 +1,37 @@
+# Copyright (c) 2016 Ericsson AB.
+
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+NOVA_QUOTA_FIELDS = ("metadata_items",
+                     "cores",
+                     "instances",
+                     "ram",
+                     "key_pairs",
+                     "floating_ips",
+                     "fixed_ips",
+                     "security_groups",
+                     "security_group_rules",)
+
+CINDER_QUOTA_FIELDS = ("volumes",
+                       "snapshots",
+                       "gigabytes",
+                       "backups",)
+
+NEUTRON_QUOTA_FIELDS = ("network",
+                        "subnet",
+                        "port",
+                        "router",
+                        "floatingip",
+                        "security_group",
+                        "security_group_rule",
+                        )

kingbird/drivers/README.rst

@@ -2,5 +2,4 @@
 OpenStack Drivers
 ================================
 
-Driver for openstack communication based on python-openstackclient.
+Driver for openstack communication based on python native clients.
-Implements nova, cinder & neutron clients based on the same.

kingbird/drivers/openstack/cinder_v2.py

@@ -14,6 +14,7 @@ from oslo_log import log
 
 from cinderclient.v2 import client as ci_client
 
+from kingbird.common import exceptions
 from kingbird.drivers import base
 
 LOG = log.getLogger(__name__)
@@ -22,12 +23,15 @@ LOG = log.getLogger(__name__)
 class CinderClient(base.DriverBase):
     '''Cinder V2 driver.'''
     def __init__(self, region, **kwargs):
-        self.cinder_client = ci_client.Client(
+        try:
-            auth_url=kwargs['auth_url'],
+            self.cinder_client = ci_client.Client(
-            username=kwargs['user_name'],
+                auth_url=kwargs['auth_url'],
-            api_key=kwargs['password'],
+                username=kwargs['user_name'],
-            tenant_id=kwargs['tenant_id'],
+                api_key=kwargs['password'],
-            region_name=region)
+                tenant_id=kwargs['tenant_id'],
+                region_name=region)
+        except exceptions.HttpException:
+            raise
 
     def get_resource_usages(self, project_id):
         '''Calcualte resources usage and return the dict'''

kingbird/drivers/openstack/keystone_v3.py

@@ -30,17 +30,32 @@ class KeystoneClient(base.DriverBase):
     '''Keystone V3 driver.'''
 
     def __init__(self, **kwargs):
-        auth = v3.Password(
+        try:
-            auth_url=kwargs['auth_url'],
+            auth = v3.Password(
-            username=kwargs['user_name'],
+                auth_url=kwargs['auth_url'],
-            password=kwargs['password'],
+                username=kwargs['user_name'],
-            project_name=kwargs['tenant_name'])
+                password=kwargs['password'],
-        sess = session.Session(auth=auth)
+                project_name=kwargs['tenant_name'],
-        self.keystone_client = client.Client(session=sess)
+                project_domain_name=kwargs['project_domain'],
+                user_domain_name=kwargs['user_domain'])
+            sess = session.Session(auth=auth)
+            self.keystone_client = client.Client(session=sess)
+            self.services_list = self.keystone_client.services.list()
+        except exceptions.HttpException:
+            raise
 
     def get_enabled_projects(self):
         try:
             return [current_project.id for current_project in
                     self.keystone.projects.list() if current_project.enabled]
-        except exceptions.HttpException as ex:
+        except exceptions.HttpException:
-            raise ex
+            raise
+
+    def is_service_enabled(self, service):
+        try:
+            for current_service in self.services_list:
+                if service in current_service.type:
+                    return True
+            return False
+        except exceptions.HttpException:
+            raise

kingbird/drivers/openstack/neutron_v2.py

@@ -12,6 +12,7 @@
 
 from oslo_log import log
 
+from kingbird.common import exceptions
 from kingbird.drivers import base
 
 from neutronclient.neutron import client
@@ -23,13 +24,17 @@ API_VERSION = '2.0'
 class NeutronClient(base.DriverBase):
     '''Neutron V2 driver.'''
     def __init__(self, region, **kwargs):
-        self.neutron_client = client.Client(
+        try:
-            API_VERSION,
+            self.neutron_client = client.Client(
-            username=kwargs['user_name'],
+                API_VERSION,
-            password=kwargs['password'],
+                username=kwargs['user_name'],
-            tenant_name=kwargs['tenant_name'],
+                password=kwargs['password'],
-            auth_url=kwargs['auth_url'],
+                tenant_name=kwargs['tenant_name'],
-            region_name=region)
+                auth_url=kwargs['auth_url'],
+                region_name=region)
+            self.extension_list = self.neutron_client.list_extensions()
+        except exceptions.HttpException:
+            raise
 
     def get_resource_usages(self, project_id):
         '''Calcualte resources usage and return the dict'''
@@ -42,3 +47,9 @@ class NeutronClient(base.DriverBase):
     def delete_quota_limits(self, project_id):
         '''Delete/Reset the limits'''
         pass
+
+    def is_extension_supported(self, extension):
+        for current_extension in self.extension_list['extensions']:
+            if extension in current_extension['alias']:
+                return True
+        return False

kingbird/drivers/openstack/nova_v2.py

@@ -12,6 +12,8 @@
 
 from oslo_log import log
 
+from kingbird.common import consts
+from kingbird.common import exceptions
 from kingbird.drivers import base
 
 from novaclient import client as nv_client
@@ -22,11 +24,16 @@ API_VERSION = '2.1'
 
 class NovaClient(base.DriverBase):
     '''Nova V2.1 driver.'''
-    def __init__(self, region, **kwargs):
+    def __init__(self, region, disabled_quotas, **kwargs):
-        self.nova_client = nv_client.Client(
+        try:
-            API_VERSION, kwargs['user_name'],
+            self.nova_client = nv_client.Client(
-            kwargs['password'], kwargs['tenant_name'],
+                API_VERSION, kwargs['user_name'],
-            kwargs['auth_url'], region_name=region)
+                kwargs['password'], kwargs['tenant_name'],
+                kwargs['auth_url'], region_name=region)
+            self.enabled_quotas = list(set(consts.NOVA_QUOTA_FIELDS) -
+                                       set(disabled_quotas))
+        except exceptions.HttpException:
+            raise
 
     def get_resource_usages(self, project_id):
         '''Calcualte resources usage and return the dict'''

kingbird/drivers/openstack/sdk.py

@@ -17,6 +17,7 @@ import collections
 
 from oslo_log import log
 
+from kingbird.common import consts
 from kingbird.common import exceptions
 from kingbird.common.i18n import _
 from kingbird.common.i18n import _LE
@@ -33,18 +34,19 @@ LOG = log.getLogger(__name__)
 
 admin_creds_opts = [
     cfg.StrOpt('auth_url',
-               default='http://127.0.0.1:5000/v3',
                help='keystone authorization url'),
     cfg.StrOpt('identity_url',
-               default='http://127.0.0.1:35357/v3',
                help='keystone service url'),
     cfg.StrOpt('admin_username',
+               default='admin',
                help='username of admin account, needed when'
                     ' auto_refresh_endpoint set to True'),
     cfg.StrOpt('admin_password',
+               default='admin',
                help='password of admin account, needed when'
                     ' auto_refresh_endpoint set to True'),
     cfg.StrOpt('admin_tenant',
+               default='admin',
                help='tenant name of admin account, needed when'
                     ' auto_refresh_endpoint set to True'),
     cfg.StrOpt('admin_tenant_id',
@@ -71,6 +73,21 @@ class OpenStackDriver(object):
     def __init__(self, region_name):
         # Check if objects are cached and try to use those
         self.region_name = region_name
+        self.services_list = []
+        admin_kwargs = {
+            'user_name': cfg.CONF.admin_creds.admin_username,
+            'password': cfg.CONF.admin_creds.admin_password,
+            'tenant_name': cfg.CONF.admin_creds.admin_tenant,
+            'auth_url': cfg.CONF.admin_creds.auth_url,
+            'tenant_id': cfg.CONF.admin_creds.admin_tenant_id,
+            'project_domain':
+                cfg.CONF.admin_creds.admin_tenant_domain_name,
+            'user_domain': cfg.CONF.admin_creds.admin_user_domain_name
+            }
+        if 'keystone' in OpenStackDriver.os_clients_dict:
+            self.keystone_client = OpenStackDriver.os_clients_dict['keystone']
+        else:
+            self.keystone_client = KeystoneClient(**admin_kwargs)
         if region_name in OpenStackDriver.os_clients_dict:
             LOG.info(_LI('Using cached OS client objects'))
             self.nova_client = OpenStackDriver.os_clients_dict[
@@ -82,16 +99,11 @@ class OpenStackDriver(object):
         else:
             # Create new objects and cache them
             LOG.debug(_("Creating fresh OS Clients objects"))
-            admin_kwargs = {
-                'user_name': cfg.CONF.admin_creds.admin_username,
-                'password': cfg.CONF.admin_creds.admin_password,
-                'tenant_name': cfg.CONF.admin_creds.admin_tenant,
-                'auth_url': cfg.CONF.admin_creds.auth_url,
-                'tenant_id': cfg.CONF.admin_creds.admin_tenant_id
-                }
-            self.nova_client = NovaClient(region_name, **admin_kwargs)
             self.cinder_client = CinderClient(region_name, **admin_kwargs)
             self.neutron_client = NeutronClient(region_name, **admin_kwargs)
+            self.disabled_quotas = self._get_disabled_quotas(region_name)
+            self.nova_client = NovaClient(region_name, self.disabled_quotas,
+                                          **admin_kwargs)
             OpenStackDriver.os_clients_dict[
                 region_name] = collections.defaultdict(dict)
             OpenStackDriver.os_clients_dict[region_name][
@@ -100,11 +112,6 @@ class OpenStackDriver(object):
                 'cinder'] = self.cinder_client
             OpenStackDriver.os_clients_dict[region_name][
                 'neutron'] = self.neutron_client
-        if 'keystone' in OpenStackDriver.os_clients_dict:
-            self.keystone_client = OpenStackDriver.os_clients_dict['keystone']
-        else:
-            self.keystone_client = KeystoneClient(**admin_kwargs)
-            OpenStackDriver.os_clients_dict['keystone'] = self.keystone_client
 
     def get_enabled_projects(self):
         try:
@@ -152,3 +159,27 @@ class OpenStackDriver(object):
             del OpenStackDriver.os_clients_dict[self.region_name]
         except Exception as exception:
             LOG.error(_LE('Error Occurred: %s'), exception.message)
+
+    def _get_disabled_quotas(self, region):
+        disabled_quotas = []
+        # Neutron
+        if not self.keystone_client.is_service_enabled('network'):
+            disabled_quotas.extend(consts.NEUTRON_QUOTA_FIELDS)
+        else:
+            # Remove the nova network quotas
+            disabled_quotas.extend(['floating_ips', 'fixed_ips'])
+            if self.neutron_client.is_extension_supported('security-group'):
+                # If Neutron security group is supported, disable Nova quotas
+                disabled_quotas.extend(['security_groups',
+                                        'security_group_rules'])
+            else:
+                # If Nova security group is used, disable Neutron quotas
+                disabled_quotas.extend(['security_group',
+                                        'security_group_rule'])
+            try:
+                if not self.neutron_client.is_extension_supported('quotas'):
+                    disabled_quotas.extend(consts.NEUTRON_QUOTA_FIELDS)
+            except Exception:
+                LOG.exception("There was an error checking if the Neutron "
+                              "quotas extension is enabled.")
+        return disabled_quotas

kingbird/tests/unit/drivers/test_keystone_v3.py

@@ -12,8 +12,6 @@
 
 import mock
 
-import keystoneclient
-
 from kingbird.drivers.openstack import keystone_v3
 from kingbird.tests import base
 from kingbird.tests import utils
@@ -22,20 +20,28 @@ FAKE_ADMIN_CREDS = {
     'user_name': 'fake_user',
     'password': 'pass1234',
     'tenant_name': 'test_tenant',
-    'auth_url': 'http://127.0.0.1:5000/v3'
+    'auth_url': 'http://127.0.0.1:5000/v3',
+    'project_domain': 'domain1',
+    'user_domain': 'user_dom'
     }
 
+FAKE_SERVICE = [
+    'endpoint_volume',
+    'endpoint_network'
+    ]
+
 
 class TestKeystoneClient(base.KingbirdTestCase):
     def setUp(self):
         super(TestKeystoneClient, self).setUp()
         self.ctx = utils.dummy_context()
 
-    def test_init(self):
+    @mock.patch.object(keystone_v3, 'KeystoneClient')
+    def test_init(self, mock_keystone):
+        mock_keystone().services_list = FAKE_SERVICE
         key_client = keystone_v3.KeystoneClient(**FAKE_ADMIN_CREDS)
-        self.assertIsNotNone(key_client.keystone_client)
+        self.assertEqual(key_client.services_list,
-        self.assertIsInstance(key_client.keystone_client,
+                         FAKE_SERVICE)
-                              keystoneclient.v3.client.Client)
 
     @mock.patch.object(keystone_v3, 'KeystoneClient')
     def test_get_enabled_projects(self, mock_key_client):
@@ -46,3 +52,10 @@ class TestKeystoneClient(base.KingbirdTestCase):
         except Exception:
             raised = True
         self.assertFalse(raised, 'get_enabled_projects Failed')
+
+    @mock.patch.object(keystone_v3, 'KeystoneClient')
+    def test_is_service_enabled(self, mock_keystone):
+        key_client = keystone_v3.KeystoneClient(**FAKE_ADMIN_CREDS)
+        mock_keystone().is_service_enabled.return_value = True
+        network_enabled = key_client.is_service_enabled('network')
+        self.assertEqual(network_enabled, True)

kingbird/tests/unit/drivers/test_neutron_v2.py

@@ -10,7 +10,7 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-import neutronclient
+import mock
 
 from kingbird.drivers.openstack import neutron_v2
 from kingbird.tests import base
@@ -23,18 +23,32 @@ FAKE_ADMIN_CREDS = {
     'auth_url': 'http://127.0.0.1:5000/v3'
     }
 
+FAKE_EXTENSIONS = {
+    'extensions': ['fake_extension1',
+                   'fake_extension2']
+    }
+
 
 class TestNeutronClient(base.KingbirdTestCase):
     def setUp(self):
         super(TestNeutronClient, self).setUp()
         self.ctx = utils.dummy_context()
 
-    def test_init(self):
+    @mock.patch.object(neutron_v2, 'NeutronClient')
+    def test_init(self, mock_neutron):
+        mock_neutron().extension_list = FAKE_EXTENSIONS
         neutron_client = neutron_v2.NeutronClient('fake_region',
                                                   **FAKE_ADMIN_CREDS)
-        self.assertIsNotNone(neutron_client)
+        self.assertEqual(FAKE_EXTENSIONS,
-        self.assertIsInstance(neutron_client.neutron_client,
+                         neutron_client.extension_list)
-                              neutronclient.v2_0.client.Client)
+
+    @mock.patch.object(neutron_v2, 'NeutronClient')
+    def test_is_extension_supported(self, mock_neutron):
+        neutron_client = neutron_v2.NeutronClient('fake_region',
+                                                  **FAKE_ADMIN_CREDS)
+        mock_neutron().is_extension_supported.return_value = True
+        extension_enabled = neutron_client.is_extension_supported('quotas')
+        self.assertEqual(extension_enabled, True)
 
     def test_get_resource_usages(self):
         pass

kingbird/tests/unit/drivers/test_nova_v2.py

@@ -12,6 +12,7 @@
 
 import novaclient
 
+from kingbird.common import consts
 from kingbird.drivers.openstack import nova_v2
 from kingbird.tests import base
 from kingbird.tests import utils
@@ -22,6 +23,7 @@ FAKE_ADMIN_CREDS = {
     'tenant_name': 'test_tenant',
     'auth_url': 'http://127.0.0.1:5000/v3'
     }
+DISABLED_QUOTAS = ["floating_ips", "fixed_ips", "security_groups"]
 
 
 class TestNovaClient(base.KingbirdTestCase):
@@ -30,8 +32,12 @@ class TestNovaClient(base.KingbirdTestCase):
         self.ctx = utils.dummy_context()
 
     def test_init(self):
-        nv_client = nova_v2.NovaClient('fake_region', **FAKE_ADMIN_CREDS)
+        nv_client = nova_v2.NovaClient('fake_region', DISABLED_QUOTAS,
+                                       **FAKE_ADMIN_CREDS)
         self.assertIsNotNone(nv_client)
+        expected_quotas = list(set(consts.NOVA_QUOTA_FIELDS) -
+                               set(DISABLED_QUOTAS))
+        self.assertEqual(nv_client.enabled_quotas, expected_quotas)
         self.assertIsInstance(nv_client.nova_client,
                               novaclient.v2.client.Client)
 

kingbird/tests/unit/drivers/test_openstack_driver.py

@@ -17,6 +17,17 @@ from kingbird.tests import base
 from kingbird.tests import utils
 
 
+class FakeService(object):
+
+    '''Fake service class used to test service enable testcase
+
+    '''
+
+    def __init__(self, type_service, name):
+        self.type = type_service
+        self.name = name
+
+
 class TestOpenStackDriver(base.KingbirdTestCase):
     def setUp(self):
         super(TestOpenStackDriver, self).setUp()
@@ -132,3 +143,18 @@ class TestOpenStackDriver(base.KingbirdTestCase):
         self.assertEqual(os_driver_2.cinder_client, os_driver_4.cinder_client)
         self.assertEqual(os_driver_2.neutron_client,
                          os_driver_4.neutron_client)
+
+    @mock.patch.object(sdk, 'KeystoneClient')
+    @mock.patch.object(sdk, 'NovaClient')
+    @mock.patch.object(sdk, 'NeutronClient')
+    @mock.patch.object(sdk, 'CinderClient')
+    def test_get_disabled_quotas(self, mock_cinder_client,
+                                 mock_network_client, mock_nova_client,
+                                 mock_keystone_client):
+        input_disable_quotas = ["floating_ips", "security_groups",
+                                "security_group_rules"]
+        os_driver = sdk.OpenStackDriver('fake_region9')
+        output_disabled_quotas = os_driver._get_disabled_quotas('fake_region9')
+        self.assertIn(input_disable_quotas[0], output_disabled_quotas)
+        self.assertIn(input_disable_quotas[1], output_disabled_quotas)
+        self.assertIn(input_disable_quotas[2], output_disabled_quotas)