From 5285612606d2e71b30d357bec37039b9e5b2be43 Mon Sep 17 00:00:00 2001 From: Ashish Singh Date: Tue, 9 Feb 2016 17:28:21 +0530 Subject: [PATCH] Use quota fields as per services enabled There are few quotas such as fixed-ips, floating-ips, sec-group etc.. which can be managed by neutron as well as nova-network based on the service enablement. Use neutron to manage these quota fields if neutron is enabled or else use nova. Added consts.py file to keep track of all the constants required for KB. Change-Id: I7fed28fdc27b5f090e12baf2a25edbc8592f2df1 --- kingbird/common/consts.py | 37 +++++++++++ kingbird/drivers/README.rst | 3 +- kingbird/drivers/openstack/cinder_v2.py | 16 +++-- kingbird/drivers/openstack/keystone_v3.py | 33 +++++++--- kingbird/drivers/openstack/neutron_v2.py | 25 +++++--- kingbird/drivers/openstack/nova_v2.py | 17 ++++-- kingbird/drivers/openstack/sdk.py | 61 ++++++++++++++----- .../tests/unit/drivers/test_keystone_v3.py | 27 +++++--- .../tests/unit/drivers/test_neutron_v2.py | 24 ++++++-- kingbird/tests/unit/drivers/test_nova_v2.py | 8 ++- .../unit/drivers/test_openstack_driver.py | 26 ++++++++ 11 files changed, 220 insertions(+), 57 deletions(-) create mode 100644 kingbird/common/consts.py diff --git a/kingbird/common/consts.py b/kingbird/common/consts.py new file mode 100644 index 0000000..5bb867c --- /dev/null +++ b/kingbird/common/consts.py @@ -0,0 +1,37 @@ +# Copyright (c) 2016 Ericsson AB. + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +NOVA_QUOTA_FIELDS = ("metadata_items", + "cores", + "instances", + "ram", + "key_pairs", + "floating_ips", + "fixed_ips", + "security_groups", + "security_group_rules",) + +CINDER_QUOTA_FIELDS = ("volumes", + "snapshots", + "gigabytes", + "backups",) + +NEUTRON_QUOTA_FIELDS = ("network", + "subnet", + "port", + "router", + "floatingip", + "security_group", + "security_group_rule", + ) diff --git a/kingbird/drivers/README.rst b/kingbird/drivers/README.rst index 0a3bd82..fce9934 100644 --- a/kingbird/drivers/README.rst +++ b/kingbird/drivers/README.rst @@ -2,5 +2,4 @@ OpenStack Drivers ================================ -Driver for openstack communication based on python-openstackclient. -Implements nova, cinder & neutron clients based on the same. +Driver for openstack communication based on python native clients. diff --git a/kingbird/drivers/openstack/cinder_v2.py b/kingbird/drivers/openstack/cinder_v2.py index 69281be..af7324d 100644 --- a/kingbird/drivers/openstack/cinder_v2.py +++ b/kingbird/drivers/openstack/cinder_v2.py @@ -14,6 +14,7 @@ from oslo_log import log from cinderclient.v2 import client as ci_client +from kingbird.common import exceptions from kingbird.drivers import base LOG = log.getLogger(__name__) @@ -22,12 +23,15 @@ LOG = log.getLogger(__name__) class CinderClient(base.DriverBase): '''Cinder V2 driver.''' def __init__(self, region, **kwargs): - self.cinder_client = ci_client.Client( - auth_url=kwargs['auth_url'], - username=kwargs['user_name'], - api_key=kwargs['password'], - tenant_id=kwargs['tenant_id'], - region_name=region) + try: + self.cinder_client = ci_client.Client( + auth_url=kwargs['auth_url'], + username=kwargs['user_name'], + api_key=kwargs['password'], + tenant_id=kwargs['tenant_id'], + region_name=region) + except exceptions.HttpException: + raise def get_resource_usages(self, project_id): '''Calcualte resources usage and return the dict''' diff --git a/kingbird/drivers/openstack/keystone_v3.py b/kingbird/drivers/openstack/keystone_v3.py index 66d6784..53d0971 100644 --- a/kingbird/drivers/openstack/keystone_v3.py +++ b/kingbird/drivers/openstack/keystone_v3.py @@ -30,17 +30,32 @@ class KeystoneClient(base.DriverBase): '''Keystone V3 driver.''' def __init__(self, **kwargs): - auth = v3.Password( - auth_url=kwargs['auth_url'], - username=kwargs['user_name'], - password=kwargs['password'], - project_name=kwargs['tenant_name']) - sess = session.Session(auth=auth) - self.keystone_client = client.Client(session=sess) + try: + auth = v3.Password( + auth_url=kwargs['auth_url'], + username=kwargs['user_name'], + password=kwargs['password'], + project_name=kwargs['tenant_name'], + project_domain_name=kwargs['project_domain'], + user_domain_name=kwargs['user_domain']) + sess = session.Session(auth=auth) + self.keystone_client = client.Client(session=sess) + self.services_list = self.keystone_client.services.list() + except exceptions.HttpException: + raise def get_enabled_projects(self): try: return [current_project.id for current_project in self.keystone.projects.list() if current_project.enabled] - except exceptions.HttpException as ex: - raise ex + except exceptions.HttpException: + raise + + def is_service_enabled(self, service): + try: + for current_service in self.services_list: + if service in current_service.type: + return True + return False + except exceptions.HttpException: + raise diff --git a/kingbird/drivers/openstack/neutron_v2.py b/kingbird/drivers/openstack/neutron_v2.py index 613506a..cd13874 100644 --- a/kingbird/drivers/openstack/neutron_v2.py +++ b/kingbird/drivers/openstack/neutron_v2.py @@ -12,6 +12,7 @@ from oslo_log import log +from kingbird.common import exceptions from kingbird.drivers import base from neutronclient.neutron import client @@ -23,13 +24,17 @@ API_VERSION = '2.0' class NeutronClient(base.DriverBase): '''Neutron V2 driver.''' def __init__(self, region, **kwargs): - self.neutron_client = client.Client( - API_VERSION, - username=kwargs['user_name'], - password=kwargs['password'], - tenant_name=kwargs['tenant_name'], - auth_url=kwargs['auth_url'], - region_name=region) + try: + self.neutron_client = client.Client( + API_VERSION, + username=kwargs['user_name'], + password=kwargs['password'], + tenant_name=kwargs['tenant_name'], + auth_url=kwargs['auth_url'], + region_name=region) + self.extension_list = self.neutron_client.list_extensions() + except exceptions.HttpException: + raise def get_resource_usages(self, project_id): '''Calcualte resources usage and return the dict''' @@ -42,3 +47,9 @@ class NeutronClient(base.DriverBase): def delete_quota_limits(self, project_id): '''Delete/Reset the limits''' pass + + def is_extension_supported(self, extension): + for current_extension in self.extension_list['extensions']: + if extension in current_extension['alias']: + return True + return False diff --git a/kingbird/drivers/openstack/nova_v2.py b/kingbird/drivers/openstack/nova_v2.py index d763a0b..7d5f1ab 100644 --- a/kingbird/drivers/openstack/nova_v2.py +++ b/kingbird/drivers/openstack/nova_v2.py @@ -12,6 +12,8 @@ from oslo_log import log +from kingbird.common import consts +from kingbird.common import exceptions from kingbird.drivers import base from novaclient import client as nv_client @@ -22,11 +24,16 @@ API_VERSION = '2.1' class NovaClient(base.DriverBase): '''Nova V2.1 driver.''' - def __init__(self, region, **kwargs): - self.nova_client = nv_client.Client( - API_VERSION, kwargs['user_name'], - kwargs['password'], kwargs['tenant_name'], - kwargs['auth_url'], region_name=region) + def __init__(self, region, disabled_quotas, **kwargs): + try: + self.nova_client = nv_client.Client( + API_VERSION, kwargs['user_name'], + kwargs['password'], kwargs['tenant_name'], + kwargs['auth_url'], region_name=region) + self.enabled_quotas = list(set(consts.NOVA_QUOTA_FIELDS) - + set(disabled_quotas)) + except exceptions.HttpException: + raise def get_resource_usages(self, project_id): '''Calcualte resources usage and return the dict''' diff --git a/kingbird/drivers/openstack/sdk.py b/kingbird/drivers/openstack/sdk.py index 25c79e6..10ba813 100644 --- a/kingbird/drivers/openstack/sdk.py +++ b/kingbird/drivers/openstack/sdk.py @@ -17,6 +17,7 @@ import collections from oslo_log import log +from kingbird.common import consts from kingbird.common import exceptions from kingbird.common.i18n import _ from kingbird.common.i18n import _LE @@ -33,18 +34,19 @@ LOG = log.getLogger(__name__) admin_creds_opts = [ cfg.StrOpt('auth_url', - default='http://127.0.0.1:5000/v3', help='keystone authorization url'), cfg.StrOpt('identity_url', - default='http://127.0.0.1:35357/v3', help='keystone service url'), cfg.StrOpt('admin_username', + default='admin', help='username of admin account, needed when' ' auto_refresh_endpoint set to True'), cfg.StrOpt('admin_password', + default='admin', help='password of admin account, needed when' ' auto_refresh_endpoint set to True'), cfg.StrOpt('admin_tenant', + default='admin', help='tenant name of admin account, needed when' ' auto_refresh_endpoint set to True'), cfg.StrOpt('admin_tenant_id', @@ -71,6 +73,21 @@ class OpenStackDriver(object): def __init__(self, region_name): # Check if objects are cached and try to use those self.region_name = region_name + self.services_list = [] + admin_kwargs = { + 'user_name': cfg.CONF.admin_creds.admin_username, + 'password': cfg.CONF.admin_creds.admin_password, + 'tenant_name': cfg.CONF.admin_creds.admin_tenant, + 'auth_url': cfg.CONF.admin_creds.auth_url, + 'tenant_id': cfg.CONF.admin_creds.admin_tenant_id, + 'project_domain': + cfg.CONF.admin_creds.admin_tenant_domain_name, + 'user_domain': cfg.CONF.admin_creds.admin_user_domain_name + } + if 'keystone' in OpenStackDriver.os_clients_dict: + self.keystone_client = OpenStackDriver.os_clients_dict['keystone'] + else: + self.keystone_client = KeystoneClient(**admin_kwargs) if region_name in OpenStackDriver.os_clients_dict: LOG.info(_LI('Using cached OS client objects')) self.nova_client = OpenStackDriver.os_clients_dict[ @@ -82,16 +99,11 @@ class OpenStackDriver(object): else: # Create new objects and cache them LOG.debug(_("Creating fresh OS Clients objects")) - admin_kwargs = { - 'user_name': cfg.CONF.admin_creds.admin_username, - 'password': cfg.CONF.admin_creds.admin_password, - 'tenant_name': cfg.CONF.admin_creds.admin_tenant, - 'auth_url': cfg.CONF.admin_creds.auth_url, - 'tenant_id': cfg.CONF.admin_creds.admin_tenant_id - } - self.nova_client = NovaClient(region_name, **admin_kwargs) self.cinder_client = CinderClient(region_name, **admin_kwargs) self.neutron_client = NeutronClient(region_name, **admin_kwargs) + self.disabled_quotas = self._get_disabled_quotas(region_name) + self.nova_client = NovaClient(region_name, self.disabled_quotas, + **admin_kwargs) OpenStackDriver.os_clients_dict[ region_name] = collections.defaultdict(dict) OpenStackDriver.os_clients_dict[region_name][ @@ -100,11 +112,6 @@ class OpenStackDriver(object): 'cinder'] = self.cinder_client OpenStackDriver.os_clients_dict[region_name][ 'neutron'] = self.neutron_client - if 'keystone' in OpenStackDriver.os_clients_dict: - self.keystone_client = OpenStackDriver.os_clients_dict['keystone'] - else: - self.keystone_client = KeystoneClient(**admin_kwargs) - OpenStackDriver.os_clients_dict['keystone'] = self.keystone_client def get_enabled_projects(self): try: @@ -152,3 +159,27 @@ class OpenStackDriver(object): del OpenStackDriver.os_clients_dict[self.region_name] except Exception as exception: LOG.error(_LE('Error Occurred: %s'), exception.message) + + def _get_disabled_quotas(self, region): + disabled_quotas = [] + # Neutron + if not self.keystone_client.is_service_enabled('network'): + disabled_quotas.extend(consts.NEUTRON_QUOTA_FIELDS) + else: + # Remove the nova network quotas + disabled_quotas.extend(['floating_ips', 'fixed_ips']) + if self.neutron_client.is_extension_supported('security-group'): + # If Neutron security group is supported, disable Nova quotas + disabled_quotas.extend(['security_groups', + 'security_group_rules']) + else: + # If Nova security group is used, disable Neutron quotas + disabled_quotas.extend(['security_group', + 'security_group_rule']) + try: + if not self.neutron_client.is_extension_supported('quotas'): + disabled_quotas.extend(consts.NEUTRON_QUOTA_FIELDS) + except Exception: + LOG.exception("There was an error checking if the Neutron " + "quotas extension is enabled.") + return disabled_quotas diff --git a/kingbird/tests/unit/drivers/test_keystone_v3.py b/kingbird/tests/unit/drivers/test_keystone_v3.py index e261b63..2f03cdd 100644 --- a/kingbird/tests/unit/drivers/test_keystone_v3.py +++ b/kingbird/tests/unit/drivers/test_keystone_v3.py @@ -12,8 +12,6 @@ import mock -import keystoneclient - from kingbird.drivers.openstack import keystone_v3 from kingbird.tests import base from kingbird.tests import utils @@ -22,20 +20,28 @@ FAKE_ADMIN_CREDS = { 'user_name': 'fake_user', 'password': 'pass1234', 'tenant_name': 'test_tenant', - 'auth_url': 'http://127.0.0.1:5000/v3' + 'auth_url': 'http://127.0.0.1:5000/v3', + 'project_domain': 'domain1', + 'user_domain': 'user_dom' } +FAKE_SERVICE = [ + 'endpoint_volume', + 'endpoint_network' + ] + class TestKeystoneClient(base.KingbirdTestCase): def setUp(self): super(TestKeystoneClient, self).setUp() self.ctx = utils.dummy_context() - def test_init(self): + @mock.patch.object(keystone_v3, 'KeystoneClient') + def test_init(self, mock_keystone): + mock_keystone().services_list = FAKE_SERVICE key_client = keystone_v3.KeystoneClient(**FAKE_ADMIN_CREDS) - self.assertIsNotNone(key_client.keystone_client) - self.assertIsInstance(key_client.keystone_client, - keystoneclient.v3.client.Client) + self.assertEqual(key_client.services_list, + FAKE_SERVICE) @mock.patch.object(keystone_v3, 'KeystoneClient') def test_get_enabled_projects(self, mock_key_client): @@ -46,3 +52,10 @@ class TestKeystoneClient(base.KingbirdTestCase): except Exception: raised = True self.assertFalse(raised, 'get_enabled_projects Failed') + + @mock.patch.object(keystone_v3, 'KeystoneClient') + def test_is_service_enabled(self, mock_keystone): + key_client = keystone_v3.KeystoneClient(**FAKE_ADMIN_CREDS) + mock_keystone().is_service_enabled.return_value = True + network_enabled = key_client.is_service_enabled('network') + self.assertEqual(network_enabled, True) diff --git a/kingbird/tests/unit/drivers/test_neutron_v2.py b/kingbird/tests/unit/drivers/test_neutron_v2.py index 0fb0a6c..b10b85a 100644 --- a/kingbird/tests/unit/drivers/test_neutron_v2.py +++ b/kingbird/tests/unit/drivers/test_neutron_v2.py @@ -10,7 +10,7 @@ # License for the specific language governing permissions and limitations # under the License. -import neutronclient +import mock from kingbird.drivers.openstack import neutron_v2 from kingbird.tests import base @@ -23,18 +23,32 @@ FAKE_ADMIN_CREDS = { 'auth_url': 'http://127.0.0.1:5000/v3' } +FAKE_EXTENSIONS = { + 'extensions': ['fake_extension1', + 'fake_extension2'] + } + class TestNeutronClient(base.KingbirdTestCase): def setUp(self): super(TestNeutronClient, self).setUp() self.ctx = utils.dummy_context() - def test_init(self): + @mock.patch.object(neutron_v2, 'NeutronClient') + def test_init(self, mock_neutron): + mock_neutron().extension_list = FAKE_EXTENSIONS neutron_client = neutron_v2.NeutronClient('fake_region', **FAKE_ADMIN_CREDS) - self.assertIsNotNone(neutron_client) - self.assertIsInstance(neutron_client.neutron_client, - neutronclient.v2_0.client.Client) + self.assertEqual(FAKE_EXTENSIONS, + neutron_client.extension_list) + + @mock.patch.object(neutron_v2, 'NeutronClient') + def test_is_extension_supported(self, mock_neutron): + neutron_client = neutron_v2.NeutronClient('fake_region', + **FAKE_ADMIN_CREDS) + mock_neutron().is_extension_supported.return_value = True + extension_enabled = neutron_client.is_extension_supported('quotas') + self.assertEqual(extension_enabled, True) def test_get_resource_usages(self): pass diff --git a/kingbird/tests/unit/drivers/test_nova_v2.py b/kingbird/tests/unit/drivers/test_nova_v2.py index 00c6669..47b7077 100644 --- a/kingbird/tests/unit/drivers/test_nova_v2.py +++ b/kingbird/tests/unit/drivers/test_nova_v2.py @@ -12,6 +12,7 @@ import novaclient +from kingbird.common import consts from kingbird.drivers.openstack import nova_v2 from kingbird.tests import base from kingbird.tests import utils @@ -22,6 +23,7 @@ FAKE_ADMIN_CREDS = { 'tenant_name': 'test_tenant', 'auth_url': 'http://127.0.0.1:5000/v3' } +DISABLED_QUOTAS = ["floating_ips", "fixed_ips", "security_groups"] class TestNovaClient(base.KingbirdTestCase): @@ -30,8 +32,12 @@ class TestNovaClient(base.KingbirdTestCase): self.ctx = utils.dummy_context() def test_init(self): - nv_client = nova_v2.NovaClient('fake_region', **FAKE_ADMIN_CREDS) + nv_client = nova_v2.NovaClient('fake_region', DISABLED_QUOTAS, + **FAKE_ADMIN_CREDS) self.assertIsNotNone(nv_client) + expected_quotas = list(set(consts.NOVA_QUOTA_FIELDS) - + set(DISABLED_QUOTAS)) + self.assertEqual(nv_client.enabled_quotas, expected_quotas) self.assertIsInstance(nv_client.nova_client, novaclient.v2.client.Client) diff --git a/kingbird/tests/unit/drivers/test_openstack_driver.py b/kingbird/tests/unit/drivers/test_openstack_driver.py index dd6ae9a..16d9da1 100644 --- a/kingbird/tests/unit/drivers/test_openstack_driver.py +++ b/kingbird/tests/unit/drivers/test_openstack_driver.py @@ -17,6 +17,17 @@ from kingbird.tests import base from kingbird.tests import utils +class FakeService(object): + + '''Fake service class used to test service enable testcase + + ''' + + def __init__(self, type_service, name): + self.type = type_service + self.name = name + + class TestOpenStackDriver(base.KingbirdTestCase): def setUp(self): super(TestOpenStackDriver, self).setUp() @@ -132,3 +143,18 @@ class TestOpenStackDriver(base.KingbirdTestCase): self.assertEqual(os_driver_2.cinder_client, os_driver_4.cinder_client) self.assertEqual(os_driver_2.neutron_client, os_driver_4.neutron_client) + + @mock.patch.object(sdk, 'KeystoneClient') + @mock.patch.object(sdk, 'NovaClient') + @mock.patch.object(sdk, 'NeutronClient') + @mock.patch.object(sdk, 'CinderClient') + def test_get_disabled_quotas(self, mock_cinder_client, + mock_network_client, mock_nova_client, + mock_keystone_client): + input_disable_quotas = ["floating_ips", "security_groups", + "security_group_rules"] + os_driver = sdk.OpenStackDriver('fake_region9') + output_disabled_quotas = os_driver._get_disabled_quotas('fake_region9') + self.assertIn(input_disable_quotas[0], output_disabled_quotas) + self.assertIn(input_disable_quotas[1], output_disabled_quotas) + self.assertIn(input_disable_quotas[2], output_disabled_quotas)