From 00eda5e6ce1ac4bee789ea0877beda5bb7e7eb0a Mon Sep 17 00:00:00 2001 From: DhuldevValekar Date: Tue, 21 Mar 2017 05:58:54 +0000 Subject: [PATCH] NFP- Devstack plugin mode installation changes. 1. Replaced keystone command with openstack. 2. Fixing token handling Change-Id: I2887cd43fc7a8990a1c44272c6474aa36624018c Closes-bug:1674318 --- devstack/lib/nfp | 25 +--- devstack/settings | 9 +- .../alembic_migrations/versions/HEAD | 2 +- ...60c5682e74_nfp_rename_tenant_to_project.py | 131 ++++++++++++++++++ .../unit/nfp/orchestrator/test_heat_driver.py | 36 +++-- .../nfp/orchestrator/test_openstack_driver.py | 41 +++--- gbpservice/nfp/orchestrator/db/nfp_db.py | 19 +-- .../openstack/openstack_driver.py | 49 +++---- .../tests/contrib/devstack/local-nfp.conf | 10 +- 9 files changed, 228 insertions(+), 94 deletions(-) create mode 100644 gbpservice/neutron/db/migration/alembic_migrations/versions/c460c5682e74_nfp_rename_tenant_to_project.py diff --git a/devstack/lib/nfp b/devstack/lib/nfp index 9fab9b20e..865820d36 100644 --- a/devstack/lib/nfp +++ b/devstack/lib/nfp @@ -63,21 +63,8 @@ function init_nfpgbpservice { # assign_user_role_credential() - Assign Service role to the users function assign_user_role_credential { - source $DEVSTACK_DIR/openrc admin admin - - serviceTenantID=`keystone tenant-list | grep -w '[^.]service[^.]' | awk '{print $2}'` - serviceRoleID=`keystone role-list | grep -w '[^.]service[^.]' | awk '{print $2}'` - adminRoleID=`keystone role-list | grep -w '[^.]admin[^.]' | awk '{print $2}'` - - keystone user-role-add\ - --user nova\ - --tenant $serviceTenantID\ - --role $serviceRoleID - - keystone user-role-add\ - --user neutron\ - --tenant $serviceTenantID\ - --role $adminRoleID + openstack --os-cloud=devstack-admin role add --project service --user nova service + openstack --os-cloud=devstack-admin role add --project service --user neutron admin } # create_ext_net() - Create an external network @@ -456,9 +443,11 @@ function copy_nfp_files_and_start_process { auth_uri=`iniget $NEUTRON_CONF keystone_authtoken auth_uri` auth_protocol=$(echo $auth_uri | tr ':/' ' ' | awk '{print $1}') auth_host=$(echo $auth_uri | tr ':/' ' ' | awk '{print $2}') - auth_port=$(echo $auth_uri | tr ':/' ' ' | awk '{print $3}') - auth_version=$(echo $auth_uri | tr ':/' ' ' | awk '{print $4}') - auth_version=${auth_version:-v2.0} + #auth_port=$(echo $auth_uri | tr ':/' ' ' | awk '{print $3}') + #auth_version=$(echo $auth_uri | tr ':/' ' ' | awk '{print $4}') + #auth_version=${auth_version:-v2.0} + auth_port=5000 + auth_version=v2.0 iniset -sudo /etc/nfp.ini nfp_keystone_authtoken admin_user $admin_user iniset -sudo /etc/nfp.ini nfp_keystone_authtoken admin_password $admin_password diff --git a/devstack/settings b/devstack/settings index 405cf232e..c23806b36 100755 --- a/devstack/settings +++ b/devstack/settings @@ -49,14 +49,9 @@ if [[ $ENABLE_APIC_AIM_GATE = True ]]; then disable_service q-meta else enable_service q-dhcp - enable_service q-fwaas -fi -[[ $ENABLE_NFP = True ]] && [[ $NFP_DEVSTACK_MODE = advanced ]] && enable_service neutron-vpnaas -if [[ $ENABLE_NFP = True ]] && [[ $NFP_DEVSTACK_MODE = advanced ]] && [[ $ENABLE_LBAASV2 = True ]]; then - enable_service q-lbaasv2 -else - enable_service q-lbaas fi +enable_service q-fwaas-v1 +enable_service q-lbaasv2 enable_service q-meta enable_service neutron enable_service group-policy diff --git a/gbpservice/neutron/db/migration/alembic_migrations/versions/HEAD b/gbpservice/neutron/db/migration/alembic_migrations/versions/HEAD index 0c40567e0..c872fd562 100644 --- a/gbpservice/neutron/db/migration/alembic_migrations/versions/HEAD +++ b/gbpservice/neutron/db/migration/alembic_migrations/versions/HEAD @@ -1 +1 @@ -032f3315c315 +c460c5682e74 diff --git a/gbpservice/neutron/db/migration/alembic_migrations/versions/c460c5682e74_nfp_rename_tenant_to_project.py b/gbpservice/neutron/db/migration/alembic_migrations/versions/c460c5682e74_nfp_rename_tenant_to_project.py new file mode 100644 index 000000000..23c5c56aa --- /dev/null +++ b/gbpservice/neutron/db/migration/alembic_migrations/versions/c460c5682e74_nfp_rename_tenant_to_project.py @@ -0,0 +1,131 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +"""nfp rename tenant to project + +Revision ID: c460c5682e74 +Create Date: 2017-03-21 09:08:39.314811 + +""" + +# revision identifiers, used by Alembic. +revision = 'c460c5682e74' +down_revision = '032f3315c315' + +from alembic import op +import sqlalchemy as sa + + +_INSPECTOR = None + + +def get_inspector(): + """Reuse inspector""" + + global _INSPECTOR + + if _INSPECTOR: + return _INSPECTOR + + else: + bind = op.get_bind() + _INSPECTOR = sa.engine.reflection.Inspector.from_engine(bind) + + return _INSPECTOR + + +def get_tables(): + """ + Returns hardcoded list of NFP tables which have ``tenant_id`` column. + + """ + + tables = [ + 'nfp_port_infos', + 'nfp_network_infos', + 'nfp_network_function_instances', + 'nfp_network_functions', + 'nfp_network_function_devices', + 'nfd_cluster_mapping_info', + ] + + return tables + + +def get_columns(table): + """Returns list of columns for given table.""" + inspector = get_inspector() + return inspector.get_columns(table) + + +def get_data(): + """Returns combined list of tuples: [(table, column)]. + + List is built, based on retrieved tables, where column with name + ``tenant_id`` exists. + """ + + output = [] + tables = get_tables() + for table in tables: + columns = get_columns(table) + + for column in columns: + if column['name'] == 'tenant_id': + output.append((table, column)) + + return output + + +def alter_column(table, column): + old_name = 'tenant_id' + new_name = 'project_id' + + op.alter_column( + table_name=table, + column_name=old_name, + new_column_name=new_name, + existing_type=column['type'], + existing_nullable=column['nullable'] + ) + + +def recreate_index(index, table_name): + old_name = index['name'] + new_name = old_name.replace('tenant', 'project') + + op.drop_index(op.f(old_name), table_name) + op.create_index(new_name, table_name, ['project_id']) + + +def upgrade(): + inspector = get_inspector() + + data = get_data() + for table, column in data: + alter_column(table, column) + + indexes = inspector.get_indexes(table) + for index in indexes: + if 'tenant_id' in index['name']: + recreate_index(index, table) + + +def contract_creation_exceptions(): + """Special migration for the blueprint to support Keystone V3. + We drop all tenant_id columns and create project_id columns instead. + """ + return { + sa.Column: ['.'.join([table, 'project_id']) for table in get_tables()], + sa.Index: get_tables() + } diff --git a/gbpservice/neutron/tests/unit/nfp/orchestrator/test_heat_driver.py b/gbpservice/neutron/tests/unit/nfp/orchestrator/test_heat_driver.py index 6d41808b3..871f5b8b7 100644 --- a/gbpservice/neutron/tests/unit/nfp/orchestrator/test_heat_driver.py +++ b/gbpservice/neutron/tests/unit/nfp/orchestrator/test_heat_driver.py @@ -11,6 +11,8 @@ # under the License. import copy +from keystoneauth1.identity import v2 +from keystoneauth1 import session from keystoneclient.v2_0 import client as identity_client import mock from oslo_config import cfg @@ -101,10 +103,10 @@ class TestHeatDriver(unittest2.TestCase): mock.patch(heat_client.__name__ + ".HeatClient", new=MockHeatClient).start() - @mock.patch.object(identity_client, "Client") - def test_keystone(self, mock_obj): - keystone_client = mock_obj.return_value - keystone_client.auth_token = 'abcd123' + @mock.patch.object(v2, "Password") + @mock.patch.object(session.Session, "get_token") + def test_keystone(self, mock_session, mock_v2): + mock_session.return_value = 'abcd123' password = 'neutron_pass' tenant_name = 'services' username = 'neutron' @@ -127,9 +129,11 @@ class TestHeatDriver(unittest2.TestCase): self.assertIsNotNone(heat_client_obj) @mock.patch.object(identity_client, "Client") - def test_resource_owner_tenant_id(self, mock_obj): + @mock.patch.object(v2, "Password") + @mock.patch.object(session.Session, "get_token") + def test_resource_owner_tenant_id(self, mock_session, mock_v2, mock_obj): + mock_session.return_value = True keystone_client = mock_obj.return_value - keystone_client.auth_token = True keystone_client.tenants.find().id = '8ae6701128994ab281dde6b92207bb19' expected_resource_owner_tenant_id = '8ae6701128994ab281dde6b92207bb19' resource_owner_tenant_id = ( @@ -200,10 +204,11 @@ class TestHeatDriver(unittest2.TestCase): self.assertEqual(resource_owner_context, expected_resource_owner_context) - @mock.patch.object(identity_client, "Client") - def test_get_tenant_context(self, mock_obj): - keystone_client = mock_obj.return_value - keystone_client.auth_token = True + @mock.patch.object(v2, "Password") + @mock.patch.object(session.Session, "get_token") + def test_get_tenant_context(self, mock_session, mock_v2): + mock_session.return_value = True + tenant_id = '8ae6701128994ab281dde6b92207bb19' expected_tenant_context = (True, tenant_id) tenant_context = self.heat_driver_obj._get_tenant_context(tenant_id) @@ -321,7 +326,6 @@ class TestHeatDriver(unittest2.TestCase): return_value=None) nfp_logging.get_logging_context = mock.Mock( return_value={'auth_token': '7fd6701128994ab281ccb6b92207bb15'}) - instance = mock_obj.return_value instance.auth_token = True stack_id = '70754fdd-0325-4856-8a39-f171b65617d6' @@ -330,7 +334,13 @@ class TestHeatDriver(unittest2.TestCase): @mock.patch.object(heat_client.HeatClient, 'get') @mock.patch.object(identity_client, "Client") - def test_is_config_complete(self, mock_obj, heat_get_mock_obj): + @mock.patch.object(v2, "Password") + @mock.patch.object(session.Session, "get_token") + def test_is_config_complete(self, mock_session, mock_v2, mock_obj, + heat_get_mock_obj): + mock_session.return_value = True + keystone_client = mock_obj.return_value + keystone_client.tenants.find().id = '8ae6701128994ab281dde6b92207bb19' stack_id = '70754fdd-0325-4856-8a39-f171b65617d6' tenant_id = '8ae6701128994ab281dde6b92207bb19' self.heat_driver_obj._assign_admin_user_to_project = mock.Mock( @@ -341,8 +351,6 @@ class TestHeatDriver(unittest2.TestCase): return_value=None) heat_get_mock_obj.return_value = MockStackObject( 'CREATE_COMPLETE') - instance = mock_obj.return_value - instance.auth_token = True expected_status = 'COMPLETED' status = self.heat_driver_obj.is_config_complete( stack_id, tenant_id, self.mock_dict.network_function_details) diff --git a/gbpservice/neutron/tests/unit/nfp/orchestrator/test_openstack_driver.py b/gbpservice/neutron/tests/unit/nfp/orchestrator/test_openstack_driver.py index 49efa4dcc..d92c51ec3 100644 --- a/gbpservice/neutron/tests/unit/nfp/orchestrator/test_openstack_driver.py +++ b/gbpservice/neutron/tests/unit/nfp/orchestrator/test_openstack_driver.py @@ -16,6 +16,8 @@ import unittest2 from gbpclient.v2_0 import client as gbp_client from gbpservice.nfp.orchestrator.openstack import openstack_driver +from keystoneauth1.identity import v2 +from keystoneauth1 import session from keystoneclient.v2_0 import client as identity_client from neutronclient.v2_0 import client as neutron_client from novaclient import client as nova_client @@ -63,39 +65,40 @@ class TestKeystoneClient(SampleData): 'None', group='nfp_keystone_authtoken') - def test_get_admin_token(self, mock_obj): - instance = mock_obj.return_value - instance.auth_token = True + @mock.patch.object(v2, "Password") + @mock.patch.object(session.Session, "get_token") + def test_get_admin_token(self, mock_session, mock_v2, mock_obj): + mock_session.return_value = True retval = self.keystone_obj.get_admin_token() self.assertTrue(retval) - mock_obj.assert_called_once_with(auth_url=self.AUTH_URL, - password='neutron_pass', - tenant_id=None, - tenant_name='service', - username='neutron') + mock_v2.assert_called_once_with(auth_url=self.AUTH_URL, + password='neutron_pass', + tenant_name='service', + username='neutron') - def test_get_scoped_keystone_token(self, mock_obj): - instance = mock_obj.return_value - instance.auth_token = True + @mock.patch.object(v2, "Password") + @mock.patch.object(session.Session, "get_token") + def test_get_scoped_keystone_token(self, mock_session, mock_v2, mock_obj): + mock_session.return_value = True retval = self.keystone_obj.get_scoped_keystone_token(self.USERNAME, self.PASSWORD, self.TENANT_NAME, self.TENANT_ID) self.assertTrue(retval) - mock_obj.assert_called_once_with(auth_url=self.AUTH_URL, - password=self.PASSWORD, - tenant_id=self.TENANT_ID, - tenant_name=self.TENANT_NAME, - username=self.USERNAME) + mock_v2.assert_called_once_with(auth_url=self.AUTH_URL, + password=self.PASSWORD, + tenant_name=self.TENANT_NAME, + username=self.USERNAME) - def test_get_tenant_id(self, mock_obj): + @mock.patch.object(v2, "Password") + @mock.patch.object(session.Session, "get_token") + def test_get_tenant_id(self, mock_session, mock_v2, mock_obj): instance = mock_obj.return_value instance.tenants.find().id = True retval = self.keystone_obj.get_tenant_id(self.AUTH_TOKEN, "service") self.assertTrue(retval) - mock_obj.assert_called_once_with(auth_url=self.AUTH_URL, - token=self.AUTH_TOKEN) + mock_obj.assert_called_once_with(session=mock.ANY) @mock.patch.object(nova_client, "Client") diff --git a/gbpservice/nfp/orchestrator/db/nfp_db.py b/gbpservice/nfp/orchestrator/db/nfp_db.py index 851e47b4d..13b3b5fc7 100644 --- a/gbpservice/nfp/orchestrator/db/nfp_db.py +++ b/gbpservice/nfp/orchestrator/db/nfp_db.py @@ -36,7 +36,7 @@ class NFPDbBase(common_db_mixin.CommonDbMixin): id=uuidutils.generate_uuid(), name=network_function['name'], description=network_function.get('description'), - tenant_id=network_function['tenant_id'], + project_id=network_function['tenant_id'], service_id=network_function['service_id'], service_chain_id=network_function.get('service_chain_id'), service_profile_id=network_function['service_profile_id'], @@ -115,7 +115,7 @@ class NFPDbBase(common_db_mixin.CommonDbMixin): nfp_db_model.NetworkFunctionInstance( id=uuidutils.generate_uuid(), name=network_function_instance['name'], - tenant_id=network_function_instance['tenant_id'], + project_id=network_function_instance['tenant_id'], description=network_function_instance.get('description'), network_function_id=network_function_instance[ 'network_function_id'], @@ -293,7 +293,7 @@ class NFPDbBase(common_db_mixin.CommonDbMixin): uuidutils.generate_uuid()), name=network_function_device['name'], description=network_function_device.get('description'), - tenant_id=network_function_device['tenant_id'], + project_id=network_function_device['tenant_id'], mgmt_ip_address=network_function_device[ 'mgmt_ip_address'], service_vendor=network_function_device.get('service_vendor'), @@ -495,7 +495,7 @@ class NFPDbBase(common_db_mixin.CommonDbMixin): def _make_network_function_dict(self, network_function, fields=None): res = {'id': network_function['id'], - 'tenant_id': network_function['tenant_id'], + 'tenant_id': network_function['project_id'], 'name': network_function['name'], 'description': network_function['description'], 'service_id': network_function['service_id'], @@ -512,7 +512,7 @@ class NFPDbBase(common_db_mixin.CommonDbMixin): def _make_network_function_instance_dict(self, nfi, fields=None): res = {'id': nfi['id'], - 'tenant_id': nfi['tenant_id'], + 'tenant_id': nfi['project_id'], 'name': nfi['name'], 'description': nfi['description'], 'ha_state': nfi['ha_state'], @@ -526,7 +526,7 @@ class NFPDbBase(common_db_mixin.CommonDbMixin): def _make_network_function_device_dict(self, nfd, fields=None): res = {'id': nfd['id'], - 'tenant_id': nfd['tenant_id'], + 'tenant_id': nfd['project_id'], 'name': nfd['name'], 'description': nfd['description'], 'mgmt_ip_address': nfd['mgmt_ip_address'], @@ -547,7 +547,7 @@ class NFPDbBase(common_db_mixin.CommonDbMixin): def add_cluster_info(self, session, cluster_info): with session.begin(subtransactions=True): cluster_info = nfp_db_model.ClusterInfo( - id=cluster_info['id'], tenant_id=cluster_info['tenant_id'], + id=cluster_info['id'], project_id=cluster_info['tenant_id'], network_function_device_id=cluster_info[ 'network_function_device_id'], cluster_group=cluster_info[ @@ -556,6 +556,7 @@ class NFPDbBase(common_db_mixin.CommonDbMixin): cluster_name=cluster_info.get('cluster_name', None) ) session.add(cluster_info) + setattr(cluster_info, 'tenant_id', cluster_info['project_id']) return cluster_info def insert_cluster_records(self, session, cluster_infos): @@ -563,7 +564,7 @@ class NFPDbBase(common_db_mixin.CommonDbMixin): for cluster_info in cluster_infos: cluster_info = nfp_db_model.ClusterInfo( id=cluster_info['id'], - tenant_id=cluster_info['tenant_id'], + project_id=cluster_info['tenant_id'], network_function_device_id=cluster_info[ 'network_function_device_id'], cluster_group=cluster_info['cluster_group'], @@ -606,7 +607,7 @@ class NFPDbBase(common_db_mixin.CommonDbMixin): sorts=None, limit=None, marker=None, page_reverse=False): return { - 'id': cluster_info['id'], 'tenant_id': cluster_info['tenant_id'], + 'id': cluster_info['id'], 'tenant_id': cluster_info['project_id'], 'network_function_device_id': cluster_info[ 'network_function_device_id'], 'cluster_group': cluster_info['cluster_group'], diff --git a/gbpservice/nfp/orchestrator/openstack/openstack_driver.py b/gbpservice/nfp/orchestrator/openstack/openstack_driver.py index 2f07ccc53..2b9d470f8 100644 --- a/gbpservice/nfp/orchestrator/openstack/openstack_driver.py +++ b/gbpservice/nfp/orchestrator/openstack/openstack_driver.py @@ -11,6 +11,9 @@ # under the License. from gbpclient.v2_0 import client as gbp_client +from keystoneauth1.identity import v2 +from keystoneauth1.identity import v3 +from keystoneauth1 import session from keystoneclient.v2_0 import client as identity_client from keystoneclient.v3 import client as keyclientv3 from neutronclient.v2_0 import client as neutron_client @@ -78,17 +81,15 @@ class KeystoneClient(OpenstackApi): err = "Tenant Not specified for getting a scoped token" LOG.error(err) raise Exception(err) - - keystone = identity_client.Client( - username=user, - password=password, - tenant_name=tenant_name, - tenant_id=tenant_id, - auth_url=self.identity_service) try: - scoped_token = keystone.auth_token + auth = v2.Password(username=user, + password=password, + tenant_name=tenant_name, + auth_url=self.identity_service) + sess = session.Session(auth=auth) + scoped_token = sess.get_token(auth=auth) except Exception as err: - err = ("Failed to get scoped token from" + err = ("Failed to get token from" " Openstack Keystone service" " KeyError :: %s" % (err)) self.config.nfp_keystone_authtoken.auth_port, @@ -114,8 +115,7 @@ class KeystoneClient(OpenstackApi): :return: Tenant UUID """ try: - keystone = identity_client.Client(token=token, - auth_url=self.identity_service) + keystone = self._get_v2_keystone_admin_client() tenant = keystone.tenants.find(name=tenant_name) return tenant.id except Exception as ex: @@ -134,14 +134,12 @@ class KeystoneClient(OpenstackApi): keystone resources. """ keystone_conf = self.config.nfp_keystone_authtoken - - v2client = identity_client.Client( - username=keystone_conf.admin_user, - password=keystone_conf.admin_password, - tenant_name=keystone_conf.admin_tenant_name, - tenant_id=None, - auth_url=self.identity_service) - + auth = v2.Password(username=keystone_conf.admin_user, + password=keystone_conf.admin_password, + tenant_name=keystone_conf.admin_tenant_name, + auth_url=self.identity_service) + sess = session.Session(auth=auth) + v2client = identity_client.Client(session=sess) return v2client def _get_v3_keystone_admin_client(self): @@ -153,11 +151,14 @@ class KeystoneClient(OpenstackApi): v3_auth_url = ('%s://%s:%s/%s/' % ( keystone_conf.auth_protocol, keystone_conf.auth_host, keystone_conf.auth_port, self.config.heat_driver.keystone_version)) - v3client = keyclientv3.Client( - username=keystone_conf.admin_user, - password=keystone_conf.admin_password, - domain_name="default", - auth_url=v3_auth_url) + auth = v3.Password(auth_url=v3_auth_url, + user_domain_name='Default', + username=keystone_conf.admin_user, + password=keystone_conf.admin_password, + project_domain_name="Default", + project_name=keystone_conf.admin_tenant_name) + sess = session.Session(auth=auth) + v3client = keyclientv3.Client(session=sess) return v3client diff --git a/gbpservice/tests/contrib/devstack/local-nfp.conf b/gbpservice/tests/contrib/devstack/local-nfp.conf index 3c5398bb7..82898b8b8 100644 --- a/gbpservice/tests/contrib/devstack/local-nfp.conf +++ b/gbpservice/tests/contrib/devstack/local-nfp.conf @@ -45,8 +45,14 @@ enable_service q-svc enable_service q-agt enable_service q-dhcp enable_service q-l3 -enable_service q-fwaas -enable_service q-lbaas +#enable_service q-fwaas +enable_plugin neutron-fwaas http://git.openstack.org/openstack/neutron-fwaas stable/newton +enable_plugin neutron-lbaas https://git.openstack.org/openstack/neutron-lbaas stable/newton +enable_plugin octavia https://git.openstack.org/openstack/octavia +enable_plugin neutron https://github.com/openstack/neutron.git stable/newton + +enable_service q-fwaas-v1 +enable_service q-lbaasv2 enable_service q-meta enable_service neutron enable_service group-policy