Create associations between projects and endpoints

OS-EP-FILTER Implementation

There are new methods to create endpoint and project associations.
A full CRUD API to assign projects to endpoints as well as
the ability to check all the projects associated with a given
endpoint.

The association is used to pick what endpoints are visible
for the given project and a filtered catalog is built
accordingly.

During a project-scoped token request, if project-endpoint
associations have been created, the returned catalog will only
list the project linked endpoints.

blueprint endpoint-filtering

Change-Id: Idaa7f448a67e3bae01ba12686be37ba058183cf6

etc/keystone-paste.ini

@@ -30,6 +30,9 @@ paste.filter_factory = keystone.contrib.oauth1.routers:OAuth1Extension.factory
 [filter:s3_extension]
 paste.filter_factory = keystone.contrib.s3:S3Extension.factory
 
+[filter:endpoint_filter_extension]
+paste.filter_factory = keystone.contrib.endpoint_filter.routers:EndpointFilterExtension.factory
+
 [filter:url_normalize]
 paste.filter_factory = keystone.middleware:NormalizingFilter.factory
 

etc/keystone.conf.sample

@@ -134,6 +134,12 @@
 
 # template_file = default_catalog.templates
 
+[endpoint_filter]
+# extension for creating associations between project and endpoints in order to
+# provide a tailored catalog for project-scoped token requests.
+# driver = keystone.contrib.endpoint_filter.backends.sql.EndpointFilter
+# return_all_endpoints_if_no_filter = True
+
 [token]
 # Provides token persistence.
 # driver = keystone.token.backends.sql.Token

etc/policy.json

@@ -86,5 +86,11 @@
     "identity:list_roles_for_trust": [["@"]],
     "identity:check_role_for_trust": [["@"]],
     "identity:get_role_for_trust": [["@"]],
-    "identity:delete_trust": [["@"]]
+    "identity:delete_trust": [["@"]],
+
+    "identity:list_projects_for_endpoint": [["rule:admin_required"]],
+    "identity:add_endpoint_to_project": [["rule:admin_required"]],
+    "identity:check_endpoint_in_project": [["rule:admin_required"]],
+    "identity:list_endpoints_for_project": [["rule:admin_required"]],
+    "identity:remove_endpoint_from_project": [["rule:admin_required"]]
 }

keystone/common/config.py

@@ -127,6 +127,11 @@ FILE_OPTIONS = {
     'ec2': [
         cfg.StrOpt('driver',
                    default='keystone.contrib.ec2.backends.kvs.Ec2')],
+    'endpoint_filter': [
+        cfg.StrOpt('driver',
+                   default='keystone.contrib.endpoint_filter.backends'
+                           '.sql.EndpointFilter'),
+        cfg.BoolOpt('return_all_endpoints_if_no_filter', default=True)],
     'stats': [
         cfg.StrOpt('driver',
                    default=('keystone.contrib.stats.backends'

keystone/contrib/endpoint_filter/__init__.py

@@ -0,0 +1,18 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+# flake8: noqa
+
+# Copyright 2013 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from keystone.contrib.endpoint_filter.core import *

keystone/contrib/endpoint_filter/backends/catalog_sql.py

@@ -0,0 +1,71 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2013 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from keystone.catalog.backends import sql
+from keystone.catalog import core as catalog_core
+from keystone.common import dependency
+from keystone import config
+from keystone import exception
+
+CONF = config.CONF
+
+
+@dependency.requires('endpoint_filter_api')
+class EndpointFilterCatalog(sql.Catalog):
+    def get_v3_catalog(self, user_id, project_id, metadata=None):
+        d = dict(CONF.iteritems())
+        d.update({'tenant_id': project_id, 'user_id': user_id})
+
+        services = {}
+
+        refs = self.endpoint_filter_api.list_endpoints_for_project(project_id)
+
+        if (len(refs) == 0 and
+                CONF.endpoint_filter.return_all_endpoints_if_no_filter):
+            return super(EndpointFilterCatalog, self).get_v3_catalog(
+                user_id, project_id, metadata=metadata)
+
+        for entry in refs:
+            try:
+                endpoint = self.get_endpoint(entry.endpoint_id)
+                service_id = endpoint['service_id']
+                services.setdefault(
+                    service_id,
+                    self.get_service(service_id))
+                service = services[service_id]
+                del endpoint['service_id']
+                endpoint['url'] = catalog_core.format_url(
+                    endpoint['url'], d)
+                # populate filtered endpoints
+                if 'endpoints' in services[service_id]:
+                    service['endpoints'].append(endpoint)
+                else:
+                    service['endpoints'] = [endpoint]
+            except exception.EndpointNotFound:
+                # remove bad reference from association
+                self.endpoint_filter_api.remove_endpoint_from_project(
+                    entry.endpoint_id, project_id)
+
+        # format catalog
+        catalog = []
+        for service_id, service in services.iteritems():
+            formatted_service = {}
+            formatted_service['id'] = service['id']
+            formatted_service['type'] = service['type']
+            formatted_service['endpoints'] = service['endpoints']
+            catalog.append(formatted_service)
+
+        return catalog

keystone/contrib/endpoint_filter/backends/sql.py

@@ -0,0 +1,83 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2013 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from keystone.common import sql
+from keystone.common.sql import migration
+from keystone import exception
+
+
+class ProjectEndpoint(sql.ModelBase, sql.DictBase):
+    """project-endpoint relationship table."""
+    __tablename__ = 'project_endpoint'
+    attributes = ['endpoint_id', 'project_id']
+    endpoint_id = sql.Column(sql.String(64),
+                             primary_key=True,
+                             nullable=False)
+    project_id = sql.Column(sql.String(64),
+                            primary_key=True,
+                            nullable=False)
+
+
+class EndpointFilter(sql.Base):
+    # Internal interface to manage the database
+
+    def db_sync(self, version=None):
+        migration.db_sync(version=version)
+
+    @sql.handle_conflicts(type='project_endpoint')
+    def add_endpoint_to_project(self, endpoint_id, project_id):
+        session = self.get_session()
+        with session.begin():
+            endpoint_filter_ref = ProjectEndpoint(endpoint_id=endpoint_id,
+                                                  project_id=project_id)
+            session.add(endpoint_filter_ref)
+            session.flush()
+
+    def _get_project_endpoint_ref(self, session, endpoint_id, project_id):
+        endpoint_filter_ref = session.query(ProjectEndpoint).get(
+            (endpoint_id, project_id))
+        if endpoint_filter_ref is None:
+            msg = _('Endpoint %(endpoint_id)s not found in project '
+                    '%(project_id)s') % {'endpoint_id': endpoint_id,
+                                         'project_id': project_id}
+            raise exception.NotFound(msg)
+        return endpoint_filter_ref
+
+    def check_endpoint_in_project(self, endpoint_id, project_id):
+        session = self.get_session()
+        self._get_project_endpoint_ref(session, endpoint_id, project_id)
+
+    def remove_endpoint_from_project(self, endpoint_id, project_id):
+        session = self.get_session()
+        endpoint_filter_ref = self._get_project_endpoint_ref(
+            session, endpoint_id, project_id)
+        with session.begin():
+            session.delete(endpoint_filter_ref)
+            session.flush()
+
+    def list_endpoints_for_project(self, project_id):
+        session = self.get_session()
+        query = session.query(ProjectEndpoint)
+        query = query.filter_by(project_id=project_id)
+        endpoint_filter_refs = query.all()
+        return endpoint_filter_refs
+
+    def list_projects_for_endpoint(self, endpoint_id):
+        session = self.get_session()
+        query = session.query(ProjectEndpoint)
+        query = query.filter_by(endpoint_id=endpoint_id)
+        endpoint_filter_refs = query.all()
+        return endpoint_filter_refs

keystone/contrib/endpoint_filter/configuration.rst

@@ -0,0 +1,42 @@
+..
+      Copyright 2011-2013 OpenStack, Foundation
+      All Rights Reserved.
+
+      Licensed under the Apache License, Version 2.0 (the "License"); you may
+      not use this file except in compliance with the License. You may obtain
+      a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+      Unless required by applicable law or agreed to in writing, software
+      distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+      WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+      License for the specific language governing permissions and limitations
+      under the License.
+
+==================================
+Enabling Endpoint Filter Extension
+==================================
+
+To enable the endpoint filter extension:
+
+1. add the endpoint filter extension catalog driver to the ``[catalog]`` section
+   in ``keystone.conf``. example::
+
+    [catalog]
+    driver = keystone.contrib.endpoint_filter.backends.catalog_sql.EndpointFilterCatalog
+
+2. add the ``endpoint_filter_extension`` filter to the ``api_v3`` pipeline in
+   ``keystone-paste.ini``. example::
+
+    [pipeline:api_v3]
+    pipeline = access_log sizelimit url_normalize token_auth admin_token_auth xml_body json_body ec2_extension s3_extension endpoint_filter_extension service_v3
+
+3. create the endpoint filter extension tables if using the provided sql backend. example::
+    ./bin/keystone-manage db_sync --extension endpoint_filter
+
+4. optional: change ``return_all_endpoints_if_no_filter`` the ``[endpoint_filter]`` section
+   in ``keystone.conf`` to return an empty catalog if no associations are made. example::
+
+    [endpoint_filter]
+    return_all_endpoints_if_no_filter = False

keystone/contrib/endpoint_filter/controllers.py

@@ -0,0 +1,76 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2013 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+from keystone.catalog import controllers as catalog_controllers
+from keystone.common import controller
+from keystone.common import dependency
+from keystone.identity import controllers as identity_controllers
+
+
+@dependency.requires('catalog_api', 'identity_api', 'endpoint_filter_api')
+class EndpointFilterV3Controller(controller.V3Controller):
+
+    @controller.protected
+    def add_endpoint_to_project(self, context, project_id, endpoint_id):
+        """Establishes an association between an endpoint and a project."""
+        # NOTE(gyee): we just need to make sure endpoint and project exist
+        # first. We don't really care whether if project is disabled.
+        # The relationship can still be establed even with a disabled project
+        # as there are no security implications.
+        self.catalog_api.get_endpoint(endpoint_id)
+        self.identity_api.get_project(project_id)
+        # NOTE(gyee): we may need to cleanup any existing project-endpoint
+        # associations here if either project or endpoint is not found.
+        self.endpoint_filter_api.add_endpoint_to_project(endpoint_id,
+                                                         project_id)
+
+    @controller.protected
+    def check_endpoint_in_project(self, context, project_id, endpoint_id):
+        """Verifies endpoint is currently associated with given project."""
+        self.catalog_api.get_endpoint(endpoint_id)
+        self.identity_api.get_project(project_id)
+        # TODO(gyee): we may need to cleanup any existing project-endpoint
+        # associations here if either project or endpoint is not found.
+        self.endpoint_filter_api.check_endpoint_in_project(endpoint_id,
+                                                           project_id)
+
+    @controller.protected
+    def list_endpoints_for_project(self, context, project_id):
+        """Lists all endpoints currently associated with a given project."""
+        self.identity_api.get_project(project_id)
+        refs = self.endpoint_filter_api.list_endpoints_for_project(project_id)
+
+        endpoints = [self.catalog_api.get_endpoint(
+            ref.endpoint_id) for ref in refs]
+        return catalog_controllers.EndpointV3.wrap_collection(context,
+                                                              endpoints)
+
+    @controller.protected
+    def remove_endpoint_from_project(self, context, project_id, endpoint_id):
+        """Remove the endpoint from the association with given project."""
+        self.endpoint_filter_api.remove_endpoint_from_project(endpoint_id,
+                                                              project_id)
+
+    @controller.protected
+    def list_projects_for_endpoint(self, context, endpoint_id):
+        """Return a list of projects associated with the endpoint."""
+        refs = self.endpoint_filter_api.list_project_endpoints(endpoint_id)
+
+        projects = [self.identity_api.get_project(
+            ref.project_id) for ref in refs]
+        return identity_controllers.ProjectV3.wrap_collection(context,
+                                                              projects)

keystone/contrib/endpoint_filter/core.py

@@ -0,0 +1,122 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2013 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+from keystone.common import dependency
+from keystone.common import extension
+from keystone.common import manager
+from keystone import config
+from keystone import exception
+from keystone.openstack.common import log as logging
+
+
+CONF = config.CONF
+LOG = logging.getLogger(__name__)
+
+extension_data = {
+    'name': 'Openstack Keystone Endpoint Filter API',
+    'namespace': 'http://docs.openstack.org/identity/api/ext/'
+                 'OS-EP-FILTER/v1.0',
+    'alias': 'OS-EP-FILTER',
+    'updated': '2013-07-23T12:00:0-00:00',
+    'description': 'Openstack Keystone Endpoint Filter API.',
+    'links': [
+        {
+            'rel': 'describedby',
+            # TODO(ayoung): needs a description
+            'type': 'text/html',
+            'href': 'https://github.com/openstack/identity-api/blob/master'
+                    '/openstack-identity-api/v3/src/markdown/'
+                    'identity-api-v3-os-ep-filter-ext.md',
+        }
+    ]}
+extension.register_admin_extension(extension_data['alias'], extension_data)
+
+
+@dependency.provider('endpoint_filter_api')
+class Manager(manager.Manager):
+    """Default pivot point for the Endpoint Filter backend.
+
+    See :mod:`keystone.common.manager.Manager` for more details on how this
+    dynamically calls the backend.
+
+    """
+
+    def __init__(self):
+        super(Manager, self).__init__(CONF.endpoint_filter.driver)
+
+
+class Driver(object):
+    """Interface description for an Endpoint Filter driver."""
+
+    def add_endpoint_to_project(self, endpoint_id, project_id):
+        """Creates an endpoint to project association.
+
+        :param endpoint_id: identity of endpoint to associate
+        :type endpoint_id: string
+        :param project_id: identity of the project to be associated with
+        :type project_id: string
+        :raises: keystone.exception.Conflict,
+        :returns: None.
+
+        """
+        raise exception.NotImplemented()
+
+    def remove_endpoint_from_project(self, endpoint_id, project_id):
+        """Removes an endpoint to project association.
+
+        :param endpoint_id: identity of endpoint to remove
+        :type endpoint_id: string
+        :param project_id: identity of the project associated with
+        :type project_id: string
+        :raises: exception.NotFound
+        :returns: None.
+
+        """
+        raise exception.NotImplemented()
+
+    def check_endpoint_in_project(self, endpoint_id, project_id):
+        """Checks if an endpoint is associated with a project.
+
+        :param endpoint_id: identity of endpoint to check
+        :type endpoint_id: string
+        :param project_id: identity of the project associated with
+        :type project_id: string
+        :raises: exception.NotFound
+        :returns: None.
+
+        """
+        raise exception.NotImplemented()
+
+    def list_endpoints_for_project(self, project_id):
+        """List all endpoints associated with a project.
+
+        :param project_id: identity of the project to check
+        :type project_id: string
+        :returns: a list of identity endpoint ids or an empty list.
+
+        """
+        raise exception.NotImplemented()
+
+    def list_projects_for_endpoint(self, endpoint_id):
+        """List all projects associated with an endpoint.
+
+        :param endpoint_id: identity of endpoint to check
+        :type endpoint_id: string
+        :returns: a list of projects or an empty list.
+
+        """
+        raise exception.NotImplemented()

keystone/contrib/endpoint_filter/migrate_repo/migrate.cfg

@@ -0,0 +1,25 @@
+[db_settings]
+# Used to identify which repository this database is versioned under.
+# You can use the name of your project.
+repository_id=endpoint_filter
+
+# The name of the database table used to track the schema version.
+# This name shouldn't already be used by your project.
+# If this is changed once a database is under version control, you'll need to
+# change the table name in each database too.
+version_table=migrate_version
+
+# When committing a change script, Migrate will attempt to generate the
+# sql for all supported databases; normally, if one of them fails - probably
+# because you don't have that database installed - it is ignored and the
+# commit continues, perhaps ending successfully.
+# Databases in this list MUST compile successfully during a commit, or the
+# entire commit will fail. List the databases your application will actually
+# be using to ensure your updates to that database work properly.
+# This must be a list; example: ['postgres','sqlite']
+required_dbs=[]
+
+# When creating new change scripts, Migrate will stamp the new script with
+# a version number. By default this is latest_version + 1. You can set this
+# to 'true' to tell Migrate to use the UTC timestamp instead.
+use_timestamp_numbering=False

keystone/contrib/endpoint_filter/migrate_repo/versions/001_add_endpoint_filtering_table.py

@@ -0,0 +1,49 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2013 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import sqlalchemy as sql
+
+
+def upgrade(migrate_engine):
+    # Upgrade operations go here. Don't create your own engine; bind
+    # migrate_engine to your metadata
+    meta = sql.MetaData()
+    meta.bind = migrate_engine
+
+    endpoint_filtering_table = sql.Table(
+        'project_endpoint',
+        meta,
+        sql.Column(
+            'endpoint_id',
+            sql.String(64),
+            primary_key=True,
+            nullable=False),
+        sql.Column(
+            'project_id',
+            sql.String(64),
+            primary_key=True,
+            nullable=False))
+
+    endpoint_filtering_table.create(migrate_engine, checkfirst=True)
+
+
+def downgrade(migrate_engine):
+    meta = sql.MetaData()
+    meta.bind = migrate_engine
+    # Operations to reverse the above upgrade go here.
+    for table_name in ['project_endpoint']:
+        table = sql.Table(table_name, meta, autoload=True)
+        table.drop()

keystone/contrib/endpoint_filter/routers.py

@@ -0,0 +1,47 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2013 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from keystone.common import wsgi
+from keystone.contrib.endpoint_filter import controllers
+
+
+class EndpointFilterExtension(wsgi.ExtensionRouter):
+
+    PATH_PREFIX = '/OS-EP-FILTER'
+    PATH_PROJECT_ENDPOINT = '/projects/{project_id}/endpoints/{endpoint_id}'
+
+    def add_routes(self, mapper):
+        endpoint_filter_controller = controllers.EndpointFilterV3Controller()
+        mapper.connect(self.PATH_PREFIX + '/endpoints/{endpoint_id}/projects',
+                       controller=endpoint_filter_controller,
+                       action='list_projects_for_endpoint',
+                       conditions=dict(method=['GET']))
+        mapper.connect(self.PATH_PREFIX + self.PATH_PROJECT_ENDPOINT,
+                       controller=endpoint_filter_controller,
+                       action='add_endpoint_to_project',
+                       conditions=dict(method=['PUT']))
+        mapper.connect(self.PATH_PREFIX + self.PATH_PROJECT_ENDPOINT,
+                       controller=endpoint_filter_controller,
+                       action='check_endpoint_in_project',
+                       conditions=dict(method=['HEAD']))
+        mapper.connect(self.PATH_PREFIX + '/projects/{project_id}/endpoints',
+                       controller=endpoint_filter_controller,
+                       action='list_endpoints_for_project',
+                       conditions=dict(method=['GET']))
+        mapper.connect(self.PATH_PREFIX + self.PATH_PROJECT_ENDPOINT,
+                       controller=endpoint_filter_controller,
+                       action='remove_endpoint_from_project',
+                       conditions=dict(method=['DELETE']))

keystone/service.py

@@ -23,6 +23,7 @@ from keystone import catalog
 from keystone.common import dependency
 from keystone.common import wsgi
 from keystone import config
+from keystone.contrib import endpoint_filter
 from keystone.contrib import oauth1
 from keystone import controllers
 from keystone import credential
@@ -47,6 +48,7 @@ DRIVERS = dict(
     assignment_api=assignment.Manager(),
     catalog_api=catalog.Manager(),
     credentials_api=credential.Manager(),
+    endpoint_filter_api=endpoint_filter.Manager(),
     identity_api=_IDENTITY_API,
     oauth1_api=oauth1.Manager(),
     policy_api=policy.Manager(),

keystone/tests/core.py

@@ -41,6 +41,7 @@ from keystone.common import sql
 from keystone.common import utils
 from keystone.common import wsgi
 from keystone import config
+from keystone.contrib import endpoint_filter
 from keystone.contrib import oauth1
 from keystone import credential
 from keystone import exception
@@ -264,8 +265,9 @@ class TestCase(NoModule, unittest.TestCase):
         # identity driver is available to the assignment manager because the
         # assignment manager gets the default assignment driver from the
         # identity driver.
-        for manager in [identity, assignment, catalog, credential, policy,
-                        token, token_provider, trust, oauth1]:
+        for manager in [identity, assignment, catalog, credential,
+                        endpoint_filter, policy, token, token_provider,
+                        trust, oauth1]:
             # manager.__name__ is like keystone.xxx[.yyy],
             # converted to xxx[_yyy]
             manager_name = ('%s_api' %

keystone/tests/test_associate_project_endpoint_extension.conf

@@ -0,0 +1,2 @@
+[catalog]
+driver = keystone.contrib.endpoint_filter.backends.catalog_sql.Endpoint_Filter_Catalog

keystone/tests/test_associate_project_endpoint_extension.py

@@ -0,0 +1,454 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2013 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import os
+import uuid
+
+from keystone.common.sql import migration
+from keystone import contrib
+from keystone.openstack.common import importutils
+from keystone.tests import core as test
+
+import test_v3
+
+
+# TODO(gyee): we need to generalize this one and stash it into tests.core
+def _generate_paste_config(filter_name, new_paste_file_name):
+    # Generate a file, based on keystone-paste.ini, that includes
+    # endpoint_filter_extension in the pipeline
+
+    with open(test.etcdir('keystone-paste.ini'), 'r') as f:
+        contents = f.read()
+
+    new_contents = contents.replace(' service_v3',
+                                    ' %s service_v3' % (filter_name))
+
+    with open(new_paste_file_name, 'w') as f:
+        f.write(new_contents)
+
+
+class TestExtensionCase(test_v3.RestfulTestCase):
+
+    EXTENSION_NAME = 'endpoint_filter'
+    EXTENSION_FILTER_NAME = 'endpoint_filter_extension'
+    PASTE_INI = 'keystone-endpoint-filter-paste.ini'
+
+    def setup_database(self):
+        self.conf_files = super(TestExtensionCase, self).config_files()
+        self.conf_files.append(
+            test.testsdir('test_associate_project_endpoint_extension.conf'))
+        super(TestExtensionCase, self).setup_database()
+        package_name = "%s.%s.migrate_repo" % (contrib.__name__,
+                                               self.EXTENSION_NAME)
+        package = importutils.import_module(package_name)
+        self.repo_path = os.path.abspath(
+            os.path.dirname(package.__file__))
+        migration.db_version_control(version=None, repo_path=self.repo_path)
+        migration.db_sync(version=None, repo_path=self.repo_path)
+
+    def setUp(self):
+        self._paste_file_name = test.tmpdir(self.PASTE_INI)
+        _generate_paste_config(self.EXTENSION_FILTER_NAME,
+                               self._paste_file_name)
+        super(TestExtensionCase, self).setUp(app_conf='config:%s' % (
+            self._paste_file_name))
+        self.default_request_url = (
+            '/OS-EP-FILTER/projects/%(project_id)s'
+            '/endpoints/%(endpoint_id)s' % {
+            'project_id': self.default_domain_project_id,
+            'endpoint_id': self.endpoint_id})
+
+    def tearDown(self):
+        super(TestExtensionCase, self).tearDown()
+        os.remove(self._paste_file_name)
+        self.conf_files.pop()
+
+
+class AssociateEndpointProjectFilterCRUDTestCase(TestExtensionCase):
+    """Test OS-EP-FILTER endpoint to project associations extension."""
+
+    # endpoint-project associations crud tests
+    # PUT
+    def test_create_endpoint_project_assoc(self):
+        """PUT /OS-EP-FILTER/projects/{project_id}/endpoints/{endpoint_id}
+
+        Valid endpoint and project id test case.
+
+        """
+        self.put(self.default_request_url,
+                 body='',
+                 expected_status=204)
+
+    def test_create_endpoint_project_assoc_noproj(self):
+        """PUT OS-EP-FILTER/projects/{project_id}/endpoints/{endpoint_id}
+
+        Invalid project id test case.
+
+        """
+        self.put('/OS-EP-FILTER/projects/%(project_id)s'
+                 '/endpoints/%(endpoint_id)s' % {
+                 'project_id': uuid.uuid4().hex,
+                 'endpoint_id': self.endpoint_id},
+                 body='',
+                 expected_status=404)
+
+    def test_create_endpoint_project_assoc_noendp(self):
+        """PUT /OS-EP-FILTER/projects/{project_id}/endpoints/{endpoint_id}
+
+        Invalid endpoint id test case.
+
+        """
+        self.put('/OS-EP-FILTER/projects/%(project_id)s'
+                 '/endpoints/%(endpoint_id)s' % {
+                 'project_id': self.default_domain_project_id,
+                 'endpoint_id': uuid.uuid4().hex},
+                 body='',
+                 expected_status=404)
+
+    def test_create_endpoint_project_assoc_unexpected_body(self):
+        """PUT /OS-EP-FILTER/projects/{project_id}/endpoints/{endpoint_id}
+
+        Unexpected body in request. The body should be ignored.
+
+        """
+        self.put(self.default_request_url,
+                 body={'project_id': self.default_domain_project_id},
+                 expected_status=204)
+
+    # HEAD
+    def test_check_endpoint_project_assoc(self):
+        """HEAD /OS-EP-FILTER/projects/{project_id}/endpoints/{endpoint_id}
+
+        Valid project and endpoint id test case.
+
+        """
+        self.put(self.default_request_url,
+                 body='',
+                 expected_status=204)
+        self.head('/OS-EP-FILTER/projects/%(project_id)s'
+                  '/endpoints/%(endpoint_id)s' % {
+                  'project_id': self.default_domain_project_id,
+                  'endpoint_id': self.endpoint_id},
+                  expected_status=204)
+
+    def test_check_endpoint_project_assoc_noproj(self):
+        """HEAD /OS-EP-FILTER/projects/{project_id}/endpoints/{endpoint_id}
+
+        Invalid project id test case.
+
+        """
+        self.put(self.default_request_url)
+        self.head('/OS-EP-FILTER/projects/%(project_id)s'
+                  '/endpoints/%(endpoint_id)s' % {
+                  'project_id': uuid.uuid4().hex,
+                  'endpoint_id': self.endpoint_id},
+                  body='',
+                  expected_status=404)
+
+    def test_check_endpoint_project_assoc_noendp(self):
+        """HEAD /OS-EP-FILTER/projects/{project_id}/endpoints/{endpoint_id}
+
+        Invalid endpoint id test case.
+
+        """
+        self.put(self.default_request_url)
+        self.head('/OS-EP-FILTER/projects/%(project_id)s'
+                  '/endpoints/%(endpoint_id)s' % {
+                  'project_id': self.default_domain_project_id,
+                  'endpoint_id': uuid.uuid4().hex},
+                  body='',
+                  expected_status=404)
+
+    # GET
+    def test_get_endpoint_project_assoc(self):
+        """GET /OS-EP-FILTER/projects/{project_id}/endpoints success."""
+        self.put(self.default_request_url)
+        r = self.get('/OS-EP-FILTER/projects/%(project_id)s/endpoints' % {
+                     'project_id': self.default_domain_project_id})
+        self.assertValidEndpointListResponse(r, self.endpoint)
+
+    def test_get_endpoint_project_assoc_noproj(self):
+        """GET /OS-EP-FILTER/projects/{project_id}/endpoints no project."""
+        self.put(self.default_request_url)
+        self.get('/OS-EP-FILTER/projects/%(project_id)s/endpoints' % {
+                 'project_id': uuid.uuid4().hex},
+                 body='',
+                 expected_status=404)
+
+    # DELETE
+    def test_remove_endpoint_project_assoc(self):
+        """DELETE /OS-EP-FILTER/projects/{project_id}/endpoints/{endpoint_id}
+
+        Valid project id and endpoint id test case.
+
+        """
+        self.put(self.default_request_url)
+        self.delete('/OS-EP-FILTER/projects/%(project_id)s'
+                    '/endpoints/%(endpoint_id)s' % {
+                    'project_id': self.default_domain_project_id,
+                    'endpoint_id': self.endpoint_id},
+                    expected_status=204)
+
+    def test_remove_endpoint_project_assoc_noproj(self):
+        """DELETE /OS-EP-FILTER/projects/{project_id}/endpoints/{endpoint_id}
+
+        Invalid project id test case.
+
+        """
+        self.put(self.default_request_url)
+        self.delete('/OS-EP-FILTER/projects/%(project_id)s'
+                    '/endpoints/%(endpoint_id)s' % {
+                    'project_id': uuid.uuid4().hex,
+                    'endpoint_id': self.endpoint_id},
+                    body='',
+                    expected_status=404)
+
+    def test_remove_endpoint_project_assoc_noendp(self):
+        """DELETE /OS-EP-FILTER/projects/{project_id}/endpoints/{endpoint_id}
+
+        Invalid endpoint id test case.
+
+        """
+        self.put(self.default_request_url)
+        self.delete('/OS-EP-FILTER/projects/%(project_id)s'
+                    '/endpoints/%(endpoint_id)s' % {
+                    'project_id': self.default_domain_project_id,
+                    'endpoint_id': uuid.uuid4().hex},
+                    body='',
+                    expected_status=404)
+
+
+class AssociateProjectEndpointFilterTokenRequestTestCase(TestExtensionCase):
+    """Test OS-EP-FILTER catalog filtering extension."""
+
+    def test_default_project_id_scoped_token_with_user_id_ep_filter(self):
+        # create a second project to work with
+        ref = self.new_project_ref(domain_id=self.domain_id)
+        r = self.post('/projects', body={'project': ref})
+        project = self.assertValidProjectResponse(r, ref)
+
+        # grant the user a role on the project
+        self.put(
+            '/projects/%(project_id)s/users/%(user_id)s/roles/%(role_id)s' % {
+                'user_id': self.user['id'],
+                'project_id': project['id'],
+                'role_id': self.role['id']})
+
+        # set the user's preferred project
+        body = {'user': {'default_project_id': project['id']}}
+        r = self.patch('/users/%(user_id)s' % {
+            'user_id': self.user['id']},
+            body=body)
+        self.assertValidUserResponse(r)
+
+        # add one endpoint to the project
+        self.put('/OS-EP-FILTER/projects/%(project_id)s'
+                 '/endpoints/%(endpoint_id)s' % {
+                 'project_id': project['id'],
+                 'endpoint_id': self.endpoint_id},
+                 body='',
+                 expected_status=204)
+
+        # attempt to authenticate without requesting a project
+        auth_data = self.build_authentication_request(
+            user_id=self.user['id'],
+            password=self.user['password'])
+        r = self.post('/auth/tokens', body=auth_data)
+        self.assertValidProjectScopedTokenResponse(
+            r,
+            require_catalog=True,
+            endpoint_filter=True,
+            ep_filter_assoc=1)
+        self.assertEqual(r.result['token']['project']['id'], project['id'])
+
+    def test_implicit_project_id_scoped_token_with_user_id_ep_filter(self):
+        # attempt to authenticate without requesting a project
+
+        # add one endpoint to default project
+        self.put('/OS-EP-FILTER/projects/%(project_id)s'
+                 '/endpoints/%(endpoint_id)s' % {
+                 'project_id': self.project['id'],
+                 'endpoint_id': self.endpoint_id},
+                 body='',
+                 expected_status=204)
+
+        auth_data = self.build_authentication_request(
+            user_id=self.user['id'],
+            password=self.user['password'],
+            project_id=self.project['id'])
+        r = self.post('/auth/tokens', body=auth_data)
+        self.assertValidProjectScopedTokenResponse(
+            r,
+            require_catalog=True,
+            endpoint_filter=True,
+            ep_filter_assoc=1)
+        self.assertEqual(r.result['token']['project']['id'],
+                         self.project['id'])
+
+    def test_default_project_id_scoped_token_ep_filter_no_catalog(self):
+        # create a second project to work with
+        ref = self.new_project_ref(domain_id=self.domain_id)
+        r = self.post('/projects', body={'project': ref})
+        project = self.assertValidProjectResponse(r, ref)
+
+        # grant the user a role on the project
+        self.put(
+            '/projects/%(project_id)s/users/%(user_id)s/roles/%(role_id)s' % {
+                'user_id': self.user['id'],
+                'project_id': project['id'],
+                'role_id': self.role['id']})
+
+        # set the user's preferred project
+        body = {'user': {'default_project_id': project['id']}}
+        r = self.patch('/users/%(user_id)s' % {
+            'user_id': self.user['id']},
+            body=body)
+        self.assertValidUserResponse(r)
+
+        # add one endpoint to the project
+        self.put('/OS-EP-FILTER/projects/%(project_id)s'
+                 '/endpoints/%(endpoint_id)s' % {
+                 'project_id': project['id'],
+                 'endpoint_id': self.endpoint_id},
+                 body='',
+                 expected_status=204)
+
+        # attempt to authenticate without requesting a project
+        auth_data = self.build_authentication_request(
+            user_id=self.user['id'],
+            password=self.user['password'])
+        r = self.post('/auth/tokens?nocatalog', body=auth_data)
+        self.assertValidProjectScopedTokenResponse(
+            r,
+            require_catalog=False,
+            endpoint_filter=True,
+            ep_filter_assoc=1)
+        self.assertEqual(r.result['token']['project']['id'], project['id'])
+
+    def test_implicit_project_id_scoped_token_ep_filter_no_catalog(self):
+        # attempt to authenticate without requesting a project
+
+        # add one endpoint to default project
+        self.put('/OS-EP-FILTER/projects/%(project_id)s'
+                 '/endpoints/%(endpoint_id)s' % {
+                 'project_id': self.project['id'],
+                 'endpoint_id': self.endpoint_id},
+                 body='',
+                 expected_status=204)
+
+        auth_data = self.build_authentication_request(
+            user_id=self.user['id'],
+            password=self.user['password'],
+            project_id=self.project['id'])
+        r = self.post('/auth/tokens?nocatalog', body=auth_data)
+        self.assertValidProjectScopedTokenResponse(
+            r,
+            require_catalog=False,
+            endpoint_filter=True,
+            ep_filter_assoc=1)
+        self.assertEqual(r.result['token']['project']['id'],
+                         self.project['id'])
+
+    def test_default_project_id_scoped_token_ep_filter_full_catalog(self):
+        # create a second project to work with
+        ref = self.new_project_ref(domain_id=self.domain_id)
+        r = self.post('/projects', body={'project': ref})
+        project = self.assertValidProjectResponse(r, ref)
+
+        # grant the user a role on the project
+        self.put(
+            '/projects/%(project_id)s/users/%(user_id)s/roles/%(role_id)s' % {
+                'user_id': self.user['id'],
+                'project_id': project['id'],
+                'role_id': self.role['id']})
+
+        # set the user's preferred project
+        body = {'user': {'default_project_id': project['id']}}
+        r = self.patch('/users/%(user_id)s' % {
+            'user_id': self.user['id']},
+            body=body)
+        self.assertValidUserResponse(r)
+
+        # attempt to authenticate without requesting a project
+        auth_data = self.build_authentication_request(
+            user_id=self.user['id'],
+            password=self.user['password'])
+        r = self.post('/auth/tokens?nocatalog', body=auth_data)
+        self.assertValidProjectScopedTokenResponse(
+            r,
+            require_catalog=False,
+            endpoint_filter=True)
+        self.assertEqual(r.result['token']['project']['id'], project['id'])
+
+    def test_implicit_project_id_scoped_token_ep_filter_full_catalog(self):
+        # attempt to authenticate without requesting a project
+
+        auth_data = self.build_authentication_request(
+            user_id=self.user['id'],
+            password=self.user['password'],
+            project_id=self.project['id'])
+        r = self.post('/auth/tokens?nocatalog', body=auth_data)
+        self.assertValidProjectScopedTokenResponse(
+            r,
+            require_catalog=False,
+            endpoint_filter=True,)
+        self.assertEqual(r.result['token']['project']['id'],
+                         self.project['id'])
+
+    def test_implicit_project_id_scoped_token_handling_bad_reference(self):
+        # handling the case with an endpoint that is not associate with
+
+        # add first endpoint to default project
+        self.put('/OS-EP-FILTER/projects/%(project_id)s'
+                 '/endpoints/%(endpoint_id)s' % {
+                 'project_id': self.project['id'],
+                 'endpoint_id': self.endpoint_id},
+                 body='',
+                 expected_status=204)
+
+        # create a second temporary endpoint
+        self.endpoint_id2 = uuid.uuid4().hex
+        self.endpoint2 = self.new_endpoint_ref(service_id=self.service_id)
+        self.endpoint2['id'] = self.endpoint_id2
+        self.catalog_api.create_endpoint(
+            self.endpoint_id2,
+            self.endpoint2.copy())
+
+        # add second endpoint to default project
+        self.put('/OS-EP-FILTER/projects/%(project_id)s'
+                 '/endpoints/%(endpoint_id)s' % {
+                 'project_id': self.project['id'],
+                 'endpoint_id': self.endpoint_id2},
+                 body='',
+                 expected_status=204)
+
+        # remove the temporary reference
+        # this will create inconsistency in the endpoint filter table
+        # which is fixed during the catalog creation for token request
+        self.catalog_api.delete_endpoint(self.endpoint_id2)
+
+        auth_data = self.build_authentication_request(
+            user_id=self.user['id'],
+            password=self.user['password'],
+            project_id=self.project['id'])
+        r = self.post('/auth/tokens', body=auth_data)
+        self.assertValidProjectScopedTokenResponse(
+            r,
+            require_catalog=True,
+            endpoint_filter=True,
+            ep_filter_assoc=1)
+        self.assertEqual(r.result['token']['project']['id'],
+                         self.project['id'])

keystone/tests/test_drivers.py

@@ -3,6 +3,7 @@ import unittest2 as unittest
 
 from keystone import assignment
 from keystone import catalog
+from keystone.contrib import endpoint_filter
 from keystone.contrib import oauth1
 from keystone import exception
 from keystone import identity
@@ -60,3 +61,7 @@ class TestDrivers(unittest.TestCase):
     def test_oauth1_driver_unimplemented(self):
         interface = oauth1.Driver()
         self.assertInterfaceNotImplemented(interface)
+
+    def test_endpoint_filter_driver_unimplemented(self):
+        interface = endpoint_filter.Driver()
+        self.assertInterfaceNotImplemented(interface)

keystone/tests/test_sql_migrate_extensions.py

@@ -26,6 +26,8 @@ To run these tests against a live database:
     all data will be lost.
 """
 
+
+from keystone.contrib import endpoint_filter
 from keystone.contrib import example
 from keystone.contrib import oauth1
 
@@ -108,3 +110,21 @@ class SqlUpgradeOAuth1Extension(test_sql_upgrade.SqlMigrateBase):
         self.assertTableDoesNotExist('consumer')
         self.assertTableDoesNotExist('request_token')
         self.assertTableDoesNotExist('access_token')
+
+
+class EndpointFilterExtension(test_sql_upgrade.SqlMigrateBase):
+    def repo_package(self):
+        return endpoint_filter
+
+    def test_upgrade(self):
+        self.assertTableDoesNotExist('project_endpoint')
+        self.upgrade(1, repository=self.repo_path)
+        self.assertTableColumns('project_endpoint',
+                                ['endpoint_id', 'project_id'])
+
+    def test_downgrade(self):
+        self.upgrade(1, repository=self.repo_path)
+        self.assertTableColumns('project_endpoint',
+                                ['endpoint_id', 'project_id'])
+        self.downgrade(0, repository=self.repo_path)
+        self.assertTableDoesNotExist('project_endpoint')

keystone/tests/test_v3.py

@@ -30,7 +30,13 @@ class RestfulTestCase(test_content_types.RestfulTestCase):
     def config_files(self):
         return self._config_file_list
 
-    def setUp(self, load_sample_data=True):
+    def setup_database(self):
+        test.setup_test_database()
+
+    def teardown_database(self):
+        test.teardown_test_database()
+
+    def setUp(self, load_sample_data=True, app_conf='keystone'):
         """Setup for v3 Restful Test Cases.
 
         If a child class wants to create their own sample data
@@ -40,13 +46,13 @@ class RestfulTestCase(test_content_types.RestfulTestCase):
         """
         self.config(self.config_files())
 
-        test.setup_test_database()
+        self.setup_database()
         self.load_backends()
 
         self.public_app = webtest.TestApp(
-            self.loadapp('keystone', name='main'))
+            self.loadapp(app_conf, name='main'))
         self.admin_app = webtest.TestApp(
-            self.loadapp('keystone', name='admin'))
+            self.loadapp(app_conf, name='admin'))
 
         if load_sample_data:
             self.domain_id = uuid.uuid4().hex
@@ -97,15 +103,29 @@ class RestfulTestCase(test_content_types.RestfulTestCase):
                 self.default_domain_user_id, self.project_id,
                 self.role_id)
 
-        self.public_server = self.serveapp('keystone', name='main')
-        self.admin_server = self.serveapp('keystone', name='admin')
+            self.service_id = uuid.uuid4().hex
+            self.service = self.new_service_ref()
+            self.service['id'] = self.service_id
+            self.catalog_api.create_service(
+                self.service_id,
+                self.service.copy())
+
+            self.endpoint_id = uuid.uuid4().hex
+            self.endpoint = self.new_endpoint_ref(service_id=self.service_id)
+            self.endpoint['id'] = self.endpoint_id
+            self.catalog_api.create_endpoint(
+                self.endpoint_id,
+                self.endpoint.copy())
+
+        self.public_server = self.serveapp(app_conf, name='main')
+        self.admin_server = self.serveapp(app_conf, name='admin')
 
     def tearDown(self):
         self.public_server.kill()
         self.admin_server.kill()
         self.public_server = None
         self.admin_server = None
-        test.teardown_test_database()
+        self.teardown_database()
         # need to reset the plug-ins
         auth.controllers.AUTH_METHODS = {}
         #drop the policy rules
@@ -448,10 +468,17 @@ class RestfulTestCase(test_content_types.RestfulTestCase):
 
     def assertValidScopedTokenResponse(self, r, *args, **kwargs):
         require_catalog = kwargs.pop('require_catalog', True)
+        endpoint_filter = kwargs.pop('endpoint_filter', False)
+        ep_filter_assoc = kwargs.pop('ep_filter_assoc', 0)
         token = self.assertValidTokenResponse(r, *args, **kwargs)
 
         if require_catalog:
             self.assertIn('catalog', token)
+            # sub test for the OS-EP-FILTER extension enabled
+            if endpoint_filter:
+                # verify the catalog hs no more than the endpoints
+                # associated in the catalog using the ep filter assoc
+                self.assertTrue(len(token['catalog']) < ep_filter_assoc + 1)
         else:
             self.assertNotIn('catalog', token)
 

keystone/tests/test_v3_catalog.py

@@ -1,4 +1,3 @@
-import uuid
 
 import test_v3
 
@@ -9,20 +8,6 @@ class CatalogTestCase(test_v3.RestfulTestCase):
     def setUp(self):
         super(CatalogTestCase, self).setUp()
 
-        self.service_id = uuid.uuid4().hex
-        self.service = self.new_service_ref()
-        self.service['id'] = self.service_id
-        self.catalog_api.create_service(
-            self.service_id,
-            self.service.copy())
-
-        self.endpoint_id = uuid.uuid4().hex
-        self.endpoint = self.new_endpoint_ref(service_id=self.service_id)
-        self.endpoint['id'] = self.endpoint_id
-        self.catalog_api.create_endpoint(
-            self.endpoint_id,
-            self.endpoint.copy())
-
     # service crud tests
 
     def test_create_service(self):