Create V9 version of resource driver interface

In preparation for projects acting as domains (which will result
in driver interface changes), a V9 version of the resource
interface is created, along with the wrapper support scaffolding
for V8 drivers.

Partially Implements: blueprint reseller

Change-Id: Iec6f7fe2347b64c8f721e968b816e6c1b4332d0a

keystone/resource/V8_backends/sql.py

@@ -0,0 +1,262 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from oslo_config import cfg
+from oslo_log import log
+
+from keystone.common import clean
+from keystone.common import driver_hints
+from keystone.common import sql
+from keystone import exception
+from keystone.i18n import _LE
+from keystone import resource as keystone_resource
+
+
+CONF = cfg.CONF
+LOG = log.getLogger(__name__)
+
+
+class Resource(keystone_resource.ResourceDriverV8):
+
+    def default_assignment_driver(self):
+        return 'sql'
+
+    def _get_project(self, session, project_id):
+        project_ref = session.query(Project).get(project_id)
+        if project_ref is None:
+            raise exception.ProjectNotFound(project_id=project_id)
+        return project_ref
+
+    def get_project(self, tenant_id):
+        with sql.transaction() as session:
+            return self._get_project(session, tenant_id).to_dict()
+
+    def get_project_by_name(self, tenant_name, domain_id):
+        with sql.transaction() as session:
+            query = session.query(Project)
+            query = query.filter_by(name=tenant_name)
+            query = query.filter_by(domain_id=domain_id)
+            try:
+                project_ref = query.one()
+            except sql.NotFound:
+                raise exception.ProjectNotFound(project_id=tenant_name)
+            return project_ref.to_dict()
+
+    @driver_hints.truncated
+    def list_projects(self, hints):
+        with sql.transaction() as session:
+            query = session.query(Project)
+            project_refs = sql.filter_limit_query(Project, query, hints)
+            return [project_ref.to_dict() for project_ref in project_refs]
+
+    def list_projects_from_ids(self, ids):
+        if not ids:
+            return []
+        else:
+            with sql.transaction() as session:
+                query = session.query(Project)
+                query = query.filter(Project.id.in_(ids))
+                return [project_ref.to_dict() for project_ref in query.all()]
+
+    def list_project_ids_from_domain_ids(self, domain_ids):
+        if not domain_ids:
+            return []
+        else:
+            with sql.transaction() as session:
+                query = session.query(Project.id)
+                query = (
+                    query.filter(Project.domain_id.in_(domain_ids)))
+                return [x.id for x in query.all()]
+
+    def list_projects_in_domain(self, domain_id):
+        with sql.transaction() as session:
+            self._get_domain(session, domain_id)
+            query = session.query(Project)
+            project_refs = query.filter_by(domain_id=domain_id)
+            return [project_ref.to_dict() for project_ref in project_refs]
+
+    def _get_children(self, session, project_ids):
+        query = session.query(Project)
+        query = query.filter(Project.parent_id.in_(project_ids))
+        project_refs = query.all()
+        return [project_ref.to_dict() for project_ref in project_refs]
+
+    def list_projects_in_subtree(self, project_id):
+        with sql.transaction() as session:
+            children = self._get_children(session, [project_id])
+            subtree = []
+            examined = set([project_id])
+            while children:
+                children_ids = set()
+                for ref in children:
+                    if ref['id'] in examined:
+                        msg = _LE('Circular reference or a repeated '
+                                  'entry found in projects hierarchy - '
+                                  '%(project_id)s.')
+                        LOG.error(msg, {'project_id': ref['id']})
+                        return
+                    children_ids.add(ref['id'])
+
+                examined.update(children_ids)
+                subtree += children
+                children = self._get_children(session, children_ids)
+            return subtree
+
+    def list_project_parents(self, project_id):
+        with sql.transaction() as session:
+            project = self._get_project(session, project_id).to_dict()
+            parents = []
+            examined = set()
+            while project.get('parent_id') is not None:
+                if project['id'] in examined:
+                    msg = _LE('Circular reference or a repeated '
+                              'entry found in projects hierarchy - '
+                              '%(project_id)s.')
+                    LOG.error(msg, {'project_id': project['id']})
+                    return
+
+                examined.add(project['id'])
+                parent_project = self._get_project(
+                    session, project['parent_id']).to_dict()
+                parents.append(parent_project)
+                project = parent_project
+            return parents
+
+    def is_leaf_project(self, project_id):
+        with sql.transaction() as session:
+            project_refs = self._get_children(session, [project_id])
+            return not project_refs
+
+    # CRUD
+    @sql.handle_conflicts(conflict_type='project')
+    def create_project(self, tenant_id, tenant):
+        tenant['name'] = clean.project_name(tenant['name'])
+        with sql.transaction() as session:
+            tenant_ref = Project.from_dict(tenant)
+            session.add(tenant_ref)
+            return tenant_ref.to_dict()
+
+    @sql.handle_conflicts(conflict_type='project')
+    def update_project(self, tenant_id, tenant):
+        if 'name' in tenant:
+            tenant['name'] = clean.project_name(tenant['name'])
+
+        with sql.transaction() as session:
+            tenant_ref = self._get_project(session, tenant_id)
+            old_project_dict = tenant_ref.to_dict()
+            for k in tenant:
+                old_project_dict[k] = tenant[k]
+            new_project = Project.from_dict(old_project_dict)
+            for attr in Project.attributes:
+                if attr != 'id':
+                    setattr(tenant_ref, attr, getattr(new_project, attr))
+            tenant_ref.extra = new_project.extra
+            return tenant_ref.to_dict(include_extra_dict=True)
+
+    @sql.handle_conflicts(conflict_type='project')
+    def delete_project(self, tenant_id):
+        with sql.transaction() as session:
+            tenant_ref = self._get_project(session, tenant_id)
+            session.delete(tenant_ref)
+
+    # domain crud
+
+    @sql.handle_conflicts(conflict_type='domain')
+    def create_domain(self, domain_id, domain):
+        with sql.transaction() as session:
+            ref = Domain.from_dict(domain)
+            session.add(ref)
+        return ref.to_dict()
+
+    @driver_hints.truncated
+    def list_domains(self, hints):
+        with sql.transaction() as session:
+            query = session.query(Domain)
+            refs = sql.filter_limit_query(Domain, query, hints)
+            return [ref.to_dict() for ref in refs]
+
+    def list_domains_from_ids(self, ids):
+        if not ids:
+            return []
+        else:
+            with sql.transaction() as session:
+                query = session.query(Domain)
+                query = query.filter(Domain.id.in_(ids))
+                domain_refs = query.all()
+                return [domain_ref.to_dict() for domain_ref in domain_refs]
+
+    def _get_domain(self, session, domain_id):
+        ref = session.query(Domain).get(domain_id)
+        if ref is None:
+            raise exception.DomainNotFound(domain_id=domain_id)
+        return ref
+
+    def get_domain(self, domain_id):
+        with sql.transaction() as session:
+            return self._get_domain(session, domain_id).to_dict()
+
+    def get_domain_by_name(self, domain_name):
+        with sql.transaction() as session:
+            try:
+                ref = (session.query(Domain).
+                       filter_by(name=domain_name).one())
+            except sql.NotFound:
+                raise exception.DomainNotFound(domain_id=domain_name)
+            return ref.to_dict()
+
+    @sql.handle_conflicts(conflict_type='domain')
+    def update_domain(self, domain_id, domain):
+        with sql.transaction() as session:
+            ref = self._get_domain(session, domain_id)
+            old_dict = ref.to_dict()
+            for k in domain:
+                old_dict[k] = domain[k]
+            new_domain = Domain.from_dict(old_dict)
+            for attr in Domain.attributes:
+                if attr != 'id':
+                    setattr(ref, attr, getattr(new_domain, attr))
+            ref.extra = new_domain.extra
+            return ref.to_dict()
+
+    def delete_domain(self, domain_id):
+        with sql.transaction() as session:
+            ref = self._get_domain(session, domain_id)
+            session.delete(ref)
+
+
+class Domain(sql.ModelBase, sql.DictBase):
+    __tablename__ = 'domain'
+    attributes = ['id', 'name', 'enabled']
+    id = sql.Column(sql.String(64), primary_key=True)
+    name = sql.Column(sql.String(64), nullable=False)
+    enabled = sql.Column(sql.Boolean, default=True, nullable=False)
+    extra = sql.Column(sql.JsonBlob())
+    __table_args__ = (sql.UniqueConstraint('name'), {})
+
+
+class Project(sql.ModelBase, sql.DictBase):
+    __tablename__ = 'project'
+    attributes = ['id', 'name', 'domain_id', 'description', 'enabled',
+                  'parent_id', 'is_domain']
+    id = sql.Column(sql.String(64), primary_key=True)
+    name = sql.Column(sql.String(64), nullable=False)
+    domain_id = sql.Column(sql.String(64), sql.ForeignKey('domain.id'),
+                           nullable=False)
+    description = sql.Column(sql.Text())
+    enabled = sql.Column(sql.Boolean)
+    extra = sql.Column(sql.JsonBlob())
+    parent_id = sql.Column(sql.String(64), sql.ForeignKey('project.id'))
+    is_domain = sql.Column(sql.Boolean, default=False, nullable=False,
+                           server_default='0')
+    # Unique constraint across two columns to create the separation
+    # rather than just only 'name' being unique
+    __table_args__ = (sql.UniqueConstraint('domain_id', 'name'), {})

keystone/resource/backends/ldap.py

@@ -30,7 +30,7 @@ from keystone import resource
 CONF = cfg.CONF
 
 
-class Resource(resource.ResourceDriverV8):
+class Resource(resource.ResourceDriverV9):
     @versionutils.deprecated(
         versionutils.deprecated.LIBERTY,
         remove_in=+1,

keystone/resource/backends/sql.py

@@ -25,7 +25,7 @@ CONF = cfg.CONF
 LOG = log.getLogger(__name__)
 
 
-class Resource(keystone_resource.ResourceDriverV8):
+class Resource(keystone_resource.ResourceDriverV9):
 
     def default_assignment_driver(self):
         return 'sql'

keystone/resource/core.py

@@ -16,6 +16,7 @@ import abc
 
 from oslo_config import cfg
 from oslo_log import log
+from oslo_log import versionutils
 import six
 
 from keystone.common import cache
@@ -70,6 +71,13 @@ class Manager(manager.Manager):
 
         super(Manager, self).__init__(resource_driver)
 
+        # Make sure it is a driver version we support, and if it is a legacy
+        # driver, then wrap it.
+        if isinstance(self.driver, ResourceDriverV8):
+            self.driver = V9ResourceWrapperForV8Driver(self.driver)
+        elif not isinstance(self.driver, ResourceDriverV9):
+            raise exception.UnsupportedDriverVersion(driver=resource_driver)
+
     def _get_hierarchy_depth(self, parents_list):
         return len(parents_list) + 1
 
@@ -659,8 +667,16 @@ class Manager(manager.Manager):
         pass
 
 
+# The ResourceDriverBase class is the set of driver methods from earlier
+# drivers that we still support, that have not been removed or modified. This
+# class is then used to created the augmented V8 and V9 version abstract driver
+# classes, without having to duplicate a lot of abstract method signatures.
+# If you remove a method from V9, then move the abstact methods from this Base
+# class to the V8 class. Do not modify any of the method signatures in the Base
+# class - changes should only be made in the V8 and subsequent classes.
+
 @six.add_metaclass(abc.ABCMeta)
-class ResourceDriverV8(object):
+class ResourceDriverBase(object):
 
     def _get_list_limit(self):
         return CONF.resource.list_limit or CONF.list_limit
@@ -923,6 +939,118 @@ class ResourceDriverV8(object):
             raise exception.DomainNotFound(domain_id=domain_id)
 
 
+class ResourceDriverV8(ResourceDriverBase):
+    """Removed or redefined methods from V8.
+
+    Move the abstract methods of any methods removed or modified in later
+    versions of the driver from ResourceDriverBase to here. We maintain this
+    so that legacy drivers, which will be a subclass of ResourceDriverV8, can
+    still reference them.
+
+    """
+
+    pass
+
+
+class ResourceDriverV9(ResourceDriverBase):
+    """New or redefined methods from V8.
+
+    Add any new V9 abstract methods (or those with modified signatures) to
+    this class.
+
+    """
+
+    pass
+
+
+class V9ResourceWrapperForV8Driver(ResourceDriverV9):
+    """Wrapper class to supported a V8 legacy driver.
+
+    In order to support legacy drivers without having to make the manager code
+    driver-version aware, we wrap legacy drivers so that they look like the
+    latest version. For the various changes made in a new driver, here are the
+    actions needed in this wrapper:
+
+    Method removed from new driver - remove the call-through method from this
+                                     class, since the manager will no longer be
+                                     calling it.
+    Method signature (or meaning) changed - wrap the old method in a new
+                                            signature here, and munge the input
+                                            and output parameters accordingly.
+    New method added to new driver - add a method to implement the new
+                                     functionality here if possible. If that is
+                                     not possible, then return NotImplemented,
+                                     since we do not guarantee to support new
+                                     functionality with legacy drivers.
+
+    """
+
+    @versionutils.deprecated(
+        as_of=versionutils.deprecated.MITAKA,
+        what='keystone.resource.ResourceDriverV8',
+        in_favor_of='keystone.resource.ResourceDriverV9',
+        remove_in=+2)
+    def __init__(self, wrapped_driver):
+        self.driver = wrapped_driver
+
+    def get_project_by_name(self, tenant_name, domain_id):
+        return self.driver.get_project_by_name(tenant_name, domain_id)
+
+    def create_domain(self, domain_id, domain):
+        return self.driver.create_domain(domain_id, domain)
+
+    def list_domains(self, hints):
+        return self.driver.list_domains(hints)
+
+    def list_domains_from_ids(self, domain_ids):
+        return self.driver.list_domains_from_ids(domain_ids)
+
+    def get_domain(self, domain_id):
+        return self.driver.get_domain(domain_id)
+
+    def get_domain_by_name(self, domain_name):
+        return self.driver.get_domain_by_name(domain_name)
+
+    def update_domain(self, domain_id, domain):
+        return self.driver.update_domain(domain_id, domain)
+
+    def delete_domain(self, domain_id):
+        self.driver.delete_domain(domain_id)
+
+    def create_project(self, project_id, project):
+        return self.driver.create_project(project_id, project)
+
+    def list_projects(self, hints):
+        return self.driver.list_projects(hints)
+
+    def list_projects_from_ids(self, project_ids):
+        return self.driver.list_projects_from_ids(project_ids)
+
+    def list_project_ids_from_domain_ids(self, domain_ids):
+        return self.driver.list_project_ids_from_domain_ids(domain_ids)
+
+    def list_projects_in_domain(self, domain_id):
+        return self.driver.list_projects_in_domain(domain_id)
+
+    def get_project(self, project_id):
+        return self.driver.get_project(project_id)
+
+    def update_project(self, project_id, project):
+        return self.driver.update_project(project_id, project)
+
+    def delete_project(self, project_id):
+        self.driver.delete_project(project_id)
+
+    def list_project_parents(self, project_id):
+        return self.driver.list_project_parents(project_id)
+
+    def list_projects_in_subtree(self, project_id):
+        return self.driver.list_projects_in_subtree(project_id)
+
+    def is_leaf_project(self, project_id):
+        return self.driver.is_leaf_project(project_id)
+
+
 Driver = manager.create_legacy_driver(ResourceDriverV8)
 
 

keystone/tests/unit/backend/legacy_drivers/resource/V8/sql.py

@@ -0,0 +1,30 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from keystone.tests.unit import test_backend_sql
+
+
+class SqlIdentityV8(test_backend_sql.SqlIdentity):
+    """Test that a V8 driver still passes the same tests.
+
+    We use the SQL driver as an example of a V8 legacy driver.
+
+    """
+
+    def config_overrides(self):
+        super(SqlIdentityV8, self).config_overrides()
+        # V8 SQL specific driver overrides
+        self.config_fixture.config(
+            group='resource',
+            driver='keystone.resource.V8_backends.sql.Resource')
+        self.use_specific_sql_driver_version(
+            'keystone.resource', 'backends', 'V8_')

releasenotes/notes/V9ResourceDriver-26716f97c0cc1a80.yaml

@@ -0,0 +1,8 @@
+---
+upgrade:
+  - The V8 Resource driver interface is deprecated, but still supported in
+    this release, so any custom drivers based on the V8 interface should still
+    work.
+other:
+  - Support for the V8 Resource driver interface is planned to be removed in
+    the 'O' release of OpenStack.

tox.ini

@@ -81,6 +81,8 @@ commands =
       keystone/tests/unit/backend/legacy_drivers/role/V8/sql.py
   nosetests -v \
       keystone/tests/unit/backend/legacy_drivers/federation/V8/api_v3.py
+  nosetests -v \
+      keystone/tests/unit/backend/legacy_drivers/resource/V8/sql.py
 
 [testenv:pep8]
 commands =