Add scheduler for pools

This adds a scheduler to central to decide what pool to place a newly
created zone in.

Change-Id: Ie4146212209fa4b22bc271e3f4ce76104090ac9b

designate/central/service.py

@@ -47,6 +47,7 @@ from designate import objects
 from designate import policy
 from designate import quota
 from designate import service
+from designate import scheduler
 from designate import utils
 from designate import storage
 from designate.mdns import rpcapi as mdns_rpcapi
@@ -270,6 +271,13 @@ class Service(service.RPCService, service.Service):
 
         self.network_api = network_api.get_network_api(cfg.CONF.network_api)
 
+    @property
+    def scheduler(self):
+        if not hasattr(self, '_scheduler'):
+            # Get a scheduler instance
+            self._scheduler = scheduler.get_scheduler(storage=self.storage)
+        return self._scheduler
+
     @property
     def quota(self):
         if not hasattr(self, '_quota'):
@@ -909,10 +917,8 @@ class Service(service.RPCService, service.Service):
         if zone.ttl is not None:
             self._is_valid_ttl(context, zone.ttl)
 
-        # Get the default pool_id
-        default_pool_id = cfg.CONF['service:central'].default_pool_id
-        if zone.pool_id is None:
-            zone.pool_id = default_pool_id
+        # Get a pool id
+        zone.pool_id = self.scheduler.schedule_zone(context, zone)
 
         # Handle sub-zones appropriately
         parent_zone = self._is_subzone(

designate/exceptions.py

@@ -91,6 +91,11 @@ class NeutronCommunicationFailure(CommunicationFailure):
     error_type = 'neutron_communication_failure'
 
 
+class NoFiltersConfigured(ConfigurationError):
+    error_code = 500
+    error_type = 'no_filters_configured'
+
+
 class NoServersConfigured(ConfigurationError):
     error_code = 500
     error_type = 'no_servers_configured'

designate/objects/adapters/__init__.py

@@ -16,6 +16,7 @@ from designate.objects.adapters.base import DesignateAdapter  # noqa
 # API v2
 from designate.objects.adapters.api_v2.blacklist import BlacklistAPIv2Adapter, BlacklistListAPIv2Adapter  # noqa
 from designate.objects.adapters.api_v2.zone import ZoneAPIv2Adapter, ZoneListAPIv2Adapter  # noqa
+from designate.objects.adapters.api_v2.zone_attribute import ZoneAttributeAPIv2Adapter, ZoneAttributeListAPIv2Adapter  # noqa
 from designate.objects.adapters.api_v2.zone_master import ZoneMasterAPIv2Adapter, ZoneMasterListAPIv2Adapter  # noqa
 from designate.objects.adapters.api_v2.floating_ip import FloatingIPAPIv2Adapter, FloatingIPListAPIv2Adapter  # noqa
 from designate.objects.adapters.api_v2.record import RecordAPIv2Adapter, RecordListAPIv2Adapter  # noqa

designate/objects/adapters/api_v2/zone.py

@@ -46,8 +46,11 @@ class ZoneAPIv2Adapter(base.APIv2Adapter):
             "status": {},
             "action": {},
             "version": {},
+            "attributes": {
+                "immutable": True
+            },
             "type": {
-                'immutable': True
+                "immutable": True
             },
             "masters": {},
             "created_at": {},
@@ -74,6 +77,16 @@ class ZoneAPIv2Adapter(base.APIv2Adapter):
 
             del values['masters']
 
+        if 'attributes' in values:
+
+            object.attributes = objects.adapters.DesignateAdapter.parse(
+                cls.ADAPTER_FORMAT,
+                values['attributes'],
+                objects.ZoneAttributeList(),
+                *args, **kwargs)
+
+            del values['attributes']
+
         return super(ZoneAPIv2Adapter, cls)._parse_object(
             values, object, *args, **kwargs)
 

designate/objects/adapters/api_v2/zone_attribute.py

@@ -0,0 +1,97 @@
+# Copyright 2016 Hewlett Packard Enterprise Development Company LP
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+import six
+from oslo_log import log as logging
+
+from designate.objects.adapters.api_v2 import base
+from designate import objects
+LOG = logging.getLogger(__name__)
+
+
+class ZoneAttributeAPIv2Adapter(base.APIv2Adapter):
+
+    ADAPTER_OBJECT = objects.ZoneAttribute
+
+    MODIFICATIONS = {
+        'fields': {
+            'key': {
+                'read_only': False
+            },
+            'value': {
+                'read_only': False
+            }
+        },
+        'options': {
+            'links': False,
+            'resource_name': 'pool_attribute',
+            'collection_name': 'pool_attributes',
+        }
+    }
+
+    @classmethod
+    def _render_object(cls, object, *arg, **kwargs):
+        return {object.key: object.value}
+
+    @classmethod
+    def _parse_object(cls, values, object, *args, **kwargs):
+        for key in six.iterkeys(values):
+            object.key = key
+            object.value = values[key]
+
+        return object
+
+
+class ZoneAttributeListAPIv2Adapter(base.APIv2Adapter):
+
+    ADAPTER_OBJECT = objects.ZoneAttributeList
+
+    MODIFICATIONS = {
+        'options': {
+            'links': False,
+            'resource_name': 'zone_attribute',
+            'collection_name': 'zone_attributes',
+        }
+    }
+
+    @classmethod
+    def _render_list(cls, list_object, *args, **kwargs):
+
+        r_list = {}
+
+        for object in list_object:
+            value = cls.get_object_adapter(
+                cls.ADAPTER_FORMAT,
+                object).render(cls.ADAPTER_FORMAT, object, *args, **kwargs)
+            for key in six.iterkeys(value):
+                r_list[key] = value[key]
+
+        return r_list
+
+    @classmethod
+    def _parse_list(cls, values, output_object, *args, **kwargs):
+
+        for key, value in values.items():
+            # Add the object to the list
+            output_object.append(
+                # Get the right Adapter
+                cls.get_object_adapter(
+                    cls.ADAPTER_FORMAT,
+                    # This gets the internal type of the list, and parses it
+                    # We need to do `get_object_adapter` as we need a new
+                    # instance of the Adapter
+                    output_object.LIST_ITEM_TYPE()).parse(
+                        {key: value}, output_object.LIST_ITEM_TYPE()))
+
+        # Return the filled list
+        return output_object

designate/objects/pool.py

@@ -146,3 +146,9 @@ class Pool(base.DictObjectMixin, base.PersistentObjectMixin,
 
 class PoolList(base.ListObjectMixin, base.DesignateObject):
     LIST_ITEM_TYPE = Pool
+
+    def __contains__(self, pool):
+        for p in self.objects:
+            if p.id == pool.id:
+                return True
+        return False

designate/objects/zone_attribute.py

@@ -19,9 +19,27 @@ from designate.objects import base
 class ZoneAttribute(base.DictObjectMixin, base.PersistentObjectMixin,
                     base.DesignateObject):
     FIELDS = {
-        'zone_id': {},
-        'key': {},
-        'value': {}
+        'zone_id': {
+            'schema': {
+                'type': 'string',
+                'description': 'Zone identifier',
+                'format': 'uuid',
+            },
+        },
+        'key': {
+            'schema': {
+                'type': 'string',
+                'maxLength': 50,
+            },
+            'required': True,
+        },
+        'value': {
+            'schema': {
+                'type': 'string',
+                'maxLength': 50,
+            },
+            'required': True
+        }
     }
 
     STRING_KEYS = [

designate/scheduler/__init__.py

@@ -0,0 +1,32 @@
+# Copyright 2016 Hewlett Packard Enterprise Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from oslo_config import cfg
+from oslo_log import log as logging
+
+from designate.scheduler.base import Scheduler
+
+LOG = logging.getLogger(__name__)
+
+cfg.CONF.register_opts([
+    cfg.ListOpt(
+        'scheduler_filters',
+        default=['default_pool'],
+        help='Enabled Pool Scheduling filters'),
+], group='service:central')
+
+
+def get_scheduler(storage):
+
+    return Scheduler(storage=storage)

designate/scheduler/base.py

@@ -0,0 +1,82 @@
+# Copyright 2016 Hewlett Packard Enterprise Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from oslo_config import cfg
+from oslo_log import log as logging
+from stevedore import named
+
+from designate import exceptions
+from designate.i18n import _LI
+
+LOG = logging.getLogger(__name__)
+
+
+class Scheduler(object):
+    """Scheduler that schedules zones based on the filters provided on the zone
+    and other inputs.
+
+    :raises: NoFiltersConfigured
+    """
+
+    filters = []
+    """The list of filters enabled on this scheduler"""
+
+    def __init__(self, storage):
+
+        enabled_filters = cfg.CONF['service:central'].scheduler_filters
+        # Get a storage connection
+        self.storage = storage
+        if len(enabled_filters) > 0:
+            filters = named.NamedExtensionManager(
+                namespace='designate.scheduler.filters',
+                names=enabled_filters,
+                name_order=True)
+
+            self.filters = [x.plugin(storage=self.storage) for x in filters]
+            for filter in self.filters:
+                LOG.info(_LI("Loaded Scheduler Filter: %s") % filter.name)
+
+        else:
+            raise exceptions.NoFiltersConfigured('There are no scheduling '
+                                                 'filters configured')
+
+    def schedule_zone(self, context, zone):
+        """Get a pool to create the new zone in.
+
+        :param context: :class:`designate.context.DesignateContext` - Context
+            Object from request
+        :param zone: :class:`designate.objects.zone.Zone` - Zone to be created
+        :return: string -- ID of pool to schedule the zone to.
+        :raises: MultiplePoolsFound, NoValidPoolFound
+        """
+        pools = self.storage.find_pools(context)
+
+        if len(self.filters) is 0:
+            raise exceptions.NoFiltersConfigured('There are no scheduling '
+                                                 'filters configured')
+
+        for f in self.filters:
+            LOG.debug("Running %s filter with %d pools", f.name, len(pools))
+            pools = f.filter(context, pools, zone)
+            LOG.debug(
+                "%d candidate pools remaining after %s filter",
+                len(pools),
+                f.name)
+
+        if len(pools) > 1:
+            raise exceptions.MultiplePoolsFound()
+        if len(pools) is 0:
+            raise exceptions.NoValidPoolFound('There are no pools that '
+                                              'matched your request')
+        return pools[0].id

designate/scheduler/filters/attribute_filter.py

@@ -0,0 +1,62 @@
+# Copyright 2016 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+from oslo_log import log as logging
+
+from designate.scheduler.filters.base import Filter
+
+LOG = logging.getLogger(__name__)
+
+
+class AttributeFilter(Filter):
+    """This allows users top choose the pool by supplying hints to this filter.
+    These are provided as attributes as part of the zone object provided at
+    zone create time.
+
+    .. code-block:: javascript
+        :emphasize-lines: 3,4,5
+
+        {
+            "attributes": {
+                "pool_level": "gold",
+                "fast_ttl": True,
+                "pops": "global",
+            },
+            "email": "user@example.com",
+            "name": "example.com."
+        }
+
+    The zone attributes are matched against the potential pool candiates, and
+    any pools that do not match **all** hints are removed.
+
+    .. warning::
+
+        This filter is disabled currently, and should not be used.
+        It will be enabled at a later date.
+
+    .. warning::
+
+        This should be uses in conjunction with the
+        :class:`designate.scheduler.impl_filter.filters.random_filter.RandomFilter`
+        in case of multiple Pools matching the filters, as without it, we will
+        raise an error to the user.
+        """
+
+    name = 'attribute'
+    """Name to enable in the ``[designate:central:scheduler].filters`` option
+    list
+    """
+
+    def filter(self, context, pools, zone):
+        # FIXME (graham) actually filter on attributes
+        return pools

designate/scheduler/filters/base.py

@@ -0,0 +1,49 @@
+# Copyright 2016 Hewlett Packard Enterprise Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+import abc
+
+import six
+from oslo_log import log as logging
+
+LOG = logging.getLogger(__name__)
+
+
+@six.add_metaclass(abc.ABCMeta)
+class Filter():
+    """This is the base class used for filtering Pools.
+
+    This class should implement a single public function
+    :func:`filter` which accepts
+    a :class:`designate.objects.pool.PoolList` and returns a
+    :class:`designate.objects.pool.PoolList`
+    """
+
+    name = ''
+
+    def __init__(self, storage):
+        self.storage = storage
+        LOG.debug('Loaded %s filter in chain' % self.name)
+
+    @abc.abstractmethod
+    def filter(self, context, pools, zone):
+        """Filter list of supplied pools based on attributes in the request
+
+        :param context: :class:`designate.context.DesignateContext` - Context
+            Object from request
+        :param pools: :class:`designate.objects.pool.PoolList` - List of pools
+            to choose from
+        :param zone: :class:`designate.objects.zone.Zone` - Zone to be created
+        :return: :class:`designate.objects.pool.PoolList` - Filtered list of
+            Pools
+        """

designate/scheduler/filters/default_pool_filter.py

@@ -0,0 +1,40 @@
+# Copyright 2016 Hewlett Packard Enterprise Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+from oslo_config import cfg
+
+from designate.scheduler.filters.base import Filter
+from designate.objects import Pool
+from designate.objects import PoolList
+
+
+class DefaultPoolFilter(Filter):
+    """This filter will always return the default pool specified in the
+    designate config file
+
+    .. warning::
+
+        This should be used as the only filter, as it will always return the
+        same thing - a :class:`designate.objects.pool.PoolList` with a single
+        :class:`designate.objects.pool.Pool`
+    """
+
+    name = 'default_pool'
+    """Name to enable in the ``[designate:central:scheduler].filters`` option
+    list
+    """
+
+    def filter(self, context, pools, zone):
+        pools = PoolList()
+        pools.append(Pool(id=cfg.CONF['service:central'].default_pool_id))
+        return pools

designate/scheduler/filters/fallback_filter.py

@@ -0,0 +1,49 @@
+# Copyright 2016 Hewlett Packard Enterprise Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from oslo_config import cfg
+
+from designate.scheduler.filters.base import Filter
+from designate.objects import Pool
+from designate.objects import PoolList
+
+cfg.CONF.register_opts([
+    cfg.StrOpt('default_pool_id',
+               default='794ccc2c-d751-44fe-b57f-8894c9f5c842',
+               help="The name of the default pool"),
+], group='service:central')
+
+
+class FallbackFilter(Filter):
+    """If there is no zones availible to schedule to, this filter will insert
+    the default_pool_id.
+
+    .. note::
+
+        This should be used as one of the last filters, if you want to preserve
+        behavoir from before the scheduler existed.
+    """
+
+    name = 'fallback'
+    """Name to enable in the ``[designate:central:scheduler].filters`` option
+    list
+    """
+
+    def filter(self, context, pools, zone):
+        if len(pools) is 0:
+            pools = PoolList()
+            pools.append(Pool(id=cfg.CONF['service:central'].default_pool_id))
+            return pools
+        else:
+            return pools

designate/scheduler/filters/pool_id_attribute_filter.py

@@ -0,0 +1,84 @@
+# Copyright 2016 Hewlett Packard Enterprise Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+from oslo_log import log as logging
+
+from designate.scheduler.filters.base import Filter
+from designate import exceptions
+from designate import objects
+from designate import policy
+
+LOG = logging.getLogger(__name__)
+
+
+class PoolIDAttributeFilter(Filter):
+    """This allows users with the correct role to specify the exact pool_id
+    to schedule the supplied zone to.
+
+    This is supplied as an attribute on the zone
+
+    .. code-block:: python
+        :emphasize-lines: 3
+
+        {
+            "attributes": {
+                "pool_id": "794ccc2c-d751-44fe-b57f-8894c9f5c842"
+            },
+            "email": "user@example.com",
+            "name": "example.com."
+        }
+
+    The pool is loaded to ensure it exists, and then a policy check is
+    performed to ensure the user has the correct role.
+
+    .. warning::
+
+        This should only be enabled if required, as it will raise a
+        403 Forbidden if a user without the correct role uses it.
+    """
+
+    name = 'pool_id_attribute'
+    """Name to enable in the ``[designate:central:scheduler].filters`` option
+    list
+    """
+
+    def filter(self, context, pools, zone):
+        """Attempt to load and set the pool to the one provied in the
+        Zone attributes.
+
+        :param context: :class:`designate.context.DesignateContext` - Context
+            Object from request
+        :param pools: :class:`designate.objects.pool.PoolList` - List of pools
+            to choose from
+        :param zone: :class:`designate.objects.zone.Zone` - Zone to be created
+        :return: :class:`designate.objects.pool.PoolList` -- A PoolList with
+            containing a single pool.
+        :raises: Forbidden, PoolNotFound
+        """
+
+        try:
+            if zone.attributes.get('pool_id'):
+                pool_id = zone.attributes.get('pool_id')
+                try:
+                    pool = self.storage.get_pool(context, pool_id)
+                except Exception:
+                    return objects.PoolList()
+                policy.check('zone_create_forced_pool', context, pool)
+                if pool in pools:
+                    pools = objects.PoolList()
+                    pools.append(pool)
+                return pools
+            else:
+                return pools
+        except exceptions.RelationNotLoaded:
+            return pools

designate/scheduler/filters/random_filter.py

@@ -0,0 +1,43 @@
+# Copyright 2016 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+import random
+
+from oslo_log import log as logging
+
+from designate.scheduler.filters.base import Filter
+from designate.objects import PoolList
+
+LOG = logging.getLogger(__name__)
+
+
+class RandomFilter(Filter):
+    """Randomly chooses one of the input pools if there is multiple supplied.
+
+    .. note::
+
+        This should be used as one of the last filters, as it reduces the
+        supplied pool list to one.
+    """
+    name = 'random'
+    """Name to enable in the ``[designate:central:scheduler].filters`` option
+    list
+    """
+
+    def filter(self, context, pools, zone):
+        new_list = PoolList()
+        if len(pools):
+            new_list.append(random.choice(pools))
+            return new_list
+        else:
+            return pools

designate/tests/__init__.py

@@ -318,6 +318,10 @@ class TestCase(base.BaseTestCase):
 
         self.config(network_api='fake')
 
+        self.config(
+            scheduler_filters=['pool_id_attribute', 'random'],
+            group='service:central')
+
         # "Read" Configuration
         self.CONF([], project='designate')
         utils.register_plugin_opts()

designate/tests/test_central/test_service.py

@@ -480,7 +480,8 @@ class CentralServiceTest(CentralTestCase):
 
         # Create a secondary pool
         second_pool = self.create_pool()
-        fixture["pool_id"] = second_pool.id
+        fixture["attributes"] = {}
+        fixture["attributes"]["pool_id"] = second_pool.id
 
         self.create_zone(**fixture)
 
@@ -537,15 +538,19 @@ class CentralServiceTest(CentralTestCase):
         fixture = self.get_zone_fixture()
 
         # Create first zone that's placed in default pool
-        self.create_zone(**fixture)
+        zone = self.create_zone(**fixture)
 
         # Create a secondary pool
         second_pool = self.create_pool()
-        fixture["pool_id"] = second_pool.id
+        fixture["attributes"] = {}
+        fixture["attributes"]["pool_id"] = second_pool.id
         fixture["name"] = "sub.%s" % fixture["name"]
-
         subzone = self.create_zone(**fixture)
-        self.assertIsNone(subzone.parent_zone_id)
+
+        if subzone.pool_id is not zone.pool_id:
+            self.assertIsNone(subzone.parent_zone_id)
+        else:
+            raise Exception("Foo")
 
     def test_create_superzone(self):
         # Prepare values for the zone and subzone
@@ -2903,9 +2908,14 @@ class CentralServiceTest(CentralTestCase):
     def test_update_pool_add_ns_record(self):
         # Create a server pool and 3 zones
         pool = self.create_pool(fixture=0)
-        zone = self.create_zone(pool_id=pool.id)
-        self.create_zone(fixture=1, pool_id=pool.id)
-        self.create_zone(fixture=2, pool_id=pool.id)
+        zone = self.create_zone(
+            attributes=[{'key': 'pool_id', 'value': pool.id}])
+        self.create_zone(
+            fixture=1,
+            attributes=[{'key': 'pool_id', 'value': pool.id}])
+        self.create_zone(
+            fixture=2,
+            attributes=[{'key': 'pool_id', 'value': pool.id}])
 
         ns_record_count = len(pool.ns_records)
         new_ns_record = objects.PoolNsRecord(
@@ -2952,7 +2962,8 @@ class CentralServiceTest(CentralTestCase):
     def test_update_pool_remove_ns_record(self):
         # Create a server pool and zone
         pool = self.create_pool(fixture=0)
-        zone = self.create_zone(pool_id=pool.id)
+        zone = self.create_zone(
+            attributes=[{'key': 'pool_id', 'value': pool.id}])
 
         ns_record_count = len(pool.ns_records)
 

designate/tests/unit/test_central/test_basic.py

@@ -27,6 +27,7 @@ import mock
 import testtools
 
 from designate import exceptions
+from designate import objects
 from designate.central.service import Service
 from designate.tests.fixtures import random_seed
 import designate.central.service
@@ -200,7 +201,6 @@ class MockRecord(object):
 class MockPool(object):
     ns_records = [MockRecord(), ]
 
-
 # Fixtures
 
 fx_mdns_api = fixtures.MockPatch('designate.central.service.mdns_rpcapi')
@@ -236,41 +236,26 @@ class CentralBasic(base.BaseTestCase):
         super(CentralBasic, self).setUp()
         self.CONF = self.useFixture(cfg_fixture.Config(cfg.CONF)).conf
 
-        mock_storage = mock.NonCallableMagicMock(spec_set=[
-            'count_zones', 'count_records', 'count_recordsets',
-            'count_tenants', 'create_blacklist', 'create_zone',
-            'create_pool', 'create_pool_attribute', 'create_quota',
-            'create_record', 'create_recordset', 'create_tld',
-            'create_tsigkey',
-            'create_zone_task', 'delete_blacklist', 'delete_zone',
-            'delete_pool', 'delete_pool_attribute', 'delete_quota',
-            'delete_record', 'delete_recordset', 'delete_tld',
-            'delete_tsigkey', 'delete_zone_task', 'find_blacklist',
-            'find_blacklists', 'find_zone', 'find_zones', 'find_pool',
-            'find_pool_attribute', 'find_pool_attributes', 'find_pools',
-            'find_quota', 'find_quotas', 'find_record', 'find_records',
-            'find_recordset', 'find_recordsets', 'find_recordsets_axfr',
-            'find_tenants', 'find_tld', 'find_tlds', 'find_tsigkeys',
-            'find_zone_task', 'find_zone_tasks', 'get_blacklist',
-            'get_canonical_name', 'get_cfg_opts', 'get_zone', 'get_driver',
-            'get_extra_cfg_opts', 'get_plugin_name', 'get_plugin_type',
-            'get_pool', 'get_pool_attribute', 'get_quota', 'get_record',
-            'get_recordset', 'get_tenant', 'get_tld', 'get_tsigkey',
-            'get_zone_task', 'ping', 'register_cfg_opts',
-            'register_extra_cfg_opts', 'update_blacklist', 'update_zone',
-            'update_pool', 'update_pool_attribute', 'update_quota',
-            'update_record', 'update_recordset', 'update_tld',
-            'update_tsigkey', 'update_zone_task', 'commit', 'begin',
-            'rollback', ])
+        mock_storage = mock.Mock(spec=designate.storage.base.Storage)
+
+        pool_list = objects.PoolList.from_list(
+            [
+                {'id': '794ccc2c-d751-44fe-b57f-8894c9f5c842'}
+            ]
+        )
+
         attrs = {
             'count_zones.return_value': 0,
             'find_zone.return_value': Mockzone(),
             'get_pool.return_value': MockPool(),
-            'begin.return_value': None,
+            'find_pools.return_value': pool_list,
         }
         mock_storage.configure_mock(**attrs)
-        designate.central.service.storage.get_storage.return_value = \
-            mock_storage
+
+        self.useFixture(fixtures.MockPatchObject(
+            designate.central.service.storage, 'get_storage',
+            return_value=mock_storage)
+        )
 
         designate.central.service.policy = mock.NonCallableMock(spec_set=[
             'reset',
@@ -292,6 +277,7 @@ class CentralBasic(base.BaseTestCase):
             'elevated',
             'sudo',
             'abandon',
+            'all_tenants',
         ])
 
         self.service = Service()
@@ -818,17 +804,29 @@ class CentralZoneTestCase(CentralBasic):
             ns_records=[]
         )
 
+        self.useFixture(
+            fixtures.MockPatchObject(
+                self.service.storage,
+                'find_pools',
+                return_value=objects.PoolList.from_list(
+                    [
+                        {'id': '94ccc2c-d751-44fe-b57f-8894c9f5c842'}
+                    ]
+                )
+            )
+        )
+
         with testtools.ExpectedException(exceptions.NoServersConfigured):
             self.service.create_zone(
                 self.context,
-                RoObject(tenant_id='1', name='example.com.', ttl=60,
-                         pool_id='2')
+                objects.Zone(tenant_id='1', name='example.com.', ttl=60,
+                             pool_id='2')
             )
 
     def test_create_zone(self):
         self.service._enforce_zone_quota = Mock()
         self.service._create_zone_in_storage = Mock(
-            return_value=RoObject(
+            return_value=objects.Zone(
                 name='example.com.',
                 type='PRIMARY',
             )
@@ -844,11 +842,23 @@ class CentralZoneTestCase(CentralBasic):
         self.service.storage.get_pool.return_value = RoObject(
             ns_records=[RoObject()]
         )
+        self.useFixture(
+            fixtures.MockPatchObject(
+                self.service.storage,
+                'find_pools',
+                return_value=objects.PoolList.from_list(
+                    [
+                        {'id': '94ccc2c-d751-44fe-b57f-8894c9f5c842'}
+                    ]
+                )
+            )
+        )
+
         # self.service.create_zone = unwrap(self.service.create_zone)
 
         out = self.service.create_zone(
             self.context,
-            RwObject(
+            objects.Zone(
                 tenant_id='1',
                 name='example.com.',
                 ttl=60,

designate/tests/unit/test_scheduler/test_basic.py

@@ -0,0 +1,120 @@
+# (c) Copyright 2016 Hewlett Packard Enterprise Development Company LP.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""Unit-test Pool Scheduler
+"""
+import testtools
+from mock import Mock
+from oslotest import base as test
+from oslo_config import cfg
+from oslo_config import fixture as cfg_fixture
+
+from designate import scheduler
+from designate import objects
+from designate import context
+from designate import exceptions
+
+
+class SchedulerTest(test.BaseTestCase):
+
+    def setUp(self):
+        super(SchedulerTest, self).setUp()
+
+        self.context = context.DesignateContext()
+        self.CONF = self.useFixture(cfg_fixture.Config(cfg.CONF)).conf
+
+    def test_default_operation(self):
+        zone = objects.Zone(
+            name="example.com.",
+            type="PRIMARY",
+            email="hostmaster@example.com"
+        )
+
+        attrs = {
+            'find_pools.return_value': objects.PoolList.from_list(
+                [{"id": "794ccc2c-d751-44fe-b57f-8894c9f5c842"}])
+        }
+        mock_storage = Mock(**attrs)
+
+        test_scheduler = scheduler.get_scheduler(storage=mock_storage)
+
+        zone.pool_id = test_scheduler.schedule_zone(self.context, zone)
+
+        self.assertEqual(zone.pool_id, "794ccc2c-d751-44fe-b57f-8894c9f5c842")
+
+    def test_multiple_pools(self):
+        zone = objects.Zone(
+            name="example.com.",
+            type="PRIMARY",
+            email="hostmaster@example.com"
+        )
+
+        attrs = {
+            'find_pools.return_value': objects.PoolList.from_list(
+                [
+                    {"id": "794ccc2c-d751-44fe-b57f-8894c9f5c842"},
+                    {"id": "5fabcd37-262c-4cf3-8625-7f419434b6df"}
+                ]
+            )
+        }
+
+        mock_storage = Mock(**attrs)
+
+        test_scheduler = scheduler.get_scheduler(storage=mock_storage)
+
+        zone.pool_id = test_scheduler.schedule_zone(self.context, zone)
+
+        self.assertIn(
+            zone.pool_id,
+            [
+                "794ccc2c-d751-44fe-b57f-8894c9f5c842",
+                "5fabcd37-262c-4cf3-8625-7f419434b6df",
+            ]
+        )
+
+    def test_no_pools(self):
+        zone = objects.Zone(
+            name="example.com.",
+            type="PRIMARY",
+            email="hostmaster@example.com"
+        )
+
+        attrs = {
+            'find_pools.return_value': objects.PoolList()
+        }
+        mock_storage = Mock(**attrs)
+
+        cfg.CONF.set_override(
+            'scheduler_filters',
+            ['random'],
+            'service:central',
+            enforce_type=True)
+
+        test_scheduler = scheduler.get_scheduler(storage=mock_storage)
+
+        with testtools.ExpectedException(exceptions.NoValidPoolFound):
+            test_scheduler.schedule_zone(self.context, zone)
+
+    def test_no_filters_enabled(self):
+
+        cfg.CONF.set_override(
+            'scheduler_filters', [], 'service:central', enforce_type=True)
+
+        attrs = {
+            'find_pools.return_value': objects.PoolList()
+        }
+        mock_storage = Mock(**attrs)
+
+        with testtools.ExpectedException(exceptions.NoFiltersConfigured):
+            scheduler.get_scheduler(storage=mock_storage)

designate/tests/unit/test_scheduler/test_filters.py

@@ -0,0 +1,204 @@
+# (c) Copyright 2016 Hewlett Packard Enterprise Development Company LP.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""Unit-test Pool Scheduler
+"""
+import fixtures
+import testtools
+from mock import Mock
+from oslotest import base as test
+
+from designate.scheduler.filters import default_pool_filter
+from designate.scheduler.filters import fallback_filter
+from designate.scheduler.filters import pool_id_attribute_filter
+from designate import objects
+from designate import context
+from designate import policy
+from designate import exceptions
+
+
+class SchedulerFilterTest(test.BaseTestCase):
+
+    def setUp(self):
+        super(SchedulerFilterTest, self).setUp()
+
+        self.context = context.DesignateContext()
+
+        self.zone = objects.Zone(
+            name="example.com.",
+            type="PRIMARY",
+            email="hostmaster@example.com"
+        )
+
+        attrs = {
+            'get_pool.return_value': objects.Pool(
+                id="6c346011-e581-429b-a7a2-6cdf0aba91c3")
+        }
+
+        mock_storage = Mock(**attrs)
+
+        self.test_filter = self.FILTER(storage=mock_storage)
+
+
+class SchedulerDefaultPoolFilterTest(SchedulerFilterTest):
+
+    FILTER = default_pool_filter.DefaultPoolFilter
+
+    def test_default_operation(self):
+        pools = objects.PoolList.from_list(
+            [{"id": "794ccc2c-d751-44fe-b57f-8894c9f5c842"}]
+        )
+        pools = self.test_filter.filter(self.context, pools, self.zone)
+
+        self.assertEqual(pools[0].id, "794ccc2c-d751-44fe-b57f-8894c9f5c842")
+
+    def test_multiple_pools(self):
+        pools = objects.PoolList.from_list(
+            [
+                {"id": "794ccc2c-d751-44fe-b57f-8894c9f5c842"},
+                {"id": "5fabcd37-262c-4cf3-8625-7f419434b6df"}
+            ]
+        )
+        pools = self.test_filter.filter(self.context, pools, self.zone)
+
+        self.assertEqual(pools[0].id, "794ccc2c-d751-44fe-b57f-8894c9f5c842")
+
+    def test_no_pools(self):
+        pools = objects.PoolList()
+        pools = self.test_filter.filter(self.context, pools, self.zone)
+
+        self.assertEqual(pools[0].id, "794ccc2c-d751-44fe-b57f-8894c9f5c842")
+
+
+class SchedulerFallbackFilterTest(SchedulerFilterTest):
+
+    FILTER = fallback_filter.FallbackFilter
+
+    def test_default_operation(self):
+        pools = objects.PoolList.from_list(
+            [{"id": "794ccc2c-d751-44fe-b57f-8894c9f5c842"}]
+        )
+        pools = self.test_filter.filter(self.context, pools, self.zone)
+
+        self.assertEqual(pools[0].id, "794ccc2c-d751-44fe-b57f-8894c9f5c842")
+
+    def test_multiple_pools(self):
+        pools = objects.PoolList.from_list(
+            [
+                {"id": "6c346011-e581-429b-a7a2-6cdf0aba91c3"},
+                {"id": "5fabcd37-262c-4cf3-8625-7f419434b6df"}
+            ]
+        )
+        pools = self.test_filter.filter(self.context, pools, self.zone)
+
+        self.assertEqual(len(pools), 2)
+
+        for pool in pools:
+            self.assertIn(
+                pool.id,
+                [
+                    "6c346011-e581-429b-a7a2-6cdf0aba91c3",
+                    "5fabcd37-262c-4cf3-8625-7f419434b6df",
+                ]
+            )
+
+    def test_no_pools(self):
+        pools = objects.PoolList()
+        pools = self.test_filter.filter(self.context, pools, self.zone)
+
+        self.assertEqual(pools[0].id, "794ccc2c-d751-44fe-b57f-8894c9f5c842")
+
+
+class SchedulerPoolIDAttributeFilterTest(SchedulerFilterTest):
+
+    FILTER = pool_id_attribute_filter.PoolIDAttributeFilter
+
+    def setUp(self):
+        super(SchedulerPoolIDAttributeFilterTest, self).setUp()
+
+        self.zone = objects.Zone(
+            name="example.com.",
+            type="PRIMARY",
+            email="hostmaster@example.com",
+            attributes=objects.ZoneAttributeList.from_list(
+                [
+                    {
+                        "key": "pool_id",
+                        "value": "6c346011-e581-429b-a7a2-6cdf0aba91c3"
+                    }
+                ]
+            )
+        )
+
+    def test_default_operation(self):
+        pools = objects.PoolList.from_list(
+            [{"id": "6c346011-e581-429b-a7a2-6cdf0aba91c3"}]
+        )
+        self.useFixture(fixtures.MockPatchObject(
+            policy, 'check',
+            return_value=None
+        ))
+
+        pools = self.test_filter.filter(self.context, pools, self.zone)
+
+        self.assertEqual("6c346011-e581-429b-a7a2-6cdf0aba91c3", pools[0].id)
+
+    def test_multiple_pools(self):
+        pools = objects.PoolList.from_list(
+            [
+                {"id": "6c346011-e581-429b-a7a2-6cdf0aba91c3"},
+                {"id": "5fabcd37-262c-4cf3-8625-7f419434b6df"}
+            ]
+        )
+
+        self.useFixture(fixtures.MockPatchObject(
+            policy, 'check',
+            return_value=None
+        ))
+
+        pools = self.test_filter.filter(self.context, pools, self.zone)
+
+        self.assertEqual(len(pools), 1)
+
+        self.assertEqual("6c346011-e581-429b-a7a2-6cdf0aba91c3", pools[0].id)
+
+    def test_no_pools(self):
+        pools = objects.PoolList()
+
+        self.useFixture(fixtures.MockPatchObject(
+            policy, 'check',
+            return_value=None
+        ))
+
+        pools = self.test_filter.filter(self.context, pools, self.zone)
+
+        self.assertEqual(len(pools), 0)
+
+    def test_policy_failure(self):
+        pools = objects.PoolList.from_list(
+            [{"id": "6c346011-e581-429b-a7a2-6cdf0aba91c3"}]
+        )
+
+        self.useFixture(fixtures.MockPatchObject(
+            policy, 'check',
+            side_effect=exceptions.Forbidden
+        ))
+
+        with testtools.ExpectedException(exceptions.Forbidden):
+            self.test_filter.filter(self.context, pools, self.zone)
+
+        policy.check.assert_called_once_with(
+            'zone_create_forced_pool',
+            self.context,
+            pools[0])

doc/source/index.rst

@@ -63,6 +63,7 @@ Reference Documentation
    functional-tests
    gmr
    support-matrix
+   pools
 
 Source Documentation
 ====================

doc/source/objects.rst

@@ -28,6 +28,14 @@ Objects Zone
     :show-inheritance:
 
 
+Objects Pool
+============
+.. automodule:: designate.objects.pool
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+
 Objects Quota
 =============
 .. automodule:: designate.objects.quota
@@ -76,7 +84,7 @@ Objects TLD
     :show-inheritance:
 
 
-Objects TSigKey 
+Objects TSigKey
 ===============
 .. automodule:: designate.objects.tsigkey
     :members:
@@ -142,7 +150,7 @@ Objects SOA Record
 
 Objects SPF Record
 ==================
-.. automodule:: designate.objects.rrdata_spf 
+.. automodule:: designate.objects.rrdata_spf
     :members:
     :undoc-members:
     :show-inheritance:

doc/source/pools.rst

@@ -0,0 +1,64 @@
+..
+    Copyright 2016 Hewlett Packard Enterprise Development Company LP
+
+    Licensed under the Apache License, Version 2.0 (the "License"); you may
+    not use this file except in compliance with the License. You may obtain
+    a copy of the License at
+
+        http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+    License for the specific language governing permissions and limitations
+    under the License.
+
+.. _pools:
+
+=====
+Pools
+=====
+
+Contents:
+
+.. toctree::
+   :maxdepth: 2
+   :glob:
+
+   pools/scheduler
+
+
+Overview
+========
+
+In designate we support the concept of multiple "pools" of DNS Servers.
+
+This allows operators to scale out their DNS Service by adding more pools, avoiding
+the scalling problems that some DNS servers have for number of zones, and the total
+number of records hosted by a single server.
+
+This also allows providers to have tiers of service (i.e. the difference
+between GOLD vs SILVER tiers may be the number of DNS Servers, and how they
+are distributed around the world.)
+
+In a private cloud situation, it allows operators to separate internal and
+external facing zones.
+
+To help users create zones on the  correct pool we have a "scheduler" that is
+responsible for examining the zone being created and the pools that are
+availible for use, and matching the zone to a pool.
+
+The filters are plugable (i.e. operator replaceable) and all follow a simple
+interface.
+
+The zones are matched using "zone attributes" and "pool attributes". These are
+key: value pairs that are attached to the zone when it is being created, and
+the pool. The pool attributes can be updated by the operator in the future,
+but it will **not** trigger zones to be moved from one pool to another.
+
+.. note::
+
+    Currently the only zone attribute that is accepted is the `pool_id` attribute.
+    As more filters are merged there will be support for dynamic filters.
+
+

doc/source/pools/scheduler.rst

@@ -0,0 +1,104 @@
+..
+    Copyright 2016 Hewlett Packard Enterprise Development Company, L.P.
+
+    Licensed under the Apache License, Version 2.0 (the "License"); you may
+    not use this file except in compliance with the License. You may obtain
+    a copy of the License at
+
+        http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+    License for the specific language governing permissions and limitations
+    under the License.
+
+.. _pool_scheduler:
+
+==============
+Pool Scheduler
+==============
+
+In designate we have a plugable scheduler filter interface.
+
+You can set an ordered list of filters to run on each zone create api request.
+
+We provide a few basic filters below, and creating custom filters follows a
+similar pattern to schedulers.
+
+You can create your own by extending :class:`designate.scheduler.filters.base.Filter`
+and registering a new entry point in the ``designate.scheduler.filters``
+namespace like so in your ``setup.cfg`` file:
+
+.. code-block:: ini
+
+    [entry_points]
+    designate.scheduler.filters =
+        my_custom_filter = my_extention.filters.my_custom_filter:MyCustomFilter
+
+The new filter can be added to the ``scheduler_filters`` list in the ``[service:central]``
+section like so:
+
+.. code-block:: ini
+
+    [service:central]
+
+    scheduler_filters = attribute, pool_id_attribute, fallback, random, my_custom_filter
+
+The filters list is ran from left to right, so if the list is set to:
+
+.. code-block:: ini
+
+    [service:central]
+
+    scheduler_filters = attribute, random
+
+There will be two filters ran, the :class:`designate.scheduler.filters.attribute_filter.AttributeFilter`
+followed by :class:`designate.scheduler.filters.random_filter.RandomFilter`
+
+
+Default Provided Filters
+========================
+
+Base Class - Filter
+-------------------
+
+.. autoclass:: designate.scheduler.filters.base.Filter
+    :members:
+
+Attribute Filter
+----------------
+
+.. autoclass:: designate.scheduler.filters.attribute_filter.AttributeFilter
+    :members: name
+    :show-inheritance:
+
+Pool ID Attribute Filter
+------------------------
+
+.. autoclass:: designate.scheduler.filters.pool_id_attribute_filter.PoolIDAttributeFilter
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+Random Filter
+-------------
+
+.. autoclass:: designate.scheduler.filters.random_filter.RandomFilter
+    :members: name
+    :show-inheritance:
+
+Fallback Filter
+---------------
+
+.. autoclass:: designate.scheduler.filters.fallback_filter.FallbackFilter
+    :members: name
+    :show-inheritance:
+
+
+Default Pool Filter
+-------------------
+
+.. autoclass:: designate.scheduler.filters.default_pool_filter.DefaultPoolFilter
+    :members: name
+    :show-inheritance:

etc/designate/designate.conf.sample

@@ -80,6 +80,10 @@ debug = False
 # Tenant ID to own all managed resources - like auto-created records etc.
 #managed_resource_tenant_id = 123456
 
+# What filters to use. They are applied in order listed in the option, from
+# left to right
+#scheduler_filters = default_pool
+
 #-----------------------
 # API Service
 #-----------------------

etc/designate/policy.json

@@ -89,6 +89,7 @@
     "get_pool": "rule:admin",
     "update_pool": "rule:admin",
     "delete_pool": "rule:admin",
+    "zone_create_forced_pool": "rule:admin",
 
     "diagnostics_ping": "rule:admin",
     "diagnostics_sync_zones": "rule:admin",

releasenotes/notes/pool_scheduler-32e34dda9484ef9a.yaml

@@ -0,0 +1,9 @@
+---
+features:
+  - Schedule across pools. See http://doc.openstack.org/developer/designate/pools/scheduler.html#default-provided-filters for the built in filters
+upgrade:
+  - The default option for the scheduler filters will be
+    ``attribute, pool_id_attribute, random``.
+  - To maintain exact matching behaviour (if you have multiple pools) you will
+    need to set the ``scheduler_filters`` option in ``[service:central]`` to
+    ``default_pool``

setup.cfg

@@ -102,6 +102,12 @@ designate.quota =
     noop =  designate.quota.impl_noop:NoopQuota
     storage = designate.quota.impl_storage:StorageQuota
 
+designate.scheduler.filters =
+    fallback = designate.scheduler.filters.fallback_filter:FallbackFilter
+    random = designate.scheduler.filters.random_filter:RandomFilter
+    pool_id_attribute = designate.scheduler.filters.pool_id_attribute_filter:PoolIDAttributeFilter
+    default_pool = designate.scheduler.filters.default_pool_filter:DefaultPoolFilter
+
 designate.manage =
     database = designate.manage.database:DatabaseCommands
     akamai = designate.manage.akamai:AkamaiCommands