The framework of dynamic pod binding

1. What is the problem?
Originally this patch was committed to the Tricircle and provided
dynamic pod binding support. After splitting this feature should be
put in Trio2o. So we plan to synchronize this patch from Gerrit
to Trio2o. You can find the old patch on Gerrit here[1].

In the Trio2o, each tenant is bound to multiple pods, where it creates
various types of resources. However such a binding relationship should
be dynamic instead of static. For instance, when some resources in a pod
are exhausted, tenant needs to be bound to a new pod in the same AZ.

2. What is the solution to the problem?
To deal with the above problem, the Trio2o dynamically binds tenants
to pod which has available resources. We call this feature dynamic pod
binding, which is explained in the spec[2] in detail. In this patch, we
only try to bind a tenant to a pod dynamically, when she tries to create
a VM.

3. What the features need to be implemented to the Trio2o to realize
the solution?
When a tenant creates a VM, the Trio2o first selects all available
pods for her. Then by filtering and weighing the pods, the Trio2o
selects the most suitable pod for the tenant. Next, the Trio2o queries
database for current binding relationship of the tenant. If the tenant
is not bound to any pod, we create a new binding relationship, which
binds the tenant to the selected pod. If the tenant is already bound to
a pod, and the pod is not the one selected by the Trio2o, we update
current binding relationship, which binds the tenant to a new pod. If
the tenant is already bound to a pod, and the pod is exactly the one
selected by the Trio2o, the Trio2o does nothing.

[1] https://review.openstack.org/#/c/356187
[2] https://review.openstack.org/#/c/306224

Change-Id: I1c884ce47ace919f732d98ee154bdd59ed22cd74

setup.cfg

@@ -56,3 +56,8 @@ oslo.config.opts =
 
 tempest.test_plugins =
     trio2o_tests = trio2o.tempestplugin.plugin:Trio2oTempestPlugin
+
+trio2o.common.schedulers =
+    pod_manager = trio2o.common.scheduler.pod_manager:PodManager
+    bottom_pod_filter = trio2o.common.scheduler.filters.bottom_pod_filter:BottomPodFilter
+    filter_scheduler = trio2o.common.scheduler.filter_scheduler:FilterScheduler

trio2o/api/controllers/pod.py

@@ -223,7 +223,6 @@ class BindingsController(rest.RestController):
         pod_b = kw['pod_binding']
         tenant_id = pod_b.get('tenant_id', '').strip()
         pod_id = pod_b.get('pod_id', '').strip()
-        _uuid = uuidutils.generate_uuid()
 
         if tenant_id == '' or pod_id == '':
             return Response(
@@ -249,11 +248,7 @@ class BindingsController(rest.RestController):
             return
 
         try:
-            with context.session.begin():
+            pod_binding = db_api.create_pod_binding(context, tenant_id, pod_id)
-                pod_binding = core.create_resource(context, models.PodBinding,
-                                                   {'id': _uuid,
-                                                    'tenant_id': tenant_id,
-                                                    'pod_id': pod_id})
         except db_exc.DBDuplicateEntry:
             return Response(_('Pod binding already exists'), 409)
         except db_exc.DBConstraintError:

trio2o/cinder_apigw/controllers/volume.py

@@ -29,6 +29,7 @@ import trio2o.common.context as t_context
 from trio2o.common import httpclient as hclient
 from trio2o.common.i18n import _
 from trio2o.common.i18n import _LE
+from trio2o.common.scheduler import filter_scheduler
 from trio2o.common import utils
 
 import trio2o.db.api as db_api
@@ -42,6 +43,7 @@ class VolumeController(rest.RestController):
 
     def __init__(self, tenant_id):
         self.tenant_id = tenant_id
+        self.filter_scheduler = filter_scheduler.FilterScheduler()
 
     @expose(generic=True, template='json')
     def post(self, **kw):
@@ -52,10 +54,9 @@ class VolumeController(rest.RestController):
                 400, _("Missing required element 'volume' in request body."))
 
         az = kw['volume'].get('availability_zone', '')
-        pod, pod_az = az_ag.get_pod_by_az_tenant(
+        pod, pod_az = self.filter_scheduler.select_destination(
-            context,
+            context, az, self.tenant_id, pod_group='')
-            az_name=az,
+
-            tenant_id=self.tenant_id)
         if not pod:
             LOG.error(_LE("Pod not configured or scheduling failure"))
             return utils.format_cinder_error(

trio2o/common/az_ag.py

@@ -147,7 +147,8 @@ def get_pod_by_az_tenant(context, az_name, tenant_id):
                         context, models.PodBinding,
                         {'id': uuidutils.generate_uuid(),
                          'tenant_id': tenant_id,
-                         'pod_id': pod['pod_id']})
+                         'pod_id': pod['pod_id'],
+                         'is_binding': True})
                     return pod, pod['pod_az_name']
             except Exception as e:
                 LOG.error(_LE('Fail to create pod binding: %(exception)s'),

trio2o/common/scheduler/driver.py

@@ -0,0 +1,31 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import abc
+import six
+
+from stevedore import driver
+
+
+@six.add_metaclass(abc.ABCMeta)
+class Scheduler(object):
+
+    def __init__(self):
+        self.pod_manager = driver.DriverManager(
+            namespace='trio2o.common.schedulers',
+            name='pod_manager',
+            invoke_on_load=True
+        ).driver
+
+    @abc.abstractmethod
+    def select_destination(self, context, az_name, tenant_id, spec_obj):
+        return None, None

trio2o/common/scheduler/filter_scheduler.py

@@ -0,0 +1,58 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from trio2o.common.scheduler import driver
+
+
+class FilterScheduler(driver.Scheduler):
+
+    def __init__(self, *args, **kwargs):
+        super(FilterScheduler, self).__init__(*args, **kwargs)
+
+    def select_destination(self, context, az_name, tenant_id, pod_group):
+        current_binding, current_pod = \
+            self.pod_manager.get_current_binding_and_pod(
+                context, az_name, tenant_id, pod_group)
+
+        if current_binding and current_pod:
+            return current_pod, current_pod['pod_az_name']
+        else:
+            pods = self.pod_manager.get_available_pods(
+                context, az_name, pod_group)
+            if not pods:
+                return None, None
+            # TODO(Yipei): Weigh pods and select one whose weight
+            # is the maximum. Here we chose one randomly.
+            is_current = False
+            best_pod = None
+            # select the pod by a circle in pods
+            for pod in pods:
+                if is_current:
+                    best_pod = pod
+                    break
+                if current_binding \
+                        and pod['pod_id'] == current_binding['pod_id']:
+                    is_current = True
+            if is_current and len(pods) == 1:
+                return None, None
+            if not best_pod:
+                best_pod = pods[0]
+
+            if current_binding:
+                is_successful = self.pod_manager.update_binding(
+                    context, current_binding, best_pod['pod_id'])
+            else:
+                is_successful = self.pod_manager.create_binding(
+                    context, tenant_id, best_pod['pod_id'])
+            if not is_successful:
+                return None, None
+            return best_pod, best_pod['pod_az_name']

trio2o/common/scheduler/filters/base_filters.py

@@ -0,0 +1,31 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+class BaseFilter(object):
+    """Base class for all pod filter classes."""
+    def _filter_one(self, obj, pod_group):
+        return True
+
+    def filter_all(self, filter_obj_list, pod_group):
+        for obj in filter_obj_list:
+            if self._filter_one(obj, pod_group):
+                yield obj
+
+
+class BasePodFilter(BaseFilter):
+
+    def _filter_one(self, obj, pod_group):
+        return self.is_pod_passed(obj, pod_group)
+
+    def is_pod_passed(self, pod, pod_group):
+        raise NotImplementedError()

trio2o/common/scheduler/filters/bottom_pod_filter.py

@@ -0,0 +1,23 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from trio2o.common.scheduler.filters import base_filters
+
+
+class BottomPodFilter(base_filters.BasePodFilter):
+    """Returns all bottom pods."""
+
+    def is_pod_passed(self, pod, pod_group):
+        flag = False
+        if pod['az_name'] != '':
+            flag = True
+        return flag

trio2o/common/scheduler/pod_manager.py

@@ -0,0 +1,109 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from oslo_log import log as logging
+from stevedore import driver
+
+from trio2o.common.i18n import _LE
+from trio2o.db import api as db_api
+
+LOG = logging.getLogger(__name__)
+
+
+class PodManager(object):
+    def __init__(self):
+        filter_names = ['bottom_pod_filter']
+        self.default_filters = self._choose_pod_filters(filter_names)
+
+    @staticmethod
+    def _choose_pod_filters(filter_names):
+        good_filters = []
+        for filter_name in filter_names:
+            filter_ = driver.DriverManager(
+                'trio2o.common.schedulers',
+                filter_name,
+                invoke_on_load=True
+            ).driver
+            good_filters.append(filter_)
+        return good_filters
+
+    @staticmethod
+    def get_current_binding_and_pod(context, az_name, tenant_id, pod_group):
+        filter_b = [{'key': 'tenant_id', 'comparator': 'eq',
+                    'value': tenant_id}]
+        current_bindings = db_api.get_pod_binding_by_tenant_id(
+            context, filter_b)
+        if not current_bindings:
+            return None, None
+
+        has_available_pods = False
+        for pod_b in current_bindings:
+            if pod_b['is_binding']:
+                pod = db_api.get_pod_by_pod_id(context, pod_b['pod_id'])
+                if az_name and pod['az_name'] == az_name:
+                    has_available_pods = True
+                elif az_name == '' and pod['az_name'] != '':
+                    # if the az_name is not specified, a default bottom
+                    # pod will be selected
+                    has_available_pods = True
+
+                if has_available_pods:
+                    # TODO(Yipei): check resource_affinity_tag
+                    # if the resource utilization of the pod reaches the limit,
+                    # return [], []. Considering the feature of checking
+                    # resource utilization is not implemented, we use
+                    # resource_affinity_tag to test the logic of updating
+                    # a binding relationship.
+                    if pod_group != '':
+                        return pod_b, None
+                    # TODO(Yipei): check resource utilization of the pod
+                    # if the resource utilization of the pod reaches the limit,
+                    # return pod_b, []
+
+                    # If a pod passes the above checking, both the pod and its
+                    # corresponding binding are returned.
+                    return pod_b, pod
+        return None, None
+
+    @staticmethod
+    def create_binding(context, tenant_id, pod_id):
+        try:
+            db_api.create_pod_binding(context, tenant_id, pod_id)
+        except Exception as e:
+            LOG.error(_LE('Fail to create pod binding: %(exception)s'),
+                      {'exception': e})
+            return False
+        return True
+
+    @staticmethod
+    def update_binding(context, current_binding, pod_id):
+        current_binding['is_binding'] = False
+        try:
+            db_api.change_pod_binding(
+                context, current_binding, pod_id)
+        except Exception as e:
+            LOG.error(_LE('Fail to update pod binding: %(exception)s'),
+                      {'exception': e})
+            return False
+        return True
+
+    def get_available_pods(self, context, az_name, pod_group):
+        if az_name != '':
+            filter_q = [{'key': 'az_name',
+                         'comparator': 'eq', 'value': az_name}]
+        else:
+            filter_q = None
+        pods = db_api.list_pods(context, filter_q)
+        for filter_ in self.default_filters:
+            objs_ = filter_.filter_all(pods, pod_group)
+            pods = list(objs_)
+        return pods

trio2o/db/api.py

@@ -67,12 +67,42 @@ def update_pod(context, pod_id, update_dict):
         return core.update_resource(context, models.Pod, pod_id, update_dict)
 
 
+def change_pod_binding(context, pod_binding, pod_id):
+    with context.session.begin():
+        core.update_resource(context, models.PodBinding,
+                             pod_binding['id'], pod_binding)
+        core.create_resource(context, models.PodBinding,
+                             {'id': uuidutils.generate_uuid(),
+                              'tenant_id': pod_binding['tenant_id'],
+                              'pod_id': pod_id,
+                              'is_binding': True})
+
+
+def get_pod_binding_by_tenant_id(context, filter_):
+    with context.session.begin():
+        return core.query_resource(context, models.PodBinding, filter_, [])
+
+
+def get_pod_by_pod_id(context, pod_id):
+    with context.session.begin():
+        return core.get_resource(context, models.Pod, pod_id)
+
+
 def create_pod_service_configuration(context, config_dict):
     with context.session.begin():
         return core.create_resource(context, models.PodServiceConfiguration,
                                     config_dict)
 
 
+def create_pod_binding(context, tenant_id, pod_id):
+    with context.session.begin():
+        return core.create_resource(context, models.PodBinding,
+                                    {'id': uuidutils.generate_uuid(),
+                                     'tenant_id': tenant_id,
+                                     'pod_id': pod_id,
+                                     'is_binding': True})
+
+
 def delete_pod_service_configuration(context, config_id):
     with context.session.begin():
         return core.delete_resource(context, models.PodServiceConfiguration,

trio2o/db/migrate_repo/versions/001_init.py

@@ -47,6 +47,7 @@ def upgrade(migrate_engine):
         sql.Column('id', sql.String(36), primary_key=True),
         sql.Column('tenant_id', sql.String(length=255), nullable=False),
         sql.Column('pod_id', sql.String(length=255), nullable=False),
+        sql.Column('is_binding', sql.Boolean, nullable=False),
         sql.Column('created_at', sql.DateTime),
         sql.Column('updated_at', sql.DateTime),
         migrate.UniqueConstraint(

trio2o/db/models.py

@@ -420,7 +420,7 @@ class PodBinding(core.ModelBase, core.DictBase, models.TimestampMixin):
             'tenant_id', 'pod_id',
             name='pod_binding0tenant_id0pod_id'),
     )
-    attributes = ['id', 'tenant_id', 'pod_id',
+    attributes = ['id', 'tenant_id', 'pod_id', 'is_binding',
                   'created_at', 'updated_at']
 
     id = sql.Column(sql.String(36), primary_key=True)
@@ -428,6 +428,7 @@ class PodBinding(core.ModelBase, core.DictBase, models.TimestampMixin):
     pod_id = sql.Column('pod_id', sql.String(36),
                         sql.ForeignKey('cascaded_pods.pod_id'),
                         nullable=False)
+    is_binding = sql.Column('is_binding', sql.Boolean, nullable=False)
 
 
 # Routing Model

trio2o/nova_apigw/controllers/server.py

@@ -20,7 +20,6 @@ import six
 
 import oslo_log.log as logging
 
-from trio2o.common import az_ag
 import trio2o.common.client as t_client
 from trio2o.common import constants
 import trio2o.common.context as t_context
@@ -29,12 +28,14 @@ from trio2o.common.i18n import _
 from trio2o.common.i18n import _LE
 import trio2o.common.lock_handle as t_lock
 from trio2o.common.quota import QUOTAS
+from trio2o.common.scheduler import filter_scheduler
 from trio2o.common import utils
 from trio2o.common import xrpcapi
 import trio2o.db.api as db_api
 from trio2o.db import core
 from trio2o.db import models
 
+
 LOG = logging.getLogger(__name__)
 
 MAX_METADATA_KEY_LENGTH = 255
@@ -47,6 +48,7 @@ class ServerController(rest.RestController):
         self.project_id = project_id
         self.clients = {constants.TOP: t_client.Client()}
         self.xjob_handler = xrpcapi.XJobAPI()
+        self.filter_scheduler = filter_scheduler.FilterScheduler()
 
     def _get_client(self, pod_name=constants.TOP):
         if pod_name not in self.clients:
@@ -112,9 +114,9 @@ class ServerController(rest.RestController):
                 400, _('server is not set'))
 
         az = kw['server'].get('availability_zone', '')
+        pod, b_az = self.filter_scheduler.select_destination(
+            context, az, self.project_id, pod_group='')
 
-        pod, b_az = az_ag.get_pod_by_az_tenant(
-            context, az, self.project_id)
         if not pod:
             return utils.format_nova_error(
                 500, _('Pod not configured or scheduling failure'))

trio2o/tests/functional/api/controllers/test_pod.py

@@ -613,7 +613,8 @@ class TestBindingController(API_FunctionalTest):
                 "pod_binding":
                 {
                     "tenant_id": "dddddd",
-                    "pod_id": "0ace0db2-ef33-43a6-a150-42703ffda643"
+                    "pod_id": "0ace0db2-ef33-43a6-a150-42703ffda643",
+                    "is_binding": "True"
                 },
                 "expected_error": 200
             },
@@ -622,7 +623,8 @@ class TestBindingController(API_FunctionalTest):
                 "pod_binding":
                 {
                     "tenant_id": "aaaaa",
-                    "pod_id": "0ace0db2-ef33-43a6-a150-42703ffda643"
+                    "pod_id": "0ace0db2-ef33-43a6-a150-42703ffda643",
+                    "is_binding": "True"
                 },
                 "expected_error": 200
             },

trio2o/tests/unit/common/scheduler/test_filter_scheduler.py

@@ -0,0 +1,141 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from trio2o.common import context
+from trio2o.common.scheduler import filter_scheduler
+from trio2o.db import api
+from trio2o.db import core
+from trio2o.db import models
+import unittest
+
+
+class FilterSchedulerTest(unittest.TestCase):
+
+    def setUp(self):
+        core.initialize()
+        core.ModelBase.metadata.create_all(core.get_engine())
+        self.context = context.Context()
+        self.project_id = 'test_fs_project'
+        self.az_name_1 = 'b_az_fs_1'
+        self.az_name_2 = 'b_az_fs_2'
+        self.filter_scheduler = filter_scheduler.FilterScheduler()
+
+    def _prepare_binding(self, pod_id):
+        binding = {'tenant_id': self.project_id,
+                   'pod_id': pod_id,
+                   'is_binding': True}
+        api.create_pod_binding(self.context, self.project_id, pod_id)
+        return binding
+
+    def test_select_destination(self):
+        b_pod_1 = {'pod_id': 'b_pod_fs_uuid_1', 'pod_name': 'b_region_fs_1',
+                   'az_name': self.az_name_1}
+        api.create_pod(self.context, b_pod_1)
+        b_pod_2 = {'pod_id': 'b_pod_fs_uuid_2', 'pod_name': 'b_region_fs_2',
+                   'az_name': self.az_name_2}
+        api.create_pod(self.context, b_pod_2)
+        b_pod_3 = {'pod_id': 'b_pod_fs_uuid_3', 'pod_name': 'b_region_fs_3',
+                   'az_name': self.az_name_2}
+        api.create_pod(self.context, b_pod_3)
+
+        t_pod = {'pod_id': 'b_pod_fs_uuid_t_pod',
+                 'pod_name': 'b_region_fs_t_pod',
+                 'az_name': ''}
+        api.create_pod(self.context, t_pod)
+        self._prepare_binding(b_pod_1['pod_id'])
+        binding_q = core.query_resource(
+            self.context, models.PodBinding, [{'key': 'tenant_id',
+                                               'comparator': 'eq',
+                                               'value': self.project_id}], [])
+        self.assertEqual(binding_q[0]['pod_id'], b_pod_1['pod_id'])
+        self.assertEqual(binding_q[0]['tenant_id'], self.project_id)
+        self.assertEqual(binding_q[0]['is_binding'], True)
+
+        pod_1, _ = self.filter_scheduler.select_destination(
+            self.context, '', self.project_id, pod_group='')
+        self.assertEqual(pod_1['pod_id'], b_pod_1['pod_id'])
+        binding_q = core.query_resource(
+            self.context, models.PodBinding, [{'key': 'tenant_id',
+                                               'comparator': 'eq',
+                                               'value': self.project_id}], [])
+        self.assertEqual(len(binding_q), 1)
+        self.assertEqual(binding_q[0]['pod_id'], pod_1['pod_id'])
+        self.assertEqual(binding_q[0]['tenant_id'], self.project_id)
+        self.assertEqual(binding_q[0]['is_binding'], True)
+
+        pod_2, _ = self.filter_scheduler.select_destination(
+            self.context, '', 'new_project', pod_group='')
+        binding_q = core.query_resource(
+            self.context, models.PodBinding, [{'key': 'tenant_id',
+                                               'comparator': 'eq',
+                                               'value': 'new_project'}], [])
+        self.assertEqual(len(binding_q), 1)
+        self.assertEqual(binding_q[0]['pod_id'], pod_2['pod_id'])
+        self.assertEqual(binding_q[0]['tenant_id'], 'new_project')
+        self.assertEqual(binding_q[0]['is_binding'], True)
+
+        pod_3, _ = self.filter_scheduler.select_destination(
+            self.context, self.az_name_1, 'new_project', pod_group='')
+        binding_q = core.query_resource(
+            self.context, models.PodBinding, [{'key': 'tenant_id',
+                                               'comparator': 'eq',
+                                               'value': 'new_project'}], [])
+        self.assertEqual(len(binding_q), 1)
+        self.assertEqual(binding_q[0]['pod_id'], pod_3['pod_id'])
+        self.assertEqual(binding_q[0]['tenant_id'], 'new_project')
+        self.assertEqual(binding_q[0]['is_binding'], True)
+
+        pod_4, _ = self.filter_scheduler.select_destination(
+            self.context, self.az_name_2, 'new_project', pod_group='')
+        binding_q = core.query_resource(
+            self.context, models.PodBinding, [{'key': 'tenant_id',
+                                               'comparator': 'eq',
+                                               'value': 'new_project'}], [])
+        self.assertEqual(len(binding_q), 2)
+        self.assertEqual(binding_q[1]['pod_id'], pod_4['pod_id'])
+        self.assertEqual(binding_q[1]['tenant_id'], 'new_project')
+        self.assertEqual(binding_q[1]['is_binding'], True)
+
+        pod_5, _ = self.filter_scheduler.select_destination(
+            self.context, self.az_name_2, self.project_id, pod_group='')
+        binding_q = core.query_resource(
+            self.context, models.PodBinding, [{'key': 'tenant_id',
+                                               'comparator': 'eq',
+                                               'value': self.project_id}], [])
+        self.assertEqual(len(binding_q), 2)
+        self.assertEqual(pod_5['az_name'], self.az_name_2)
+        self.assertEqual(binding_q[1]['pod_id'], pod_5['pod_id'])
+        self.assertEqual(binding_q[1]['tenant_id'], self.project_id)
+        self.assertEqual(binding_q[1]['is_binding'], True)
+
+        pod_6, _ = self.filter_scheduler.select_destination(
+            self.context, self.az_name_1, self.project_id, pod_group='test')
+        binding_q = core.query_resource(
+            self.context, models.PodBinding, [{'key': 'tenant_id',
+                                               'comparator': 'eq',
+                                               'value': self.project_id}], [])
+        self.assertEqual(len(binding_q), 2)
+        self.assertEqual(pod_6, None)
+
+        pod_7, _ = self.filter_scheduler.select_destination(
+            self.context, self.az_name_2, self.project_id, pod_group='test')
+        binding_q = core.query_resource(
+            self.context, models.PodBinding, [{'key': 'tenant_id',
+                                               'comparator': 'eq',
+                                               'value': self.project_id}], [])
+        self.assertEqual(len(binding_q), 3)
+        self.assertEqual(pod_7['az_name'], self.az_name_2)
+        self.assertEqual(binding_q[1]['tenant_id'], self.project_id)
+        self.assertEqual(binding_q[1]['is_binding'], False)
+        self.assertEqual(binding_q[2]['pod_id'], pod_7['pod_id'])
+        self.assertEqual(binding_q[2]['tenant_id'], self.project_id)
+        self.assertEqual(binding_q[2]['is_binding'], True)

trio2o/tests/unit/common/scheduler/test_pod_manager.py

@@ -0,0 +1,141 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from stevedore import driver
+
+from trio2o.common import context
+from trio2o.db import api
+from trio2o.db import core
+from trio2o.db import models
+
+import unittest
+
+
+class PodManagerTest(unittest.TestCase):
+    def setUp(self):
+        core.initialize()
+        core.ModelBase.metadata.create_all(core.get_engine())
+        self.context = context.Context()
+        self.project_id = 'test_pm_project'
+        self.az_name_2 = 'b_az_pm_2'
+        self.az_name_1 = 'b_az_pm_1'
+        self.pod_manager = driver.DriverManager(
+            namespace='trio2o.common.schedulers',
+            name='pod_manager',
+            invoke_on_load=True
+        ).driver
+        self.b_pod_1 = {'pod_id': 'b_pod_pm_uuid_1',
+                        'pod_name': 'b_region_pm_1',
+                        'az_name': self.az_name_1}
+
+        self.b_pod_2 = {'pod_id': 'b_pod_pm_uuid_2',
+                        'pod_name': 'b_region_pm_2',
+                        'az_name': self.az_name_2}
+
+        self.b_pod_3 = {'pod_id': 'b_pod_pm_uuid_3',
+                        'pod_name': 'b_region_pm_3',
+                        'az_name': self.az_name_2}
+
+        self.b_pod_4 = {'pod_id': 'b_pod_pm_uuid_4',
+                        'pod_name': 'b_region_pm_4',
+                        'az_name': self.az_name_2}
+
+    def test_get_current_binding_and_pod(self):
+        api.create_pod(self.context, self.b_pod_1)
+        api.create_pod_binding(
+            self.context, self.project_id, self.b_pod_1['pod_id'])
+
+        pod_b_1, pod_1 = self.pod_manager.get_current_binding_and_pod(
+            self.context, self.az_name_1, self.project_id, pod_group='')
+        binding_q = core.query_resource(
+            self.context, models.PodBinding,
+            [{'key': 'tenant_id',
+              'comparator': 'eq',
+              'value': self.project_id}], [])
+        self.assertEqual(len(binding_q), 1)
+        self.assertEqual(binding_q[0]['id'], pod_b_1['id'])
+
+        pod_b_2, pod_2 = self.pod_manager.get_current_binding_and_pod(
+            self.context, self.az_name_1, 'new_project_pm_1', pod_group='')
+        binding_q = core.query_resource(
+            self.context, models.PodBinding,
+            [{'key': 'tenant_id',
+              'comparator': 'eq',
+              'value': 'new_project_pm_1'}], [])
+        self.assertEqual(len(binding_q), 0)
+        self.assertEqual(pod_b_2, None)
+        self.assertEqual(pod_2, None)
+
+        pod_b_3, pod_3 = self.pod_manager.get_current_binding_and_pod(
+            self.context, 'unknown_az', self.project_id, pod_group='')
+        binding_q = core.query_resource(
+            self.context, models.PodBinding,
+            [{'key': 'tenant_id',
+              'comparator': 'eq',
+              'value': self.project_id}], [])
+        self.assertEqual(len(binding_q), 1)
+        self.assertEqual(pod_b_3, None)
+        self.assertEqual(pod_3, None)
+
+        pod_b_4, pod_4 = self.pod_manager.get_current_binding_and_pod(
+            self.context, self.az_name_1, self.project_id, pod_group='test')
+        binding_q = core.query_resource(
+            self.context, models.PodBinding,
+            [{'key': 'tenant_id',
+              'comparator': 'eq',
+              'value': self.project_id}], [])
+        self.assertEqual(len(binding_q), 1)
+        self.assertEqual(pod_b_4['id'], binding_q[0]['id'])
+        self.assertEqual(pod_4, None)
+
+    def test_create_binding(self):
+        api.create_pod(self.context, self.b_pod_2)
+        flag = self.pod_manager.create_binding(
+            self.context, 'new_project_pm_2', self.b_pod_2['pod_id'])
+        self.assertEqual(flag, True)
+        binding_q = core.query_resource(
+            self.context, models.PodBinding,
+            [{'key': 'tenant_id',
+              'comparator': 'eq',
+              'value': 'new_project_pm_2'}], [])
+        self.assertEqual(len(binding_q), 1)
+        self.assertEqual(binding_q[0]['pod_id'], self.b_pod_2['pod_id'])
+        self.assertEqual(binding_q[0]['tenant_id'], 'new_project_pm_2')
+        self.assertEqual(binding_q[0]['is_binding'], True)
+
+    def test_update_binding(self):
+        api.create_pod(self.context, self.b_pod_4)
+        api.create_pod(self.context, self.b_pod_3)
+        flag = self.pod_manager.create_binding(
+            self.context, 'new_project_pm_3', self.b_pod_3['pod_id'])
+        self.assertEqual(flag, True)
+        current_binding = core.query_resource(
+            self.context, models.PodBinding,
+            [{'key': 'tenant_id',
+              'comparator': 'eq',
+              'value': 'new_project_pm_3'}], [])
+
+        flag = self.pod_manager.update_binding(
+            self.context, current_binding[0], self.b_pod_4['pod_id'])
+        self.assertEqual(flag, True)
+        binding_q = core.query_resource(
+            self.context, models.PodBinding,
+            [{'key': 'tenant_id',
+              'comparator': 'eq',
+              'value': 'new_project_pm_3'}], [])
+        self.assertEqual(len(binding_q), 2)
+        self.assertEqual(binding_q[0]['pod_id'], self.b_pod_3['pod_id'])
+        self.assertEqual(binding_q[0]['tenant_id'], 'new_project_pm_3')
+        self.assertEqual(binding_q[0]['is_binding'], False)
+        self.assertEqual(binding_q[1]['pod_id'], self.b_pod_4['pod_id'])
+        self.assertEqual(binding_q[1]['tenant_id'], 'new_project_pm_3')
+        self.assertEqual(binding_q[1]['is_binding'], True)

trio2o/tests/unit/common/test_az_ag.py

@@ -133,6 +133,7 @@ class AZAGTest(unittest.TestCase):
             self.assertEqual(pod2['pod_name'], FAKE_SITE_NAME)
             self.assertEqual(pod2['pod_id'], FAKE_SITE_ID)
             self.assertEqual(pod2['az_name'], FAKE_AZ)
+
         else:
             self.assertEqual(pod2['pod_name'], FAKE_SITE_NAME_2)
             self.assertEqual(pod2['pod_id'], FAKE_SITE_ID_2)

trio2o/tests/unit/nova_apigw/controllers/test_server.py

@@ -26,6 +26,7 @@ from trio2o.common import constants
 from trio2o.common import context
 import trio2o.common.exceptions as t_exceptions
 from trio2o.common import lock_handle
+from trio2o.common.scheduler import filter_scheduler
 from trio2o.common import xrpcapi
 from trio2o.db import api
 from trio2o.db import core
@@ -82,6 +83,7 @@ class FakeServerController(server.ServerController):
         self.clients = {'t_region': FakeClient('t_region')}
         self.project_id = project_id
         self.xjob_handler = xrpcapi.XJobAPI()
+        self.filter_scheduler = filter_scheduler.FilterScheduler()
 
     def _get_client(self, pod_name=None):
         if not pod_name: