The framework of dynamic pod binding

1. What is the problem?
In the Tricircle, each tenant is bound to multiple pods,
where it creates various types of resources. However such a binding
relationship should be dynamic instead of static. For instance when
some resources in a pod are exhausted, tenant needs to be bound to a
new pod in same AZ.

2. What is the solution to the problem?
To deal with the above problem, the Tricircle dynamically bind tenants
to pod which has available resources. We call this feature dynamic pod
binding, which is explained in https://review.openstack.org/#/c/306224/
in detail. In this patch, we only try to binds a tenant to a pod
dynamically, when she tries to create a VM.

3. What the features need to be implemented to the Tricircle to realize
the solution?
When a tenant creates a VM, the Tricircle first selects all available
pods for her. Then by filtering and weighing the pods, the Tricircle
selects the most suitable pod for the tenant. Next, the Tricircle query
database for current binding relationship of the tenant. If the tenant
is not bound to any pod, we create a new binding relationship, which
binds the tenant to the selected pod. If the tenant is already bound to
a pod, and the pod is not the pod selected by the Tricircle, we update
current binding relationship, which binds the tenant to a new pod. If
the tenant is already bound to a pod, and the pod is exactly the pod
selected by the Tricircle, the Tricircle does noting.

Change-Id: I3972e6799f78da6ec35be556487f79b1234731b8

setup.cfg

@@ -61,3 +61,8 @@ tempest.test_plugins =
 tricircle.network.type_drivers =
     local = tricircle.network.drivers.type_local:LocalTypeDriver
     shared_vlan = tricircle.network.drivers.type_shared_vlan:SharedVLANTypeDriver
+
+tricircle.common.schedulers =
+    pod_manager = tricircle.common.scheduler.pod_manager:PodManager
+    bottom_pod_filter = tricircle.common.scheduler.filters.bottom_pod_filter:BottomPodFilter
+    filter_scheduler = tricircle.common.scheduler.filter_scheduler:FilterScheduler

tricircle/api/controllers/pod.py

@@ -224,7 +224,6 @@ class BindingsController(rest.RestController):
         pod_b = kw['pod_binding']
         tenant_id = pod_b.get('tenant_id', '').strip()
         pod_id = pod_b.get('pod_id', '').strip()
-        _uuid = uuidutils.generate_uuid()
 
         if tenant_id == '' or pod_id == '':
             return Response(
@@ -250,11 +249,8 @@ class BindingsController(rest.RestController):
             return
 
         try:
-            with context.session.begin():
-                pod_binding = core.create_resource(context, models.PodBinding,
-                                                   {'id': _uuid,
-                                                    'tenant_id': tenant_id,
-                                                    'pod_id': pod_id})
+            pod_binding = db_api.create_pod_binding(
+                context, tenant_id, pod_id)
         except db_exc.DBDuplicateEntry:
             return Response(_('Pod binding already exists'), 409)
         except db_exc.DBConstraintError:

tricircle/cinder_apigw/controllers/volume.py

@@ -29,6 +29,7 @@ import tricircle.common.context as t_context
 from tricircle.common import httpclient as hclient
 from tricircle.common.i18n import _
 from tricircle.common.i18n import _LE
+from tricircle.common.scheduler import filter_scheduler
 from tricircle.common import utils
 
 import tricircle.db.api as db_api
@@ -42,6 +43,7 @@ class VolumeController(rest.RestController):
 
     def __init__(self, tenant_id):
         self.tenant_id = tenant_id
+        self.filter_scheduler = filter_scheduler.FilterScheduler()
 
     @expose(generic=True, template='json')
     def post(self, **kw):
@@ -52,10 +54,9 @@ class VolumeController(rest.RestController):
                 400, _("Missing required element 'volume' in request body."))
 
         az = kw['volume'].get('availability_zone', '')
-        pod, pod_az = az_ag.get_pod_by_az_tenant(
-            context,
-            az_name=az,
-            tenant_id=self.tenant_id)
+        pod, pod_az = self.filter_scheduler.select_destination(
+            context, az, self.tenant_id, pod_group='')
+
         if not pod:
             LOG.error(_LE("Pod not configured or scheduling failure"))
             return utils.format_cinder_error(

tricircle/common/az_ag.py

@@ -147,7 +147,8 @@ def get_pod_by_az_tenant(context, az_name, tenant_id):
                         context, models.PodBinding,
                         {'id': uuidutils.generate_uuid(),
                          'tenant_id': tenant_id,
-                         'pod_id': pod['pod_id']})
+                         'pod_id': pod['pod_id'],
+                         'is_binding': True})
                     return pod, pod['pod_az_name']
             except Exception as e:
                 LOG.error(_LE('Fail to create pod binding: %(exception)s'),

tricircle/common/scheduler/driver.py

@@ -0,0 +1,31 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import abc
+import six
+
+from stevedore import driver
+
+
+@six.add_metaclass(abc.ABCMeta)
+class Scheduler(object):
+
+    def __init__(self):
+        self.pod_manager = driver.DriverManager(
+            namespace='tricircle.common.schedulers',
+            name='pod_manager',
+            invoke_on_load=True
+        ).driver
+
+    @abc.abstractmethod
+    def select_destination(self, context, az_name, tenant_id, spec_obj):
+        return None, None

tricircle/common/scheduler/filter_scheduler.py

@@ -0,0 +1,58 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from tricircle.common.scheduler import driver
+
+
+class FilterScheduler(driver.Scheduler):
+
+    def __init__(self, *args, **kwargs):
+        super(FilterScheduler, self).__init__(*args, **kwargs)
+
+    def select_destination(self, context, az_name, tenant_id, pod_group):
+        current_binding, current_pod = \
+            self.pod_manager.get_current_binding_and_pod(
+                context, az_name, tenant_id, pod_group)
+
+        if current_binding and current_pod:
+            return current_pod, current_pod['pod_az_name']
+        else:
+            pods = self.pod_manager.get_available_pods(
+                context, az_name, pod_group)
+            if not pods:
+                return None, None
+            # TODO(Yipei): Weigh pods and select one whose weight
+            # is the maximum. Here we chose one randomly.
+            is_current = False
+            best_pod = None
+            # select the pod by a circle in pods
+            for pod in pods:
+                if is_current:
+                    best_pod = pod
+                    break
+                if current_binding \
+                        and pod['pod_id'] == current_binding['pod_id']:
+                    is_current = True
+            if is_current and len(pods) == 1:
+                return None, None
+            if not best_pod:
+                best_pod = pods[0]
+
+            if current_binding:
+                is_successful = self.pod_manager.update_binding(
+                    context, current_binding, best_pod['pod_id'])
+            else:
+                is_successful = self.pod_manager.create_binding(
+                    context, tenant_id, best_pod['pod_id'])
+            if not is_successful:
+                return None, None
+            return best_pod, best_pod['pod_az_name']

tricircle/common/scheduler/filters/base_filters.py

@@ -0,0 +1,31 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+class BaseFilter(object):
+    """Base class for all pod filter classes."""
+    def _filter_one(self, obj, pod_group):
+        return True
+
+    def filter_all(self, filter_obj_list, pod_group):
+        for obj in filter_obj_list:
+            if self._filter_one(obj, pod_group):
+                yield obj
+
+
+class BasePodFilter(BaseFilter):
+
+    def _filter_one(self, obj, pod_group):
+        return self.is_pod_passed(obj, pod_group)
+
+    def is_pod_passed(self, pod, pod_group):
+        raise NotImplementedError()

tricircle/common/scheduler/filters/bottom_pod_filter.py

@@ -0,0 +1,23 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from tricircle.common.scheduler.filters import base_filters
+
+
+class BottomPodFilter(base_filters.BasePodFilter):
+    """Returns all bottom pods."""
+
+    def is_pod_passed(self, pod, pod_group):
+        flag = False
+        if pod['az_name'] != '':
+            flag = True
+        return flag

tricircle/common/scheduler/pod_manager.py

@@ -0,0 +1,109 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from oslo_log import log as logging
+from stevedore import driver
+
+from tricircle.common.i18n import _LE
+from tricircle.db import api as db_api
+
+LOG = logging.getLogger(__name__)
+
+
+class PodManager(object):
+    def __init__(self):
+        filter_names = ['bottom_pod_filter']
+        self.default_filters = self._choose_pod_filters(filter_names)
+
+    @staticmethod
+    def _choose_pod_filters(filter_names):
+        good_filters = []
+        for filter_name in filter_names:
+            filter_ = driver.DriverManager(
+                'tricircle.common.schedulers',
+                filter_name,
+                invoke_on_load=True
+            ).driver
+            good_filters.append(filter_)
+        return good_filters
+
+    @staticmethod
+    def get_current_binding_and_pod(context, az_name, tenant_id, pod_group):
+        filter_b = [{'key': 'tenant_id', 'comparator': 'eq',
+                    'value': tenant_id}]
+        current_bindings = db_api.get_pod_binding_by_tenant_id(
+            context, filter_b)
+        if not current_bindings:
+            return None, None
+
+        has_available_pods = False
+        for pod_b in current_bindings:
+            if pod_b['is_binding']:
+                pod = db_api.get_pod_by_pod_id(context, pod_b['pod_id'])
+                if az_name and pod['az_name'] == az_name:
+                    has_available_pods = True
+                elif az_name == '' and pod['az_name'] != '':
+                    # if the az_name is not specified, a default bottom
+                    # pod will be selected
+                    has_available_pods = True
+
+                if has_available_pods:
+                    # TODO(Yipei): check resource_affinity_tag
+                    # if the resource utilization of the pod reaches the limit,
+                    # return [], []. Considering the feature of checking
+                    # resource utilization is not implemented, we use
+                    # resource_affinity_tag to test the logic of updating
+                    # a binding relationship.
+                    if pod_group != '':
+                        return pod_b, None
+                    # TODO(Yipei): check resource utilization of the pod
+                    # if the resource utilization of the pod reaches the limit,
+                    # return pod_b, []
+
+                    # If a pod passes the above checking, both the pod and its
+                    # corresponding binding are returned.
+                    return pod_b, pod
+        return None, None
+
+    @staticmethod
+    def create_binding(context, tenant_id, pod_id):
+        try:
+            db_api.create_pod_binding(context, tenant_id, pod_id)
+        except Exception as e:
+            LOG.error(_LE('Fail to create pod binding: %(exception)s'),
+                      {'exception': e})
+            return False
+        return True
+
+    @staticmethod
+    def update_binding(context, current_binding, pod_id):
+        current_binding['is_binding'] = False
+        try:
+            db_api.change_pod_binding(
+                context, current_binding, pod_id)
+        except Exception as e:
+            LOG.error(_LE('Fail to update pod binding: %(exception)s'),
+                      {'exception': e})
+            return False
+        return True
+
+    def get_available_pods(self, context, az_name, pod_group):
+        if az_name != '':
+            filter_q = [{'key': 'az_name',
+                         'comparator': 'eq', 'value': az_name}]
+        else:
+            filter_q = None
+        pods = db_api.list_pods(context, filter_q)
+        for filter_ in self.default_filters:
+            objs_ = filter_.filter_all(pods, pod_group)
+            pods = list(objs_)
+        return pods

tricircle/db/api.py

@@ -67,12 +67,42 @@ def update_pod(context, pod_id, update_dict):
         return core.update_resource(context, models.Pod, pod_id, update_dict)
 
 
+def change_pod_binding(context, pod_binding, pod_id):
+    with context.session.begin():
+        core.update_resource(context, models.PodBinding,
+                             pod_binding['id'], pod_binding)
+        core.create_resource(context, models.PodBinding,
+                             {'id': uuidutils.generate_uuid(),
+                              'tenant_id': pod_binding['tenant_id'],
+                              'pod_id': pod_id,
+                              'is_binding': True})
+
+
+def get_pod_binding_by_tenant_id(context, filter_):
+    with context.session.begin():
+        return core.query_resource(context, models.PodBinding, filter_, [])
+
+
+def get_pod_by_pod_id(context, pod_id):
+    with context.session.begin():
+        return core.get_resource(context, models.Pod, pod_id)
+
+
 def create_pod_service_configuration(context, config_dict):
     with context.session.begin():
         return core.create_resource(context, models.PodServiceConfiguration,
                                     config_dict)
 
 
+def create_pod_binding(context, tenant_id, pod_id):
+    with context.session.begin():
+        return core.create_resource(context, models.PodBinding,
+                                    {'id': uuidutils.generate_uuid(),
+                                     'tenant_id': tenant_id,
+                                     'pod_id': pod_id,
+                                     'is_binding': True})
+
+
 def delete_pod_service_configuration(context, config_id):
     with context.session.begin():
         return core.delete_resource(context, models.PodServiceConfiguration,

tricircle/db/migrate_repo/versions/001_init.py

@@ -47,6 +47,7 @@ def upgrade(migrate_engine):
         sql.Column('id', sql.String(36), primary_key=True),
         sql.Column('tenant_id', sql.String(length=255), nullable=False),
         sql.Column('pod_id', sql.String(length=255), nullable=False),
+        sql.Column('is_binding', sql.Boolean, nullable=False),
         sql.Column('created_at', sql.DateTime),
         sql.Column('updated_at', sql.DateTime),
         migrate.UniqueConstraint(

tricircle/db/models.py

@@ -420,7 +420,7 @@ class PodBinding(core.ModelBase, core.DictBase, models.TimestampMixin):
             'tenant_id', 'pod_id',
             name='pod_binding0tenant_id0pod_id'),
     )
-    attributes = ['id', 'tenant_id', 'pod_id',
+    attributes = ['id', 'tenant_id', 'pod_id', 'is_binding',
                   'created_at', 'updated_at']
 
     id = sql.Column(sql.String(36), primary_key=True)
@@ -428,6 +428,7 @@ class PodBinding(core.ModelBase, core.DictBase, models.TimestampMixin):
     pod_id = sql.Column('pod_id', sql.String(36),
                         sql.ForeignKey('cascaded_pods.pod_id'),
                         nullable=False)
+    is_binding = sql.Column('is_binding', sql.Boolean, nullable=False)
 
 
 # Routing Model

tricircle/nova_apigw/controllers/server.py

@@ -23,7 +23,6 @@ import oslo_log.log as logging
 
 import neutronclient.common.exceptions as q_exceptions
 
-from tricircle.common import az_ag
 import tricircle.common.client as t_client
 from tricircle.common import constants
 import tricircle.common.context as t_context
@@ -32,6 +31,7 @@ from tricircle.common.i18n import _
 from tricircle.common.i18n import _LE
 import tricircle.common.lock_handle as t_lock
 from tricircle.common.quota import QUOTAS
+from tricircle.common.scheduler import filter_scheduler
 from tricircle.common import utils
 from tricircle.common import xrpcapi
 import tricircle.db.api as db_api
@@ -39,6 +39,7 @@ from tricircle.db import core
 from tricircle.db import models
 from tricircle.network import helper
 
+
 LOG = logging.getLogger(__name__)
 
 MAX_METADATA_KEY_LENGTH = 255
@@ -52,6 +53,7 @@ class ServerController(rest.RestController):
         self.clients = {constants.TOP: t_client.Client()}
         self.helper = helper.NetworkHelper()
         self.xjob_handler = xrpcapi.XJobAPI()
+        self.filter_scheduler = filter_scheduler.FilterScheduler()
 
     def _get_client(self, pod_name=constants.TOP):
         if pod_name not in self.clients:
@@ -128,9 +130,8 @@ class ServerController(rest.RestController):
                 400, _('server is not set'))
 
         az = kw['server'].get('availability_zone', '')
-
-        pod, b_az = az_ag.get_pod_by_az_tenant(
-            context, az, self.project_id)
+        pod, b_az = self.filter_scheduler.select_destination(
+            context, az, self.project_id, pod_group='')
         if not pod:
             return utils.format_nova_error(
                 500, _('Pod not configured or scheduling failure'))

tricircle/tests/functional/api/controllers/test_pod.py

@@ -613,7 +613,8 @@ class TestBindingController(API_FunctionalTest):
                 "pod_binding":
                 {
                     "tenant_id": "dddddd",
-                    "pod_id": "0ace0db2-ef33-43a6-a150-42703ffda643"
+                    "pod_id": "0ace0db2-ef33-43a6-a150-42703ffda643",
+                    "is_binding": "True"
                 },
                 "expected_error": 200
             },
@@ -622,7 +623,8 @@ class TestBindingController(API_FunctionalTest):
                 "pod_binding":
                 {
                     "tenant_id": "aaaaa",
-                    "pod_id": "0ace0db2-ef33-43a6-a150-42703ffda643"
+                    "pod_id": "0ace0db2-ef33-43a6-a150-42703ffda643",
+                    "is_binding": "True"
                 },
                 "expected_error": 200
             },

tricircle/tests/unit/common/scheduler/test_filter_scheduler.py

@@ -0,0 +1,141 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from tricircle.common import context
+from tricircle.common.scheduler import filter_scheduler
+from tricircle.db import api
+from tricircle.db import core
+from tricircle.db import models
+import unittest
+
+
+class FilterSchedulerTest(unittest.TestCase):
+
+    def setUp(self):
+        core.initialize()
+        core.ModelBase.metadata.create_all(core.get_engine())
+        self.context = context.Context()
+        self.project_id = 'test_fs_project'
+        self.az_name_1 = 'b_az_fs_1'
+        self.az_name_2 = 'b_az_fs_2'
+        self.filter_scheduler = filter_scheduler.FilterScheduler()
+
+    def _prepare_binding(self, pod_id):
+        binding = {'tenant_id': self.project_id,
+                   'pod_id': pod_id,
+                   'is_binding': True}
+        api.create_pod_binding(self.context, self.project_id, pod_id)
+        return binding
+
+    def test_select_destination(self):
+        b_pod_1 = {'pod_id': 'b_pod_fs_uuid_1', 'pod_name': 'b_region_fs_1',
+                   'az_name': self.az_name_1}
+        api.create_pod(self.context, b_pod_1)
+        b_pod_2 = {'pod_id': 'b_pod_fs_uuid_2', 'pod_name': 'b_region_fs_2',
+                   'az_name': self.az_name_2}
+        api.create_pod(self.context, b_pod_2)
+        b_pod_3 = {'pod_id': 'b_pod_fs_uuid_3', 'pod_name': 'b_region_fs_3',
+                   'az_name': self.az_name_2}
+        api.create_pod(self.context, b_pod_3)
+
+        t_pod = {'pod_id': 'b_pod_fs_uuid_t_pod',
+                 'pod_name': 'b_region_fs_t_pod',
+                 'az_name': ''}
+        api.create_pod(self.context, t_pod)
+        self._prepare_binding(b_pod_1['pod_id'])
+        binding_q = core.query_resource(
+            self.context, models.PodBinding, [{'key': 'tenant_id',
+                                               'comparator': 'eq',
+                                               'value': self.project_id}], [])
+        self.assertEqual(binding_q[0]['pod_id'], b_pod_1['pod_id'])
+        self.assertEqual(binding_q[0]['tenant_id'], self.project_id)
+        self.assertEqual(binding_q[0]['is_binding'], True)
+
+        pod_1, _ = self.filter_scheduler.select_destination(
+            self.context, '', self.project_id, pod_group='')
+        self.assertEqual(pod_1['pod_id'], b_pod_1['pod_id'])
+        binding_q = core.query_resource(
+            self.context, models.PodBinding, [{'key': 'tenant_id',
+                                               'comparator': 'eq',
+                                               'value': self.project_id}], [])
+        self.assertEqual(len(binding_q), 1)
+        self.assertEqual(binding_q[0]['pod_id'], pod_1['pod_id'])
+        self.assertEqual(binding_q[0]['tenant_id'], self.project_id)
+        self.assertEqual(binding_q[0]['is_binding'], True)
+
+        pod_2, _ = self.filter_scheduler.select_destination(
+            self.context, '', 'new_project', pod_group='')
+        binding_q = core.query_resource(
+            self.context, models.PodBinding, [{'key': 'tenant_id',
+                                               'comparator': 'eq',
+                                               'value': 'new_project'}], [])
+        self.assertEqual(len(binding_q), 1)
+        self.assertEqual(binding_q[0]['pod_id'], pod_2['pod_id'])
+        self.assertEqual(binding_q[0]['tenant_id'], 'new_project')
+        self.assertEqual(binding_q[0]['is_binding'], True)
+
+        pod_3, _ = self.filter_scheduler.select_destination(
+            self.context, self.az_name_1, 'new_project', pod_group='')
+        binding_q = core.query_resource(
+            self.context, models.PodBinding, [{'key': 'tenant_id',
+                                               'comparator': 'eq',
+                                               'value': 'new_project'}], [])
+        self.assertEqual(len(binding_q), 1)
+        self.assertEqual(binding_q[0]['pod_id'], pod_3['pod_id'])
+        self.assertEqual(binding_q[0]['tenant_id'], 'new_project')
+        self.assertEqual(binding_q[0]['is_binding'], True)
+
+        pod_4, _ = self.filter_scheduler.select_destination(
+            self.context, self.az_name_2, 'new_project', pod_group='')
+        binding_q = core.query_resource(
+            self.context, models.PodBinding, [{'key': 'tenant_id',
+                                               'comparator': 'eq',
+                                               'value': 'new_project'}], [])
+        self.assertEqual(len(binding_q), 2)
+        self.assertEqual(binding_q[1]['pod_id'], pod_4['pod_id'])
+        self.assertEqual(binding_q[1]['tenant_id'], 'new_project')
+        self.assertEqual(binding_q[1]['is_binding'], True)
+
+        pod_5, _ = self.filter_scheduler.select_destination(
+            self.context, self.az_name_2, self.project_id, pod_group='')
+        binding_q = core.query_resource(
+            self.context, models.PodBinding, [{'key': 'tenant_id',
+                                               'comparator': 'eq',
+                                               'value': self.project_id}], [])
+        self.assertEqual(len(binding_q), 2)
+        self.assertEqual(pod_5['az_name'], self.az_name_2)
+        self.assertEqual(binding_q[1]['pod_id'], pod_5['pod_id'])
+        self.assertEqual(binding_q[1]['tenant_id'], self.project_id)
+        self.assertEqual(binding_q[1]['is_binding'], True)
+
+        pod_6, _ = self.filter_scheduler.select_destination(
+            self.context, self.az_name_1, self.project_id, pod_group='test')
+        binding_q = core.query_resource(
+            self.context, models.PodBinding, [{'key': 'tenant_id',
+                                               'comparator': 'eq',
+                                               'value': self.project_id}], [])
+        self.assertEqual(len(binding_q), 2)
+        self.assertEqual(pod_6, None)
+
+        pod_7, _ = self.filter_scheduler.select_destination(
+            self.context, self.az_name_2, self.project_id, pod_group='test')
+        binding_q = core.query_resource(
+            self.context, models.PodBinding, [{'key': 'tenant_id',
+                                               'comparator': 'eq',
+                                               'value': self.project_id}], [])
+        self.assertEqual(len(binding_q), 3)
+        self.assertEqual(pod_7['az_name'], self.az_name_2)
+        self.assertEqual(binding_q[1]['tenant_id'], self.project_id)
+        self.assertEqual(binding_q[1]['is_binding'], False)
+        self.assertEqual(binding_q[2]['pod_id'], pod_7['pod_id'])
+        self.assertEqual(binding_q[2]['tenant_id'], self.project_id)
+        self.assertEqual(binding_q[2]['is_binding'], True)

tricircle/tests/unit/common/scheduler/test_pod_manager.py

@@ -0,0 +1,141 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from stevedore import driver
+
+from tricircle.common import context
+from tricircle.db import api
+from tricircle.db import core
+from tricircle.db import models
+
+import unittest
+
+
+class PodManagerTest(unittest.TestCase):
+    def setUp(self):
+        core.initialize()
+        core.ModelBase.metadata.create_all(core.get_engine())
+        self.context = context.Context()
+        self.project_id = 'test_pm_project'
+        self.az_name_2 = 'b_az_pm_2'
+        self.az_name_1 = 'b_az_pm_1'
+        self.pod_manager = driver.DriverManager(
+            namespace='tricircle.common.schedulers',
+            name='pod_manager',
+            invoke_on_load=True
+        ).driver
+        self.b_pod_1 = {'pod_id': 'b_pod_pm_uuid_1',
+                        'pod_name': 'b_region_pm_1',
+                        'az_name': self.az_name_1}
+
+        self.b_pod_2 = {'pod_id': 'b_pod_pm_uuid_2',
+                        'pod_name': 'b_region_pm_2',
+                        'az_name': self.az_name_2}
+
+        self.b_pod_3 = {'pod_id': 'b_pod_pm_uuid_3',
+                        'pod_name': 'b_region_pm_3',
+                        'az_name': self.az_name_2}
+
+        self.b_pod_4 = {'pod_id': 'b_pod_pm_uuid_4',
+                        'pod_name': 'b_region_pm_4',
+                        'az_name': self.az_name_2}
+
+    def test_get_current_binding_and_pod(self):
+        api.create_pod(self.context, self.b_pod_1)
+        api.create_pod_binding(
+            self.context, self.project_id, self.b_pod_1['pod_id'])
+
+        pod_b_1, pod_1 = self.pod_manager.get_current_binding_and_pod(
+            self.context, self.az_name_1, self.project_id, pod_group='')
+        binding_q = core.query_resource(
+            self.context, models.PodBinding,
+            [{'key': 'tenant_id',
+              'comparator': 'eq',
+              'value': self.project_id}], [])
+        self.assertEqual(len(binding_q), 1)
+        self.assertEqual(binding_q[0]['id'], pod_b_1['id'])
+
+        pod_b_2, pod_2 = self.pod_manager.get_current_binding_and_pod(
+            self.context, self.az_name_1, 'new_project_pm_1', pod_group='')
+        binding_q = core.query_resource(
+            self.context, models.PodBinding,
+            [{'key': 'tenant_id',
+              'comparator': 'eq',
+              'value': 'new_project_pm_1'}], [])
+        self.assertEqual(len(binding_q), 0)
+        self.assertEqual(pod_b_2, None)
+        self.assertEqual(pod_2, None)
+
+        pod_b_3, pod_3 = self.pod_manager.get_current_binding_and_pod(
+            self.context, 'unknown_az', self.project_id, pod_group='')
+        binding_q = core.query_resource(
+            self.context, models.PodBinding,
+            [{'key': 'tenant_id',
+              'comparator': 'eq',
+              'value': self.project_id}], [])
+        self.assertEqual(len(binding_q), 1)
+        self.assertEqual(pod_b_3, None)
+        self.assertEqual(pod_3, None)
+
+        pod_b_4, pod_4 = self.pod_manager.get_current_binding_and_pod(
+            self.context, self.az_name_1, self.project_id, pod_group='test')
+        binding_q = core.query_resource(
+            self.context, models.PodBinding,
+            [{'key': 'tenant_id',
+              'comparator': 'eq',
+              'value': self.project_id}], [])
+        self.assertEqual(len(binding_q), 1)
+        self.assertEqual(pod_b_4['id'], binding_q[0]['id'])
+        self.assertEqual(pod_4, None)
+
+    def test_create_binding(self):
+        api.create_pod(self.context, self.b_pod_2)
+        flag = self.pod_manager.create_binding(
+            self.context, 'new_project_pm_2', self.b_pod_2['pod_id'])
+        self.assertEqual(flag, True)
+        binding_q = core.query_resource(
+            self.context, models.PodBinding,
+            [{'key': 'tenant_id',
+              'comparator': 'eq',
+              'value': 'new_project_pm_2'}], [])
+        self.assertEqual(len(binding_q), 1)
+        self.assertEqual(binding_q[0]['pod_id'], self.b_pod_2['pod_id'])
+        self.assertEqual(binding_q[0]['tenant_id'], 'new_project_pm_2')
+        self.assertEqual(binding_q[0]['is_binding'], True)
+
+    def test_update_binding(self):
+        api.create_pod(self.context, self.b_pod_4)
+        api.create_pod(self.context, self.b_pod_3)
+        flag = self.pod_manager.create_binding(
+            self.context, 'new_project_pm_3', self.b_pod_3['pod_id'])
+        self.assertEqual(flag, True)
+        current_binding = core.query_resource(
+            self.context, models.PodBinding,
+            [{'key': 'tenant_id',
+              'comparator': 'eq',
+              'value': 'new_project_pm_3'}], [])
+
+        flag = self.pod_manager.update_binding(
+            self.context, current_binding[0], self.b_pod_4['pod_id'])
+        self.assertEqual(flag, True)
+        binding_q = core.query_resource(
+            self.context, models.PodBinding,
+            [{'key': 'tenant_id',
+              'comparator': 'eq',
+              'value': 'new_project_pm_3'}], [])
+        self.assertEqual(len(binding_q), 2)
+        self.assertEqual(binding_q[0]['pod_id'], self.b_pod_3['pod_id'])
+        self.assertEqual(binding_q[0]['tenant_id'], 'new_project_pm_3')
+        self.assertEqual(binding_q[0]['is_binding'], False)
+        self.assertEqual(binding_q[1]['pod_id'], self.b_pod_4['pod_id'])
+        self.assertEqual(binding_q[1]['tenant_id'], 'new_project_pm_3')
+        self.assertEqual(binding_q[1]['is_binding'], True)

tricircle/tests/unit/common/test_az_ag.py

@@ -23,7 +23,6 @@ from tricircle.db import api
 from tricircle.db import core
 from tricircle.db import models
 
-
 FAKE_AZ = 'fake_az'
 
 FAKE_SITE_ID = 'fake_pod_id'
@@ -133,6 +132,7 @@ class AZAGTest(unittest.TestCase):
             self.assertEqual(pod2['pod_name'], FAKE_SITE_NAME)
             self.assertEqual(pod2['pod_id'], FAKE_SITE_ID)
             self.assertEqual(pod2['az_name'], FAKE_AZ)
+
         else:
             self.assertEqual(pod2['pod_name'], FAKE_SITE_NAME_2)
             self.assertEqual(pod2['pod_id'], FAKE_SITE_ID_2)

tricircle/tests/unit/nova_apigw/controllers/test_server.py

@@ -25,8 +25,10 @@ from oslo_utils import uuidutils
 
 from tricircle.common import constants
 from tricircle.common import context
-import tricircle.common.exceptions as t_exceptions
+
+from tricircle.common import exceptions as t_exceptions
 from tricircle.common import lock_handle
+from tricircle.common.scheduler import filter_scheduler
 from tricircle.common import xrpcapi
 from tricircle.db import api
 from tricircle.db import core
@@ -85,6 +87,7 @@ class FakeServerController(server.ServerController):
         self.project_id = project_id
         self.helper = FakeHelper()
         self.xjob_handler = xrpcapi.XJobAPI()
+        self.filter_scheduler = filter_scheduler.FilterScheduler()
 
     def _get_client(self, pod_name=None):
         if not pod_name: