Add BGP db model and service

This patch adds BGP db model and BGP service. Currently, BGP service
will sync local cache periodically.

DF bgp service status report will be in following patch, based on
service status report implementation.

Change-Id: I73455fb442c75b8a81ccf371b25b4823c91b85a8
Partially-implements: blueprint bgp-dynamic-routing

devstack/devstackgaterc

@@ -17,7 +17,7 @@
 # http://git.openstack.org/cgit/openstack-infra/project-config/tree/jenkins/jobs/dragonflow.yaml
 #
 
-export OVERRIDE_ENABLED_SERVICES=key,n-api,n-cpu,n-cond,n-sch,n-crt,n-cauth,n-obj,g-api,g-reg,c-sch,c-api,c-vol,horizon,rabbit,mysql,dstat,df-controller,df-redis,df-redis-server,q-svc,df-l3-agent,df-metadata,q-qos,placement-api
+export OVERRIDE_ENABLED_SERVICES=key,n-api,n-cpu,n-cond,n-sch,n-crt,n-cauth,n-obj,g-api,g-reg,c-sch,c-api,c-vol,horizon,rabbit,mysql,dstat,df-controller,df-redis,df-redis-server,q-svc,df-l3-agent,df-metadata,q-qos,placement-api,df-bgp
 export DEVSTACK_LOCAL_CONFIG+=$'\n'"DF_REDIS_PUBSUB=True"
 export DEVSTACK_LOCAL_CONFIG+=$'\n'"DF_RUNNING_IN_GATE=True"
 export DEVSTACK_LOCAL_CONFIG+=$'\n'"SNAT_HOST_IP=172.24.4.100"

devstack/plugin.sh

@@ -426,6 +426,20 @@ function stop_df_metadata_agent {
     fi
 }
 
+function start_df_bgp_service {
+    if is_service_enabled df-bgp ; then
+        echo "Starting Dragonflow BGP dynamic routing service"
+        run_process df-bgp "python $DF_BGP_SERVICE --config-file $NEUTRON_CONF --config-file $DRAGONFLOW_CONF"
+    fi
+}
+
+function stop_df_bgp_service {
+    if is_service_enabled df-bgp ; then
+       echo "Stopping Dragonflow BGP dynamic routing service"
+       stop_process df-bgp
+    fi
+}
+
 # main loop
 if [[ "$Q_ENABLE_DRAGONFLOW_LOCAL_CONTROLLER" == "True" ]]; then
     if [[ "$1" == "stack" && "$2" == "install" ]]; then
@@ -476,9 +490,11 @@ if [[ "$Q_ENABLE_DRAGONFLOW_LOCAL_CONTROLLER" == "True" ]]; then
 
         start_df
         start_df_metadata_agent
+        start_df_bgp_service
     fi
 
     if [[ "$1" == "unstack" ]]; then
+        stop_df_bgp_service
         stop_df_metadata_agent
         stop_df
         if function_exists nb_db_driver_clean; then

devstack/settings

@@ -17,6 +17,9 @@ DF_METADATA_SERVICE_PORT=${DF_METADATA_SERVICE_PORT:-"18080"}
 DF_METADATA_SERVICE_INTERFACE=${DF_METADATA_SERVICE_INTERFACE:-"tap-metadata"}
 METADATA_PROXY_SHARED_SECRET=${METADATA_PROXY_SHARED_SECRET:-"secret"}
 
+# df-bgp
+DF_BGP_SERVICE=${DF_BGP_SERVICE:-"$NEUTRON_BIN_DIR/df-bgp-service"}
+
 DF_L2_RESPONDER=${DF_L2_RESPONDER:-'True'}
 
 DF_MONITOR_TABLE_POLL_TIME=${DF_MONITOR_TABLE_POLL_TIME:-30}

dragonflow/cmd/eventlet/df_bgp_service.py

@@ -0,0 +1,17 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from dragonflow.controller import df_bgp_service
+
+
+def main():
+    df_bgp_service.main()

dragonflow/conf/__init__.py

@@ -13,6 +13,7 @@
 from oslo_config import cfg
 
 from dragonflow.conf import df_active_port_detection
+from dragonflow.conf import df_bgp
 from dragonflow.conf import df_cassandra
 from dragonflow.conf import df_common_params
 from dragonflow.conf import df_dhcp
@@ -39,3 +40,4 @@ df_dnat.register_opts()
 df_ryu.register_opts()
 df_provider_networks.register_opts()
 df_snat.register_opts()
+df_bgp.register_opts()

dragonflow/conf/df_bgp.py

@@ -0,0 +1,32 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from oslo_config import cfg
+
+from dragonflow._i18n import _
+
+
+df_bgp_app_opts = [
+    cfg.IntOpt(
+        'pulse_interval',
+        default=5,
+        help=_('The interval(in seconds) of BGP service to get data updates '
+               'and advertise BGP routes'))
+]
+
+
+def register_opts():
+    cfg.CONF.register_opts(df_bgp_app_opts, group='df_bgp')
+
+
+def list_opts():
+    return {'df_bgp': df_bgp_app_opts}

dragonflow/controller/df_bgp_service.py

@@ -0,0 +1,79 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import functools
+import sys
+
+from neutron.common import config as common_config
+from oslo_service import loopingcall
+from oslo_service import service
+
+from dragonflow import conf as cfg
+from dragonflow.controller import df_db_objects_refresh
+from dragonflow.db import api_nb
+from dragonflow.db import db_store2
+from dragonflow.db import model_framework
+from dragonflow.db.models import bgp  # noqa
+
+
+class BGPService(service.Service):
+    def __init__(self):
+        super(BGPService, self).__init__()
+        self.db_store = db_store2.get_instance()
+
+        # BGP dynamic route is not a service that needs real time response.
+        # So disable pubsub here and use period task to do BGP job.
+        cfg.CONF.set_override('enable_df_pub_sub', False, group='df')
+        self.nb_api = api_nb.NbApi.get_instance(False)
+
+        self.bgp_pulse = loopingcall.FixedIntervalLoopingCall(
+            self.sync_data_from_nb_db)
+
+    def start(self):
+        super(BGPService, self).start()
+        self.nb_api.initialize(db_ip=cfg.CONF.df.remote_db_ip,
+                               db_port=cfg.CONF.df.remote_db_port)
+        self.register_bgp_models()
+        self.bgp_pulse.start(cfg.CONF.df_bgp.pulse_interval)
+
+    def stop(self):
+        super(BGPService, self).stop()
+        self.bgp_pulse.stop()
+
+    def register_bgp_models(self):
+        for model in model_framework.iter_models_by_dependency_order():
+            df_db_objects_refresh.add_refresher(
+                df_db_objects_refresh.DfObjectRefresher(
+                    model.__name__,
+                    functools.partial(self.db_store.get_keys_by_topic,
+                                      model),
+                    functools.partial(self.nb_api.get_all, model),
+                    self.update_model_object,
+                    functools.partial(self.delete_model_object, model),
+                ),
+            )
+
+    def sync_data_from_nb_db(self):
+        df_db_objects_refresh.sync_local_cache_from_nb_db()
+
+    def update_model_object(self, obj):
+        self.db_store.update(obj)
+
+    def delete_model_object(self, model, obj_id):
+        self.db_store.delete(model(id=obj_id))
+
+
+def main():
+    common_config.init(sys.argv[1:])
+    common_config.setup_logging()
+    server = BGPService()
+    service.launch(cfg.CONF, server).wait()

dragonflow/db/models/all.py

@@ -9,6 +9,7 @@
 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 #    License for the specific language governing permissions and limitations
 #    under the License.
+from dragonflow.db.models import bgp  # noqa
 from dragonflow.db.models import core  # noqa
 from dragonflow.db.models import l2  # noqa
 from dragonflow.db.models import l3  # noqa

dragonflow/db/models/bgp.py

@@ -0,0 +1,46 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from jsonmodels import fields
+
+import dragonflow.db.field_types as df_fields
+import dragonflow.db.model_framework as mf
+from dragonflow.db.models import host_route
+from dragonflow.db.models import mixins
+
+
+# NOTE(xiaohui):
+# As both BGPSpeaker and BGPPeer from neutron don't have revision_num now,
+# skip adding version to db modles.
+@mf.register_model
+@mf.construct_nb_db_model
+class BGPPeer(mf.ModelBase, mixins.Topic, mixins.Name):
+    table_name = "bgp_peer"
+
+    peer_ip = df_fields.IpAddressField(required=True)
+    remote_as = fields.IntField(required=True)
+    auth_type = fields.StringField()
+    password = fields.StringField()
+
+
+@mf.register_model
+@mf.construct_nb_db_model(indexes={'peer_id': 'peers.id'})
+class BGPSpeaker(mf.ModelBase, mixins.Topic, mixins.Name):
+    table_name = "bgp_speaker"
+
+    local_as = fields.IntField(required=True)
+    peers = df_fields.ReferenceListField(BGPPeer)
+    routes = fields.ListField(host_route.HostRoute)
+    ip_version = fields.IntField(required=True)
+
+    def remove_peer(self, peer_id):
+        self.peers[:] = [peer for peer in self.peers if peer.id != peer_id]

dragonflow/tests/unit/test_df_bgp_service.py

@@ -0,0 +1,53 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import eventlet
+import mock
+
+from dragonflow.controller import df_bgp_service
+from dragonflow.db.models import bgp
+from dragonflow.tests import base as tests_base
+
+
+class TestDFBGPService(tests_base.BaseTestCase):
+
+    @mock.patch('dragonflow.db.api_nb.NbApi.get_instance')
+    def test_sync_bgp_data_to_db_store(self, get_instance):
+        bgp_service = df_bgp_service.BGPService()
+
+        def get_all_side_effect(model, topic):
+            if model == bgp.BGPPeer:
+                return [bgp.BGPPeer(id="peer1",
+                                    topic="topic1",
+                                    name="peer1",
+                                    peer_ip="172.24.4.88",
+                                    remote_as=4321)]
+
+            if model == bgp.BGPSpeaker:
+                return [bgp.BGPSpeaker(id="speaker1",
+                                       topic="topic1",
+                                       name="speaker1",
+                                       local_as=1234,
+                                       peers=["peer1"],
+                                       ip_version=4)]
+
+        bgp_service.nb_api.get_all.side_effect = get_all_side_effect
+        with mock.patch('dragonflow.db.model_framework.iter_models',
+                        return_value={bgp.BGPSpeaker, bgp.BGPPeer}):
+            bgp_service.start()
+            self.addCleanup(bgp_service.stop)
+            # Give fixed interval a chance to run.
+            eventlet.sleep(0)
+            self.assertTrue(
+                bgp_service.db_store.get_one(bgp.BGPPeer(id="peer1")))
+            self.assertTrue(
+                bgp_service.db_store.get_one(bgp.BGPSpeaker(id="speaker1")))

setup.cfg

@@ -57,6 +57,7 @@ console_scripts =
     df-publisher-service = dragonflow.cmd.eventlet.df_publisher_service:main
     df-l3-agent = dragonflow.cmd.eventlet.df_l3_agent:main
     df-metadata-service = dragonflow.cmd.eventlet.df_metadata_service:main
+    df-bgp-service = dragonflow.cmd.eventlet.df_bgp_service:main
 dragonflow.pubsub_driver =
     zmq_pubsub_driver = dragonflow.db.pubsub_drivers.zmq_pubsub_driver:ZMQPubSub
     zmq_pubsub_multiproc_driver = dragonflow.db.pubsub_drivers.zmq_pubsub_driver:ZMQPubSubMultiproc