add cyborg-conductor & db

1. add conductor rpc
2. add cyborg-conductor command
3. add db and init `accelerators` table
4. add accelerator_create method
5. add cyborg-dbsync command

Change-Id: I07333a4df7a42878dcf950b2b7893a37670da87b

cyborg/__init__.py

@@ -12,8 +12,11 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+import eventlet
 import pbr.version
 
 
 __version__ = pbr.version.VersionInfo(
     'cyborg').version_string()
+
+eventlet.monkey_patch(os=False)

cyborg/api/app.py

@@ -28,6 +28,8 @@ def get_pecan_config():
 
 def setup_app(pecan_config=None, extra_hooks=None):
     app_hooks = [hooks.ConfigHook(),
+                 hooks.ConductorAPIHook(),
+                 hooks.ContextHook(pecan_config.app.acl_public_routes),
                  hooks.PublicUrlHook()]
     if extra_hooks:
         app_hooks.extend(extra_hooks)

cyborg/api/config.py

@@ -28,7 +28,8 @@ app = {
     'static_root': '%(confdir)s/public',
     'debug': False,
     'acl_public_routes': [
-        '/'
+        '/',
+        '/v1'
     ]
 }
 

cyborg/api/controllers/v1/__init__.py

@@ -21,6 +21,7 @@ from wsme import types as wtypes
 
 from cyborg.api.controllers import base
 from cyborg.api.controllers import link
+from cyborg.api.controllers.v1 import accelerators
 from cyborg.api import expose
 
 
@@ -49,6 +50,8 @@ class V1(base.APIBase):
 class Controller(rest.RestController):
     """Version 1 API controller root"""
 
+    accelerators = accelerators.AcceleratorsController()
+
     @expose.expose(V1)
     def get(self):
         return V1.convert()

cyborg/api/controllers/v1/accelerators.py

@@ -0,0 +1,85 @@
+# Copyright 2017 Huawei Technologies Co.,LTD.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import pecan
+from pecan import rest
+from six.moves import http_client
+import wsme
+from wsme import types as wtypes
+
+from cyborg.api.controllers import base
+from cyborg.api.controllers import link
+from cyborg.api.controllers.v1 import types
+from cyborg.api import expose
+from cyborg import objects
+
+
+class Accelerator(base.APIBase):
+    """API representation of a accelerator.
+
+    This class enforces type checking and value constraints, and converts
+    between the internal object model and the API representation of
+    a accelerator.
+    """
+
+    uuid = types.uuid
+    name = wtypes.text
+    description = wtypes.text
+    device_type = wtypes.text
+    acc_type = wtypes.text
+    acc_capability = wtypes.text
+    vendor_id = wtypes.text
+    product_id = wtypes.text
+    remotable = wtypes.IntegerType()
+
+    links = wsme.wsattr([link.Link], readonly=True)
+    """A list containing a self link"""
+
+    def __init__(self, **kwargs):
+        self.fields = []
+        for field in objects.Accelerator.fields:
+            self.fields.append(field)
+            setattr(self, field, kwargs.get(field, wtypes.Unset))
+
+    @classmethod
+    def convert_with_links(cls, db_accelerator):
+        accelerator = Accelerator(**db_accelerator.as_dict())
+        url = pecan.request.public_url
+        accelerator.links = [
+            link.Link.make_link('self', url, 'accelerators',
+                                accelerator.uuid),
+            link.Link.make_link('bookmark', url, 'accelerators',
+                                accelerator.uuid, bookmark=True)
+            ]
+
+        return accelerator
+
+
+class AcceleratorsController(rest.RestController):
+    """REST controller for Accelerators."""
+
+    @expose.expose(Accelerator, body=types.jsontype,
+                   status_code=http_client.CREATED)
+    def post(self, values):
+        """Create a new accelerator.
+
+        :param accelerator: an accelerator within the request body.
+        """
+        accelerator = pecan.request.conductor_api.accelerator_create(
+            pecan.request.context, values)
+        # Set the HTTP Location Header
+        pecan.response.location = link.build_url('accelerators',
+                                                 accelerator.uuid)
+        return Accelerator.convert_with_links(accelerator)

cyborg/api/controllers/v1/types.py

@@ -0,0 +1,64 @@
+# Copyright 2017 Huawei Technologies Co.,LTD.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import json
+
+from oslo_utils import uuidutils
+from wsme import types as wtypes
+
+from cyborg.common import exception
+
+
+class UUIDType(wtypes.UserType):
+    """A simple UUID type."""
+
+    basetype = wtypes.text
+    name = 'uuid'
+
+    @staticmethod
+    def validate(value):
+        if not uuidutils.is_uuid_like(value):
+            raise exception.InvalidUUID(uuid=value)
+        return value
+
+    @staticmethod
+    def frombasetype(value):
+        if value is None:
+            return None
+        return UUIDType.validate(value)
+
+
+class JsonType(wtypes.UserType):
+    """A simple JSON type."""
+
+    basetype = wtypes.text
+    name = 'json'
+
+    @staticmethod
+    def validate(value):
+        try:
+            json.dumps(value)
+        except TypeError:
+            raise exception.InvalidJsonType(value=value)
+        else:
+            return value
+
+    @staticmethod
+    def frombasetype(value):
+        return JsonType.validate(value)
+
+
+uuid = UUIDType()
+jsontype = JsonType()

cyborg/api/hooks.py

@@ -14,8 +14,11 @@
 #    under the License.
 
 from oslo_config import cfg
+from oslo_context import context
 from pecan import hooks
 
+from cyborg.conductor import rpcapi
+
 
 class ConfigHook(hooks.PecanHook):
     """Attach the config object to the request so controllers can get to it."""
@@ -34,3 +37,24 @@ class PublicUrlHook(hooks.PecanHook):
     def before(self, state):
         state.request.public_url = (
             cfg.CONF.api.public_endpoint or state.request.host_url)
+
+
+class ConductorAPIHook(hooks.PecanHook):
+    """Attach the conductor_api object to the request."""
+
+    def __init__(self):
+        self.conductor_api = rpcapi.ConductorAPI()
+
+    def before(self, state):
+        state.request.conductor_api = self.conductor_api
+
+
+class ContextHook(hooks.PecanHook):
+    """Configures a request context and attaches it to the request."""
+
+    def __init__(self, public_api_routes):
+        self.public_api_routes = public_api_routes
+        super(ContextHook, self).__init__()
+
+    def before(self, state):
+        state.request.context = context.get_admin_context()

cyborg/cmd/conductor.py

@@ -0,0 +1,39 @@
+# Copyright 2017 Huawei Technologies Co.,LTD.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""The Cyborg Conductor Service."""
+
+import sys
+
+from oslo_config import cfg
+from oslo_service import service
+
+from cyborg.common import constants
+from cyborg.common import service as cyborg_service
+
+
+CONF = cfg.CONF
+
+
+def main():
+    # Parse config file and command line options, then start logging
+    cyborg_service.prepare_service(sys.argv)
+
+    mgr = cyborg_service.RPCService('cyborg.conductor.manager',
+                                    'ConductorManager',
+                                    constants.CONDUCTOR_TOPIC)
+
+    launcher = service.launch(CONF, mgr)
+    launcher.wait()

cyborg/cmd/dbsync.py

@@ -0,0 +1,91 @@
+# Copyright 2017 Huawei Technologies Co.,LTD.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+Run storage database migration.
+"""
+
+import sys
+
+from oslo_config import cfg
+
+from cyborg.common.i18n import _
+from cyborg.common import service
+from cyborg.conf import CONF
+from cyborg.db import migration
+
+
+class DBCommand(object):
+
+    def upgrade(self):
+        migration.upgrade(CONF.command.revision)
+
+    def revision(self):
+        migration.revision(CONF.command.message, CONF.command.autogenerate)
+
+    def stamp(self):
+        migration.stamp(CONF.command.revision)
+
+    def version(self):
+        print(migration.version())
+
+    def create_schema(self):
+        migration.create_schema()
+
+
+def add_command_parsers(subparsers):
+    command_object = DBCommand()
+
+    parser = subparsers.add_parser(
+        'upgrade',
+        help=_("Upgrade the database schema to the latest version. "
+               "Optionally, use --revision to specify an alembic revision "
+               "string to upgrade to."))
+    parser.set_defaults(func=command_object.upgrade)
+    parser.add_argument('--revision', nargs='?')
+
+    parser = subparsers.add_parser(
+        'revision',
+        help=_("Create a new alembic revision. "
+               "Use --message to set the message string."))
+    parser.set_defaults(func=command_object.revision)
+    parser.add_argument('-m', '--message')
+    parser.add_argument('--autogenerate', action='store_true')
+
+    parser = subparsers.add_parser('stamp')
+    parser.set_defaults(func=command_object.stamp)
+    parser.add_argument('--revision', nargs='?')
+
+    parser = subparsers.add_parser(
+        'version',
+        help=_("Print the current version information and exit."))
+    parser.set_defaults(func=command_object.version)
+
+    parser = subparsers.add_parser(
+        'create_schema',
+        help=_("Create the database schema."))
+    parser.set_defaults(func=command_object.create_schema)
+
+
+def main():
+    command_opt = cfg.SubCommandOpt('command',
+                                    title='Command',
+                                    help=_('Available commands'),
+                                    handler=add_command_parsers)
+
+    CONF.register_cli_opt(command_opt)
+
+    service.prepare_service(sys.argv)
+    CONF.command.func()

cyborg/common/constants.py

@@ -0,0 +1,17 @@
+# Copyright 2017 Huawei Technologies Co.,LTD.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+CONDUCTOR_TOPIC = 'cyborg-conductor'

cyborg/common/exception.py

@@ -88,3 +88,24 @@ class CyborgException(Exception):
 
 class ConfigInvalid(CyborgException):
     _msg_fmt = _("Invalid configuration file. %(error_msg)s")
+
+
+class AcceleratorAlreadyExists(CyborgException):
+    _msg_fmt = _("Accelerator with uuid %(uuid)s already exists.")
+
+
+class Invalid(CyborgException):
+    _msg_fmt = _("Invalid parameters.")
+    code = http_client.BAD_REQUEST
+
+
+class InvalidIdentity(Invalid):
+    _msg_fmt = _("Expected a uuid/id but received %(identity)s.")
+
+
+class InvalidUUID(Invalid):
+    _msg_fmt = _("Expected a uuid but received %(uuid)s.")
+
+
+class InvalidJsonType(Invalid):
+    _msg_fmt = _("%(value)s is not JSON serializable.")

cyborg/common/paths.py

@@ -0,0 +1,48 @@
+# Copyright 2017 Huawei Technologies Co.,LTD.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import os
+
+from cyborg.conf import CONF
+
+
+def basedir_def(*args):
+    """Return an uninterpolated path relative to $pybasedir."""
+    return os.path.join('$pybasedir', *args)
+
+
+def bindir_def(*args):
+    """Return an uninterpolated path relative to $bindir."""
+    return os.path.join('$bindir', *args)
+
+
+def state_path_def(*args):
+    """Return an uninterpolated path relative to $state_path."""
+    return os.path.join('$state_path', *args)
+
+
+def basedir_rel(*args):
+    """Return a path relative to $pybasedir."""
+    return os.path.join(CONF.pybasedir, *args)
+
+
+def bindir_rel(*args):
+    """Return a path relative to $bindir."""
+    return os.path.join(CONF.bindir, *args)
+
+
+def state_path_rel(*args):
+    """Return a path relative to $state_path."""
+    return os.path.join(CONF.state_path, *args)

cyborg/common/service.py

@@ -14,9 +14,12 @@
 #    under the License.
 
 from oslo_concurrency import processutils
+from oslo_context import context
 from oslo_log import log
+import oslo_messaging as messaging
 from oslo_service import service
 from oslo_service import wsgi
+from oslo_utils import importutils
 
 from cyborg.api import app
 from cyborg.common import config
@@ -24,11 +27,56 @@ from cyborg.common import exception
 from cyborg.common.i18n import _
 from cyborg.common import rpc
 from cyborg.conf import CONF
+from cyborg import objects
+from cyborg.objects import base as objects_base
 
 
 LOG = log.getLogger(__name__)
 
 
+class RPCService(service.Service):
+    def __init__(self, manager_module, manager_class, topic, host=None):
+        super(RPCService, self).__init__()
+        self.topic = topic
+        self.host = host or CONF.host
+        manager_module = importutils.try_import(manager_module)
+        manager_class = getattr(manager_module, manager_class)
+        self.manager = manager_class(self.topic, self.host)
+        self.rpcserver = None
+
+    def start(self):
+        super(RPCService, self).start()
+
+        target = messaging.Target(topic=self.topic, server=self.host)
+        endpoints = [self.manager]
+        serializer = objects_base.CyborgObjectSerializer()
+        self.rpcserver = rpc.get_server(target, endpoints, serializer)
+        self.rpcserver.start()
+
+        admin_context = context.get_admin_context()
+        self.tg.add_dynamic_timer(
+            self.manager.periodic_tasks,
+            periodic_interval_max=CONF.periodic_interval,
+            context=admin_context)
+
+        LOG.info('Created RPC server for service %(service)s on host '
+                 '%(host)s.',
+                 {'service': self.topic, 'host': self.host})
+
+    def stop(self, graceful=True):
+        try:
+            self.rpcserver.stop()
+            self.rpcserver.wait()
+        except Exception as e:
+            LOG.exception('Service error occurred when stopping the '
+                          'RPC server. Error: %s', e)
+
+        super(RPCService, self).stop(graceful=graceful)
+        LOG.info('Stopped RPC server for service %(service)s on host '
+                 '%(host)s.',
+                 {'service': self.topic, 'host': self.host})
+
+
 def prepare_service(argv=None):
     log.register_options(CONF)
     log.set_defaults()
@@ -39,6 +87,7 @@ def prepare_service(argv=None):
 
     log.setup(CONF, 'cyborg')
     rpc.init(CONF)
+    objects.register_all()
 
 
 def process_launcher():

cyborg/conductor/conductor.conf

@@ -1,4 +0,0 @@
-[cyborg]
-transport_url=
-server_id=
-database_url=

cyborg/conductor/conductor.py

@@ -1,80 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-import conf
-import eventlet
-import handlers
-from oslo_config import cfg
-from oslo_log import log as logging
-import oslo_messaging as messaging
-import rpcapi
-from sqlalchemy import create_engine
-import time
-
-eventlet.monkey_patch()
-
-CONF = cfg.CONF
-conf.register_opts(CONF)
-
-LOG = logging.getLogger(__name__)
-logging.register_options(CONF)
-logging.setup(CONF, 'Cyborg.Conductor')
-
-CONF(['--config-file', 'conductor.conf'])
-url = messaging.TransportURL.parse(CONF, url=CONF.cyborg.transport_url)
-transport = messaging.get_notification_transport(CONF, url)
-
-message_endpoints = [
-    handlers.NotificationEndpoint
-]
-message_targets = [
-    messaging.Target(topic='info'),
-    messaging.Target(topic='update'),
-    messaging.Target(topic='warn'),
-    messaging.Target(topic='error')
-]
-rpc_targets = messaging.Target(topic='cyborg_control',
-                               server=CONF.cyborg.server_id)
-rpc_endpoints = [
-    rpcapi.RPCEndpoint()
-]
-access_policy = messaging.ExplicitRPCAccessPolicy
-rpc_server = messaging.get_rpc_server(transport,
-                                      rpc_targets,
-                                      rpc_endpoints,
-                                      executor='eventlet',
-                                      access_policy=access_policy)
-pool = "listener-workers"
-message_server = messaging.get_notification_listener(transport,
-                                                     message_targets,
-                                                     message_endpoints,
-                                                     executor='eventlet',
-                                                     allow_requeue=True)
-
-engine = create_engine(CONF.cyborg.connection, echo=True)
-engine.connect()
-
-try:
-    message_server.start()
-    rpc_server.start()
-    print("Cyborg Conductor running")
-    while True:
-        time.sleep(1)
-except KeyboardInterrupt:
-    print("Stopping server")
-
-message_server.stop()
-rpc_server.stop()
-message_server.wait()
-rpc_server.wait()

cyborg/conductor/manager.py

@@ -0,0 +1,40 @@
+# Copyright 2017 Huawei Technologies Co.,LTD.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import oslo_messaging as messaging
+
+from cyborg.conf import CONF
+from cyborg import objects
+
+
+class ConductorManager(object):
+    """Cyborg Conductor manager main class."""
+
+    RPC_API_VERSION = '1.0'
+    target = messaging.Target(version=RPC_API_VERSION)
+
+    def __init__(self, topic, host=None):
+        super(ConductorManager, self).__init__()
+        self.topic = topic
+        self.host = host or CONF.host
+
+    def periodic_tasks(self, context, raise_on_error=False):
+        pass
+
+    def accelerator_create(self, context, values):
+        """Create a new accelerator."""
+        accelerator = objects.Accelerator(context, **values)
+        accelerator.create()
+        return accelerator

cyborg/conductor/rpcapi.py

@@ -1,42 +1,53 @@
-# -*- coding: utf-8 -*-
-
+# Copyright 2017 Huawei Technologies Co.,LTD.
+# All Rights Reserved.
 #
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
 #
-#      http://www.apache.org/licenses/LICENSE-2.0
+#         http://www.apache.org/licenses/LICENSE-2.0
 #
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Client side of the conductor RPC API."""
+
+from oslo_config import cfg
+import oslo_messaging as messaging
+
+from cyborg.common import constants
+from cyborg.common import rpc
+from cyborg.objects import base as objects_base
 
 
-class RPCEndpoint(object):
+CONF = cfg.CONF
 
-    # Conductor functions exposed for external calls
-    # Mostly called by the API?
-    def __init__(self):
-        pass
 
-    # Returns a list of all accelerators managed by Cyborg
-    def list_accelerators(self, ctxt):
-        pass
+class ConductorAPI(object):
+    """Client side of the conductor RPC API.
 
-    # Returns an accelerator from the DB
-    def get_accelerator(self, ctxt, accelerator):
-        pass
+    API version history:
 
-    # Deletes an accelerator from the DB and from the agent that hosts it
-    def delete_accelerator(self, ctxt, accelerator):
-        pass
+    |    1.0 - Initial version.
 
-    # Updates an accelerator both in the DB and on the agent that hosts it
-    def update_accelerator(self, ctxt, accelerator):
-        pass
+    """
 
-    # Runs discovery on either a single agent or all agents
-    def discover_accelerators(self, ctxt, agent_id=None):
-        pass
+    RPC_API_VERSION = '1.0'
+
+    def __init__(self, topic=None):
+        super(ConductorAPI, self).__init__()
+        self.topic = topic or constants.CONDUCTOR_TOPIC
+        target = messaging.Target(topic=self.topic,
+                                  version='1.0')
+        serializer = objects_base.CyborgObjectSerializer()
+        self.client = rpc.get_client(target,
+                                     version_cap=self.RPC_API_VERSION,
+                                     serializer=serializer)
+
+    def accelerator_create(self, context, values):
+        """Signal to conductor service to create an accelerator."""
+        cctxt = self.client.prepare(topic=self.topic, server=CONF.host)
+        return cctxt.call(context, 'accelerator_create', values=values)

cyborg/conf/__init__.py

@@ -16,10 +16,12 @@
 from oslo_config import cfg
 
 from cyborg.conf import api
+from cyborg.conf import database
 from cyborg.conf import default
 
 
 CONF = cfg.CONF
 
 api.register_opts(CONF)
+database.register_opts(CONF)
 default.register_opts(CONF)

cyborg/conf/database.py

@@ -1,3 +1,5 @@
+# Copyright 2017 Huawei Technologies Co.,LTD.
+# All Rights Reserved.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
 #    not use this file except in compliance with the License. You may obtain
@@ -12,22 +14,19 @@
 #    under the License.
 
 from oslo_config import cfg
-import uuid
 
-default_opts = [
-    cfg.StrOpt('transport_url',
-               default='',
-               help='Transport url for messating, copy from transport_url= in \
-                     your Nova config default section'),
-    cfg.StrOpt('database_url',
-               default='',
-               help='Database url for storage, copy from connection= in your \
-                     Nova db config section'),
-    cfg.StrOpt('server_id',
-               default=uuid.uuid4(),
-               help='Unique ID for this conductor instance'),
+from cyborg.common.i18n import _
+
+
+opts = [
+    cfg.StrOpt('mysql_engine',
+               default='InnoDB',
+               help=_('MySQL engine to use.'))
 ]
 
+opt_group = cfg.OptGroup(name='database',
+                         title='Options for the database service')
+
 
 def register_opts(conf):
-    conf.register_opts(default_opts, group='cyborg')
+    conf.register_opts(opts, group=opt_group)

cyborg/conf/default.py

@@ -13,6 +13,9 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import os
+import socket
+
 from oslo_config import cfg
 
 from cyborg.common.i18n import _
@@ -27,6 +30,39 @@ exc_log_opts = [
                        'message.')),
 ]
 
+service_opts = [
+    cfg.HostAddressOpt('host',
+                       default=socket.getfqdn(),
+                       sample_default='localhost',
+                       help=_('Name of this node. This can be an opaque '
+                              'identifier. It is not necessarily a hostname, '
+                              'FQDN, or IP address. However, the node name '
+                              'must be valid within an AMQP key, and if using '
+                              'ZeroMQ, a valid hostname, FQDN, or IP address.')
+                       ),
+    cfg.IntOpt('periodic_interval',
+               default=60,
+               help=_('Default interval (in seconds) for running periodic '
+                      'tasks.')),
+]
+
+path_opts = [
+    cfg.StrOpt('pybasedir',
+               default=os.path.abspath(
+                   os.path.join(os.path.dirname(__file__), '../')),
+               sample_default='/usr/lib/python/site-packages/cyborg/cyborg',
+               help=_('Directory where the cyborg python module is '
+                      'installed.')),
+    cfg.StrOpt('bindir',
+               default='$pybasedir/bin',
+               help=_('Directory where cyborg binaries are installed.')),
+    cfg.StrOpt('state_path',
+               default='$pybasedir',
+               help=_("Top-level directory for maintaining cyborg's state.")),
+]
+
 
 def register_opts(conf):
     conf.register_opts(exc_log_opts)
+    conf.register_opts(service_opts)
+    conf.register_opts(path_opts)

cyborg/db/api.py

@@ -0,0 +1,47 @@
+# Copyright 2017 Huawei Technologies Co.,LTD.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Base classes for storage engines."""
+
+import abc
+
+from oslo_config import cfg
+from oslo_db import api as db_api
+import six
+
+
+_BACKEND_MAPPING = {'sqlalchemy': 'cyborg.db.sqlalchemy.api'}
+IMPL = db_api.DBAPI.from_config(cfg.CONF,
+                                backend_mapping=_BACKEND_MAPPING,
+                                lazy=True)
+
+
+def get_instance():
+    """Return a DB API instance."""
+    return IMPL
+
+
+@six.add_metaclass(abc.ABCMeta)
+class Connection(object):
+    """Base class for storage system connections."""
+
+    @abc.abstractmethod
+    def __init__(self):
+        """Constructor."""
+
+    # accelerator
+    @abc.abstractmethod
+    def accelerator_create(self, context, values):
+        """Create a new server type."""

cyborg/db/migration.py

@@ -0,0 +1,52 @@
+# Copyright 2017 Huawei Technologies Co.,LTD.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Database setup and migration commands."""
+
+from oslo_config import cfg
+from stevedore import driver
+
+
+_IMPL = None
+
+
+def get_backend():
+    global _IMPL
+    if not _IMPL:
+        cfg.CONF.import_opt('backend', 'oslo_db.options', group='database')
+        _IMPL = driver.DriverManager("cyborg.database.migration_backend",
+                                     cfg.CONF.database.backend).driver
+    return _IMPL
+
+
+def upgrade(version=None):
+    """Migrate the database to `version` or the most recent version."""
+    return get_backend().upgrade(version)
+
+
+def version():
+    return get_backend().version()
+
+
+def stamp(version):
+    return get_backend().stamp(version)
+
+
+def revision(message, autogenerate):
+    return get_backend().revision(message, autogenerate)
+
+
+def create_schema():
+    return get_backend().create_schema()

cyborg/db/sqlalchemy/alembic.ini

@@ -0,0 +1,54 @@
+# A generic, single database configuration.
+
+[alembic]
+# path to migration scripts
+script_location = %(here)s/alembic
+
+# template used to generate migration files
+# file_template = %%(rev)s_%%(slug)s
+
+# max length of characters to apply to the
+# "slug" field
+#truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+#sqlalchemy.url = driver://user:pass@localhost/dbname
+
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

cyborg/db/sqlalchemy/alembic/README

@@ -0,0 +1,12 @@
+Please see https://alembic.readthedocs.org/en/latest/index.html for general documentation
+
+To create alembic migrations use:
+$ cyborg-dbsync revision --message --autogenerate
+
+Stamp db with most recent migration version, without actually running migrations
+$ cyborg-dbsync stamp --revision head
+
+Upgrade can be performed by:
+$ cyborg-dbsync - for backward compatibility
+$ cyborg-dbsync upgrade
+# cyborg-dbsync upgrade --revision head

cyborg/db/sqlalchemy/alembic/env.py

@@ -0,0 +1,61 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from logging import config as log_config
+
+from alembic import context
+from oslo_db.sqlalchemy import enginefacade
+
+try:
+    # NOTE(whaom): This is to register the DB2 alembic code which
+    # is an optional runtime dependency.
+    from ibm_db_alembic.ibm_db import IbmDbImpl  # noqa
+except ImportError:
+    pass
+
+from cyborg.db.sqlalchemy import models
+
+# this is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
+config = context.config
+
+# Interpret the config file for Python logging.
+# This line sets up loggers basically.
+log_config.fileConfig(config.config_file_name)
+
+# add your model's MetaData object here
+# for 'autogenerate' support
+# from myapp import mymodel
+target_metadata = models.Base.metadata
+
+# other values from the config, defined by the needs of env.py,
+# can be acquired:
+# my_important_option = config.get_main_option("my_important_option")
+# ... etc.
+
+
+def run_migrations_online():
+    """Run migrations in 'online' mode.
+
+    In this scenario we need to create an Engine
+    and associate a connection with the context.
+
+    """
+    engine = enginefacade.get_legacy_facade().get_engine()
+    with engine.connect() as connection:
+        context.configure(connection=connection,
+                          target_metadata=target_metadata)
+        with context.begin_transaction():
+            context.run_migrations()
+
+
+run_migrations_online()

cyborg/db/sqlalchemy/alembic/script.py.mako

@@ -0,0 +1,18 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision}
+Create Date: ${create_date}
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = ${repr(up_revision)}
+down_revision = ${repr(down_revision)}
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+def upgrade():
+    ${upgrades if upgrades else "pass"}

cyborg/db/sqlalchemy/alembic/versions/f50980397351_initial_migration.py

@@ -0,0 +1,49 @@
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""initial migration.
+
+Revision ID: f50980397351
+Revises: None
+Create Date: 2017-08-15 08:44:36.010417
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = 'f50980397351'
+down_revision = None
+
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+    op.create_table(
+        'accelerators',
+        sa.Column('created_at', sa.DateTime(), nullable=True),
+        sa.Column('updated_at', sa.DateTime(), nullable=True),
+        sa.Column('id', sa.Integer(), nullable=False),
+        sa.Column('uuid', sa.String(length=36), nullable=False),
+        sa.Column('name', sa.String(length=255), nullable=False),
+        sa.Column('description', sa.Text(), nullable=True),
+        sa.Column('device_type', sa.Text(), nullable=False),
+        sa.Column('acc_type', sa.Text(), nullable=False),
+        sa.Column('acc_capability', sa.Text(), nullable=False),
+        sa.Column('vendor_id', sa.Text(), nullable=False),
+        sa.Column('product_id', sa.Text(), nullable=False),
+        sa.Column('remotable', sa.Integer(), nullable=False),
+        sa.PrimaryKeyConstraint('id'),
+        mysql_ENGINE='InnoDB',
+        mysql_DEFAULT_CHARSET='UTF8'
+    )

cyborg/db/sqlalchemy/api.py

@@ -0,0 +1,114 @@
+# Copyright 2017 Huawei Technologies Co.,LTD.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""SQLAlchemy storage backend."""
+
+import threading
+
+from oslo_db import exception as db_exc
+from oslo_db.sqlalchemy import enginefacade
+from oslo_db.sqlalchemy import utils as sqlalchemyutils
+from oslo_log import log
+from oslo_utils import strutils
+from oslo_utils import uuidutils
+
+from cyborg.common import exception
+from cyborg.db import api
+from cyborg.db.sqlalchemy import models
+
+
+_CONTEXT = threading.local()
+LOG = log.getLogger(__name__)
+
+
+def get_backend():
+    """The backend is this module itself."""
+    return Connection()
+
+
+def _session_for_read():
+    return enginefacade.reader.using(_CONTEXT)
+
+
+def _session_for_write():
+    return enginefacade.writer.using(_CONTEXT)
+
+
+def model_query(context, model, *args, **kwargs):
+    """Query helper for simpler session usage.
+
+    :param context: Context of the query
+    :param model: Model to query. Must be a subclass of ModelBase.
+    :param args: Arguments to query. If None - model is used.
+
+    Keyword arguments:
+
+    :keyword project_only:
+      If set to True, then will do query filter with context's project_id.
+      if set to False or absent, then will not do query filter with context's
+      project_id.
+    :type project_only: bool
+    """
+
+    if kwargs.pop("project_only", False):
+        kwargs["project_id"] = context.tenant
+
+    with _session_for_read() as session:
+        query = sqlalchemyutils.model_query(
+            model, session, args, **kwargs)
+        return query
+
+
+def add_identity_filter(query, value):
+    """Adds an identity filter to a query.
+
+    Filters results by ID, if supplied value is a valid integer.
+    Otherwise attempts to filter results by UUID.
+
+    :param query: Initial query to add filter to.
+    :param value: Value for filtering results by.
+    :return: Modified query.
+    """
+    if strutils.is_int_like(value):
+        return query.filter_by(id=value)
+    elif uuidutils.is_uuid_like(value):
+        return query.filter_by(uuid=value)
+    else:
+        raise exception.InvalidIdentity(identity=value)
+
+
+class Connection(api.Connection):
+    """SqlAlchemy connection."""
+
+    def __init__(self):
+        pass
+
+    def accelerator_create(self, context, values):
+        if not values.get('uuid'):
+            values['uuid'] = uuidutils.generate_uuid()
+
+        if not values.get('description'):
+            values['description'] = ''
+
+        accelerator = models.Accelerator()
+        accelerator.update(values)
+
+        with _session_for_write() as session:
+            try:
+                session.add(accelerator)
+                session.flush()
+            except db_exc.DBDuplicateEntry:
+                raise exception.AcceleratorAlreadyExists(uuid=values['uuid'])
+            return accelerator

cyborg/db/sqlalchemy/migration.py

@@ -0,0 +1,108 @@
+# Copyright 2017 Huawei Technologies Co.,LTD.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import os
+
+import alembic
+from alembic import config as alembic_config
+import alembic.migration as alembic_migration
+from oslo_db import exception as db_exc
+from oslo_db.sqlalchemy import enginefacade
+
+from cyborg.db.sqlalchemy import models
+
+
+def _alembic_config():
+    path = os.path.join(os.path.dirname(__file__), 'alembic.ini')
+    config = alembic_config.Config(path)
+    return config
+
+
+def version(config=None, engine=None):
+    """Current database version.
+
+    :returns: Database version
+    :rtype: string
+    """
+    if engine is None:
+        engine = enginefacade.get_legacy_facade().get_engine()
+    with engine.connect() as conn:
+        context = alembic_migration.MigrationContext.configure(conn)
+        return context.get_current_revision()
+
+
+def upgrade(revision, config=None):
+    """Used for upgrading database.
+
+    :param version: Desired database version
+    :type version: string
+    """
+    revision = revision or 'head'
+    config = config or _alembic_config()
+    alembic.command.upgrade(config, revision)
+
+
+def create_schema(config=None, engine=None):
+    """Create database schema from models description.
+
+    Can be used for initial installation instead of upgrade('head').
+    """
+    if engine is None:
+        engine = enginefacade.get_legacy_facade().get_engine()
+
+    if version(engine=engine) is not None:
+        raise db_exc.DbMigrationError("DB schema is already under version"
+                                      " control. Use upgrade() instead")
+
+    models.Base.metadata.create_all(engine)
+    stamp('head', config=config)
+
+
+def downgrade(revision, config=None):
+    """Used for downgrading database.
+
+    :param version: Desired database version
+    :type version: string
+    """
+    revision = revision or 'base'
+    config = config or _alembic_config()
+    return alembic.command.downgrade(config, revision)
+
+
+def stamp(revision, config=None):
+    """Stamps database with provided revision.
+
+    Don't run any migrations.
+
+    :param revision: Should match one from repository or head - to stamp
+                     database with most recent revision
+    :type revision: string
+    """
+    config = config or _alembic_config()
+    return alembic.command.stamp(config, revision=revision)
+
+
+def revision(message=None, autogenerate=False, config=None):
+    """Creates template for migration.
+
+    :param message: Text that will be used for migration title
+    :type message: string
+    :param autogenerate: If True - generates diff based on current database
+                         state
+    :type autogenerate: bool
+    """
+    config = config or _alembic_config()
+    return alembic.command.revision(config, message=message,
+                                    autogenerate=autogenerate)

cyborg/db/sqlalchemy/models.py

@@ -0,0 +1,72 @@
+# Copyright 2017 Huawei Technologies Co.,LTD.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""SQLAlchemy models for accelerator service."""
+
+from oslo_db import options as db_options
+from oslo_db.sqlalchemy import models
+import six.moves.urllib.parse as urlparse
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy import Column, String, Integer
+from sqlalchemy import schema
+
+from cyborg.common import paths
+from cyborg.conf import CONF
+
+
+_DEFAULT_SQL_CONNECTION = 'sqlite:///' + paths.state_path_def('cyborg.sqlite')
+db_options.set_defaults(CONF, connection=_DEFAULT_SQL_CONNECTION)
+
+
+def table_args():
+    engine_name = urlparse.urlparse(CONF.database.connection).scheme
+    if engine_name == 'mysql':
+        return {'mysql_engine': CONF.database.mysql_engine,
+                'mysql_charset': "utf8"}
+    return None
+
+
+class CyborgBase(models.TimestampMixin, models.ModelBase):
+    metadata = None
+
+    def as_dict(self):
+        d = {}
+        for c in self.__table__.columns:
+            d[c.name] = self[c.name]
+        return d
+
+
+Base = declarative_base(cls=CyborgBase)
+
+
+class Accelerator(Base):
+    """Represents the accelerators."""
+
+    __tablename__ = 'accelerators'
+    __table_args__ = (
+        schema.UniqueConstraint('uuid', name='uniq_accelerators0uuid'),
+        table_args()
+    )
+
+    id = Column(Integer, primary_key=True)
+    uuid = Column(String(36), nullable=False)
+    name = Column(String(255), nullable=False)
+    description = Column(String(255), nullable=True)
+    device_type = Column(String(255), nullable=False)
+    acc_type = Column(String(255), nullable=False)
+    acc_capability = Column(String(255), nullable=False)
+    vendor_id = Column(String(255), nullable=False)
+    product_id = Column(String(255), nullable=False)
+    remotable = Column(Integer, nullable=False)

cyborg/objects/__init__.py

@@ -0,0 +1,28 @@
+# Copyright 2017 Huawei Technologies Co.,LTD.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+# NOTE(comstud): You may scratch your head as you see code that imports
+# this module and then accesses attributes for objects such as Node,
+# etc, yet you do not see these attributes in here. Never fear, there is
+# a little bit of magic. When objects are registered, an attribute is set
+# on this module automatically, pointing to the newest/latest version of
+# the object.
+
+
+def register_all():
+    # NOTE(danms): You must make sure your object gets imported in this
+    # function in order for it to be registered by services that may
+    # need to receive it via RPC.
+    __import__('cyborg.objects.accelerator')

cyborg/objects/accelerator.py

@@ -0,0 +1,49 @@
+# Copyright 2017 Huawei Technologies Co.,LTD.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from oslo_versionedobjects import base as object_base
+
+from cyborg.db import api as dbapi
+from cyborg.objects import base
+from cyborg.objects import fields as object_fields
+
+
+@base.CyborgObjectRegistry.register
+class Accelerator(base.CyborgObject, object_base.VersionedObjectDictCompat):
+    # Version 1.0: Initial version
+    VERSION = '1.0'
+
+    dbapi = dbapi.get_instance()
+
+    fields = {
+        'uuid': object_fields.UUIDField(nullable=False),
+        'name': object_fields.StringField(nullable=False),
+        'description': object_fields.StringField(nullable=True),
+        'device_type': object_fields.StringField(nullable=False),
+        'acc_type': object_fields.StringField(nullable=False),
+        'acc_capability': object_fields.StringField(nullable=False),
+        'vendor_id': object_fields.StringField(nullable=False),
+        'product_id': object_fields.StringField(nullable=False),
+        'remotable': object_fields.IntegerField(nullable=False),
+    }
+
+    def __init__(self, *args, **kwargs):
+        super(Accelerator, self).__init__(*args, **kwargs)
+
+    def create(self, context=None):
+        """Create an Accelerator record in the DB."""
+        values = self.obj_get_changes()
+        db_accelerator = self.dbapi.accelerator_create(context, values)
+        self._from_db_object(context, self, db_accelerator)

cyborg/objects/base.py

@@ -0,0 +1,83 @@
+# Copyright 2017 Huawei Technologies Co.,LTD.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Cyborg common internal object model"""
+
+from oslo_utils import versionutils
+from oslo_versionedobjects import base as object_base
+
+from cyborg import objects
+from cyborg.objects import fields as object_fields
+
+
+class CyborgObjectRegistry(object_base.VersionedObjectRegistry):
+    def registration_hook(self, cls, index):
+        # NOTE(jroll): blatantly stolen from nova
+        # NOTE(danms): This is called when an object is registered,
+        # and is responsible for maintaining cyborg.objects.$OBJECT
+        # as the highest-versioned implementation of a given object.
+        version = versionutils.convert_version_to_tuple(cls.VERSION)
+        if not hasattr(objects, cls.obj_name()):
+            setattr(objects, cls.obj_name(), cls)
+        else:
+            cur_version = versionutils.convert_version_to_tuple(
+                getattr(objects, cls.obj_name()).VERSION)
+            if version >= cur_version:
+                setattr(objects, cls.obj_name(), cls)
+
+
+class CyborgObject(object_base.VersionedObject):
+    """Base class and object factory.
+
+    This forms the base of all objects that can be remoted or instantiated
+    via RPC. Simply defining a class that inherits from this base class
+    will make it remotely instantiatable. Objects should implement the
+    necessary "get" classmethod routines as well as "save" object methods
+    as appropriate.
+    """
+
+    OBJ_SERIAL_NAMESPACE = 'cyborg_object'
+    OBJ_PROJECT_NAMESPACE = 'cyborg'
+
+    fields = {
+        'created_at': object_fields.DateTimeField(nullable=True),
+        'updated_at': object_fields.DateTimeField(nullable=True),
+    }
+
+    def as_dict(self):
+        return dict((k, getattr(self, k))
+                    for k in self.fields
+                    if hasattr(self, k))
+
+    @staticmethod
+    def _from_db_object(context, obj, db_object):
+        """Converts a database entity to a formal object.
+
+        :param context: security context
+        :param obj: An object of the class.
+        :param db_object: A DB model of the object
+        :return: The object of the class with the database entity added
+        """
+
+        for field in obj.fields:
+            obj[field] = db_object[field]
+
+        obj.obj_reset_changes()
+        return obj
+
+
+class CyborgObjectSerializer(object_base.VersionedObjectSerializer):
+    # Base class to use for object hydration
+    OBJ_BASE_CLASS = CyborgObject

cyborg/objects/fields.py

@@ -0,0 +1,32 @@
+# Copyright 2017 Huawei Technologies Co.,LTD.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from oslo_versionedobjects import fields as object_fields
+
+
+class IntegerField(object_fields.IntegerField):
+    pass
+
+
+class UUIDField(object_fields.UUIDField):
+    pass
+
+
+class StringField(object_fields.StringField):
+    pass
+
+
+class DateTimeField(object_fields.DateTimeField):
+    pass

requirements.txt

@@ -15,3 +15,9 @@ oslo.messaging!=5.25.0,>=5.24.2 # Apache-2.0
 oslo.concurrency>=3.8.0 # Apache-2.0
 oslo.service>=1.10.0 # Apache-2.0
 oslo.serialization!=2.19.1,>=1.10.0 # Apache-2.0
+oslo.db>=4.24.0 # Apache-2.0
+oslo.utils>=3.20.0 # Apache-2.0
+oslo.versionedobjects>=1.17.0 # Apache-2.0
+SQLAlchemy!=1.1.5,!=1.1.6,!=1.1.7,!=1.1.8,>=1.0.10 # MIT
+alembic>=0.8.10 # MIT
+stevedore>=1.20.0 # Apache-2.0

setup.cfg

@@ -26,6 +26,11 @@ packages =
 [entry_points]
 console_scripts =
     cyborg-api = cyborg.cmd.api:main
+    cyborg-conductor = cyborg.cmd.conductor:main
+    cyborg-dbsync = cyborg.cmd.dbsync:main
+
+cyborg.database.migration_backend =
+    sqlalchemy = cyborg.db.sqlalchemy.migration
 
 [build_sphinx]
 source-dir = doc/source