Merge "sqlalchemy initial commit"

devstack/plugin.sh

@@ -133,6 +133,15 @@ function disable_vitrage_datasource {
 
 }
 
+# Set configuration for database backend.
+function vitrage_configure_db_backend {
+    if [ "$VITRAGE_DATABASE" = 'mysql' ] || [ "$VITRAGE_DATABASE" = 'postgresql' ] ; then
+        iniset $VITRAGE_CONF database connection $(database_connection_url vitrage)
+    else
+        die $LINENO "Unable to configure unknown VITRAGE_DATABASE $VITRAGE_DATABASE"
+    fi
+}
+
 # Configure Vitrage
 function configure_vitrage {
     iniset_rpc_backend vitrage $VITRAGE_CONF
@@ -165,6 +174,9 @@ function configure_vitrage {
     iniset $VITRAGE_CONF service_credentials region_name $REGION_NAME
     iniset $VITRAGE_CONF service_credentials auth_url $KEYSTONE_SERVICE_URI
 
+    # Configured db
+    vitrage_configure_db_backend
+
     # remove neutron vitrage datasource if neutron datasource not installed
     if ! is_service_enabled neutron; then
         disable_vitrage_datasource neutron.network neutron.port
@@ -239,6 +251,14 @@ function configure_vitrage {
 function init_vitrage {
     # Get vitrage keystone settings in place
     _vitrage_create_accounts
+
+    # Create and upgrade database only when used
+    if is_service_enabled mysql postgresql; then
+        if [ "$VITRAGE_DATABASE" = 'mysql' ] || [ "$VITRAGE_DATABASE" = 'postgresql' ] ; then
+            recreate_database vitrage
+            $VITRAGE_BIN_DIR/vitrage-dbsync
+        fi
+    fi
     # Create cache dir
     sudo install -d -o $STACK_USER $VITRAGE_AUTH_CACHE_DIR
     rm -f $VITRAGE_AUTH_CACHE_DIR/*

devstack/settings

@@ -18,6 +18,9 @@ VITRAGE_CONF=$VITRAGE_CONF_DIR/vitrage.conf
 VITRAGE_AUTH_CACHE_DIR=${VITRAGE_AUTH_CACHE_DIR:-/var/cache/vitrage}
 VITRAGE_WSGI_DIR=${VITRAGE_WSGI_DIR:-/var/www/vitrage}
 
+# Set up database backend
+VITRAGE_DATABASE=${VITRAGE_DATABASE:-mysql}
+
 # Vitrage connection info.
 VITRAGE_SERVICE_PROTOCOL=http
 VITRAGE_SERVICE_HOST=$SERVICE_HOST

etc/vitrage/vitrage-config-generator.conf

@@ -9,3 +9,4 @@ namespace = oslo.middleware
 namespace = oslo.policy
 namespace = keystonemiddleware.auth_token
 namespace = osprofiler
+namespace = oslo.db

requirements.txt

@@ -5,6 +5,7 @@
 pbr!=2.1.0,>=2.0.0 # Apache-2.0
 Babel!=2.4.0,>=2.3.4 # BSD
 lxml!=3.7.0,>=3.4.1 # BSD
+PyMySQL>=0.7.6 # MIT License
 python-ceilometerclient>=2.5.0 # Apache-2.0
 python-cinderclient>=3.2.0 # Apache-2.0
 python-dateutil>=2.4.2 # BSD
@@ -16,6 +17,7 @@ pyzabbix>=0.7.4 # LGPL
 networkx>=1.10 # BSD
 oslo.config>=4.6.0 # Apache-2.0
 oslo.context>=2.14.0 # Apache-2.0
+oslo.db>=4.24.0 # Apache-2.0
 oslo.messaging>=5.29.0 # Apache-2.0
 oslo.middleware>=3.31.0 # Apache-2.0
 oslo.serialization!=2.19.1,>=2.18.0 # Apache-2.0
@@ -29,6 +31,7 @@ Werkzeug>=0.7 # BSD License
 keystonemiddleware>=4.17.0 # Apache-2.0
 stevedore>=1.20.0 # Apache-2.0
 voluptuous>=0.8.9 # BSD License
+SQLAlchemy>=1.0.10,!=1.1.5,!=1.1.6,!=1.1.7,!=1.1.8 # MIT
 sympy>=0.7.6 # BSD
 pysnmp>=4.2.3 # BSD
 PyJWT>=1.0.1 # MIT

setup.cfg

@@ -30,6 +30,7 @@ console_scripts =
     vitrage-notifier = vitrage.cli.notifier:main
     vitrage-collector = vitrage.cli.collector:main
     vitrage-ml = vitrage.cli.machine_learning:main
+    vitrage-dbsync = vitrage.cli.storage:dbsync
 
 vitrage.entity_graph =
     networkx = vitrage.graph.driver.networkx_graph:NXGraph
@@ -41,6 +42,12 @@ oslo.config.opts =
 tempest.test_plugins =
     vitrage_tests = vitrage_tempest_tests.plugin:VitrageTempestPlugin
 
+vitrage.storage =
+    mysql = vitrage.storage.impl_sqlalchemy:Connection
+    mysql+pymysql = vitrage.storage.impl_sqlalchemy:Connection
+    postgresql = vitrage.storage.impl_sqlalchemy:Connection
+    sqlite = vitrage.storage.impl_sqlalchemy:Connection
+
 [files]
 packages =
     vitrage

test-requirements.txt

@@ -7,6 +7,7 @@ python-dateutil>=2.4.2 # BSD
 coverage!=4.4,>=4.0 # Apache-2.0
 lxml!=3.7.0,>=3.4.1 # BSD
 networkx>=1.10 # BSD
+PyMySQL>=0.7.6 # MIT License
 python-ceilometerclient>=2.5.0 # Apache-2.0
 python-cinderclient>=3.2.0 # Apache-2.0
 python-neutronclient>=6.3.0 # Apache-2.0
@@ -14,6 +15,7 @@ python-novaclient>=9.1.0 # Apache-2.0
 python-heatclient>=1.10.0 # Apache-2.0
 python-subunit>=0.0.18 # Apache-2.0/BSD
 pyzabbix>=0.7.4 # LGPL
+oslo.db>=4.24.0 # Apache-2.0
 oslo.log>=3.30.0 # Apache-2.0
 oslotest>=1.10.0 # Apache-2.0
 oslo.service>=1.24.0 # Apache-2.0
@@ -33,6 +35,7 @@ sympy>=0.7.6 # BSD
 reno>=2.5.0 # Apache-2.0
 pysnmp>=4.2.3 # BSD
 osprofiler>=1.4.0 # Apache-2.0
+SQLAlchemy>=1.0.10,!=1.1.5,!=1.1.6,!=1.1.7,!=1.1.8 # MIT
 
 # Doc requirements
 openstackdocstheme>=1.17.0 # Apache-2.0

vitrage/api/app.py

@@ -39,7 +39,8 @@ def setup_app(root, conf=None):
     app_hooks = [hooks.ConfigHook(conf),
                  hooks.TranslationHook(),
                  hooks.RPCHook(conf),
-                 hooks.ContextHook()]
+                 hooks.ContextHook(),
+                 hooks.DBHook(conf)]
 
     app = pecan.make_app(
         root,

vitrage/api/hooks.py

@@ -18,6 +18,7 @@ from pecan import hooks
 
 from vitrage import messaging
 from vitrage import rpc as vitrage_rpc
+from vitrage import storage
 
 
 class ConfigHook(hooks.PecanHook):
@@ -74,3 +75,12 @@ class ContextHook(hooks.PecanHook):
 
         # Inject the context...
         state.request.context = ctx.to_dict()
+
+
+class DBHook(hooks.PecanHook):
+
+    def __init__(self, conf):
+        self.storage = storage.get_connection_from_config(conf)
+
+    def before(self, state):
+        state.request.storage = self.storage

vitrage/cli/graph.py

@@ -61,6 +61,10 @@ def init(conf):
         'Entity Graph',
         '%s:%s:%s' % (EntityCategory.RESOURCE, OPENSTACK_CLUSTER, CLUSTER_ID),
         uuid=True)
+
+    # TODO(ihefetz) uncomment db connection creation
+    # db_connection = storage.get_connection_from_config(conf)
+
     scenario_repo = ScenarioRepository(conf)
 
     evaluator = ScenarioEvaluator(conf, e_graph, scenario_repo, evaluator_q)

vitrage/cli/storage.py

@@ -0,0 +1,21 @@
+# Copyright 2017 - Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from vitrage import service
+from vitrage import storage
+
+
+def dbsync():
+    conf = service.prepare_service()
+    storage.get_connection_from_config(conf).upgrade()

vitrage/opts.py

@@ -29,6 +29,7 @@ import vitrage.notifier
 import vitrage.notifier.plugins.snmp
 import vitrage.os_clients
 import vitrage.rpc
+import vitrage.storage
 
 LOG = log.getLogger(__name__)
 
@@ -44,6 +45,7 @@ def list_opts():
         ('datasources', vitrage.datasources.OPTS),
         ('evaluator', vitrage.evaluator.OPTS),
         ('consistency', vitrage.entity_graph.consistency.OPTS),
+        ('database', vitrage.storage.OPTS),
         ('entity_graph', vitrage.entity_graph.OPTS),
         ('service_credentials', vitrage.keystone_client.OPTS),
         ('machine_learning',

vitrage/service.py

@@ -12,6 +12,7 @@
 # under the License.
 
 from oslo_config import cfg
+from oslo_db import options as db_options
 from oslo_log import log
 from oslo_policy import opts as policy_opts
 from osprofiler import initializer as osprofiler_initializer
@@ -31,6 +32,7 @@ def prepare_service(args=None, conf=None, config_files=None):
     log.register_options(conf)
     policy_opts.set_defaults(conf)
     osprofiler_opts.set_defaults(conf)
+    db_options.set_defaults(conf)
 
     for group, options in opts.list_opts():
         conf.register_opts(list(options),

vitrage/storage/__init__.py

@@ -0,0 +1,45 @@
+# Copyright 2017 - Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from oslo_log import log
+import six.moves.urllib.parse as urlparse
+from stevedore import driver
+import tenacity
+
+_NAMESPACE = 'vitrage.storage'
+
+
+LOG = log.getLogger(__name__)
+
+
+OPTS = []
+
+
+def get_connection_from_config(conf):
+    retries = conf.database.max_retries
+    url = conf.database.connection
+    connection_scheme = urlparse.urlparse(url).scheme
+    LOG.debug('looking for %(name)r driver in %(namespace)r',
+              {'name': connection_scheme, 'namespace': _NAMESPACE})
+    mgr = driver.DriverManager(_NAMESPACE, connection_scheme)
+
+    @tenacity.retry(
+        wait=tenacity.wait_fixed(conf.database.retry_interval),
+        stop=tenacity.stop_after_attempt(retries if retries >= 0 else 5),
+        reraise=True)
+    def _get_connection():
+        """Return an open connection to the database."""
+        return mgr.driver(conf, url)
+
+    return _get_connection()

vitrage/storage/base.py

@@ -0,0 +1,24 @@
+# Copyright 2017 - Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+class Connection(object):
+    """Base class for storage system connections."""
+
+    def __init__(self, conf, url):
+        pass
+
+    @staticmethod
+    def upgrade(nocreate=False):
+        raise NotImplementedError('upgrade not implemented')

vitrage/storage/impl_sqlalchemy.py

@@ -0,0 +1,65 @@
+# Copyright 2017 - Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+from __future__ import absolute_import
+
+from oslo_db.sqlalchemy import session as db_session
+from oslo_log import log
+from sqlalchemy.engine import url as sqlalchemy_url
+
+from vitrage import storage
+from vitrage.storage import base
+from vitrage.storage.sqlalchemy import models
+
+LOG = log.getLogger(__name__)
+
+
+class Connection(base.Connection):
+
+    def __init__(self, conf, url):
+        options = dict(conf.database.items())
+        # set retries to 0 , since reconnection is already implemented
+        # in storage.__init__.get_connection_from_config function
+        options['max_retries'] = 0
+        # add vitrage opts to database group
+        for opt in storage.OPTS:
+            options.pop(opt.name, None)
+        self._engine_facade = db_session.EngineFacade(self._dress_url(url),
+                                                      **options)
+        self.conf = conf
+
+    @staticmethod
+    def _dress_url(url):
+        # If no explicit driver has been set, we default to pymysql
+        if url.startswith("mysql://"):
+            url = sqlalchemy_url.make_url(url)
+            url.drivername = "mysql+pymysql"
+            return str(url)
+        return url
+
+    def upgrade(self, nocreate=False):
+        engine = self._engine_facade.get_engine()
+        engine.connect()
+        models.Base.metadata.create_all(engine, checkfirst=False)
+        # TODO(ihefetz) upgrade logic is missing
+
+    def disconnect(self):
+        self._engine_facade.get_engine().dispose()
+
+    def clear(self):
+        engine = self._engine_facade.get_engine()
+        for table in reversed(models.Base.metadata.sorted_tables):
+            engine.execute(table.delete())
+        engine.dispose()

vitrage/storage/sqlalchemy/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2015 - Alcatel-Lucent
+# Copyright 2017 - Nokia
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you may
 # not use this file except in compliance with the License. You may obtain

vitrage/storage/sqlalchemy/models.py

@@ -0,0 +1,39 @@
+# Copyright 2017 - Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from oslo_db.sqlalchemy import models
+
+import six
+from sqlalchemy.ext.declarative import declarative_base
+
+
+class VitrageBase(models.TimestampMixin, models.ModelBase):
+    """Base class for Vitrage Models."""
+    __table_args__ = {'mysql_charset': "utf8",
+                      'mysql_engine': "InnoDB"}
+    __table_initialized__ = False
+
+    def __setitem__(self, key, value):
+        setattr(self, key, value)
+
+    def __getitem__(self, key):
+        return getattr(self, key)
+
+    def update(self, values):
+        """Make the model object behave like a dict."""
+        for k, v in six.iteritems(values):
+            setattr(self, k, v)
+
+
+Base = declarative_base(cls=VitrageBase)

vitrage/tests/functional/api/__init__.py

@@ -58,6 +58,11 @@ class FunctionalTest(base.BaseTest):
                                group='api')
 
         self.CONF.set_override('auth_mode', self.auth, group='api')
+
+        self.CONF.set_override('connection',
+                               'sqlite:///:memory:',
+                               group='database')
+
         self.app = webtest.TestApp(app.load_app(self.CONF))
 
     def put_json(self, path, params, expect_errors=False, headers=None,