small refactoring - service and storage (dao) layers has been created

Change-Id: Ia3c106bfbe87a7e9c08659fd61308164b9afb2dc

savanna/cli.py

@@ -18,7 +18,7 @@ from flask import Flask
 from oslo.config import cfg
 from savanna.openstack.common import log
 from savanna.storage.defaults import setup_defaults
-from savanna.storage.storage import setup_storage, DB
+from savanna.storage.db import setup_storage, DB
 
 CONF = cfg.CONF
 LOG = log.getLogger(__name__)

savanna/main.py

@@ -24,7 +24,7 @@ from savanna.api import v02 as api_v02
 from savanna.middleware.auth_valid import filter_factory as auth_valid
 from savanna.utils.scheduler import setup_scheduler
 from savanna.utils.api import render
-from savanna.storage.storage import setup_storage
+from savanna.storage.db import setup_storage
 
 from savanna.openstack.common import log
 

savanna/service/api.py

@@ -17,12 +17,10 @@ import eventlet
 from oslo.config import cfg
 from flask import request
 
-from savanna.storage.models import NodeTemplate, NodeType, NodeProcess, \
-    NodeTemplateConfig, Cluster, ClusterNodeCount
-from savanna.storage.storage import DB
 from savanna.utils.api import abort_and_log
 from savanna.service import cluster_ops
 from savanna.openstack.common import log as logging
+import savanna.storage.storage as storage
 
 LOG = logging.getLogger(__name__)
 
@@ -30,6 +28,105 @@ CONF = cfg.CONF
 CONF.import_opt('allow_cluster_ops', 'savanna.config')
 
 
+## Node Template ops:
+
+def get_node_template(**args):
+    return _node_template(storage.get_node_template(**args))
+
+
+def get_node_templates(**args):
+    return [_node_template(tmpl) for tmpl
+            in storage.get_node_templates(**args)]
+
+
+def create_node_template(values):
+    """
+    Creates new node template from values dict
+    :param values: dict
+    :return: created node template resource
+    """
+    values = values.pop('node_template')
+
+    name = values.pop('name')
+    node_type_id = storage.get_node_type(name=values.pop('node_type')).id
+    # todo(slukjanov): take tenant_id from headers
+    tenant_id = "tenant-01"
+    flavor_id = values.pop('flavor_id')
+
+    nt = storage.create_node_template(name, node_type_id, tenant_id,
+                                      flavor_id, values)
+
+    return get_node_template(id=nt.id)
+
+
+def terminate_node_template(**args):
+    return storage.terminate_node_template(**args)
+
+
+## Cluster ops:
+
+def get_cluster(**args):
+    return _cluster(storage.get_cluster(**args))
+
+
+def get_clusters(**args):
+    return [_cluster(cluster) for cluster in
+            storage.get_clusters(**args)]
+
+
+def create_cluster(values):
+    values = values.pop('cluster')
+
+    name = values.pop('name')
+    base_image_id = values.pop('base_image_id')
+    # todo(slukjanov): take tenant_id from headers
+    tenant_id = "tenant-01"
+    templates = values.pop('node_templates')
+
+    # todo(slukjanov): check that we can create objects in the specified tenant
+
+    cluster = storage.create_cluster(name, base_image_id, tenant_id, templates)
+
+    eventlet.spawn(_cluster_creation_job, request.headers, cluster.id)
+
+    return get_cluster(id=cluster.id)
+
+
+def _cluster_creation_job(headers, cluster_id):
+    cluster = storage.get_cluster(id=cluster_id)
+    LOG.debug("Starting cluster '%s' creation: %s", cluster_id,
+              _cluster(cluster).dict)
+
+    if CONF.allow_cluster_ops:
+        cluster_ops.launch_cluster(headers, cluster)
+    else:
+        LOG.info("Cluster ops are disabled, use --allow-cluster-ops flag")
+
+    # update cluster status
+    storage.update_cluster_status('Active', id=cluster.id)
+
+
+def terminate_cluster(**args):
+    cluster = storage.update_cluster_status('Stoping', **args)
+
+    eventlet.spawn(_cluster_termination_job, request.headers, cluster.id)
+
+
+def _cluster_termination_job(headers, cluster_id):
+    cluster = storage.get_cluster(id=cluster_id)
+    LOG.debug("Stoping cluster '%s' creation: %s", cluster_id,
+              _cluster(cluster).dict)
+
+    if CONF.allow_cluster_ops:
+        cluster_ops.stop_cluster(headers, cluster)
+    else:
+        LOG.info("Cluster ops are disabled, use --allow-cluster-ops flag")
+
+    storage.terminate_cluster(id=cluster.id)
+
+
+## Utils and DB object to Resource converters
+
 def _clean_nones(obj):
     d_type = type(obj)
     if d_type is not dict or d_type is not list:
@@ -99,56 +196,6 @@ def _node_template(nt):
     return Resource('node_template', d)
 
 
-def _template_id_by_name(template):
-    return NodeTemplate.query.filter_by(name=template).first().id
-
-
-def _type_id_by_name(_type):
-    return NodeType.query.filter_by(name=_type).first().id
-
-
-def get_node_template(**args):
-    return _node_template(NodeTemplate.query.filter_by(**args).first())
-
-
-def get_node_templates(**args):
-    return [_node_template(tmpl) for tmpl
-            in NodeTemplate.query.filter_by(**args).all()]
-
-
-def create_node_template(values):
-    """
-    Creates new node template from values dict
-    :param values: dict
-    :return: created node template resource
-    """
-    values = values.pop('node_template')
-
-    name = values.pop('name')
-    node_type_id = _type_id_by_name(values.pop('node_type'))
-    # todo(slukjanov): take tenant_id from headers
-    tenant_id = "tenant-01"
-    flavor_id = values.pop('flavor_id')
-
-    nt = NodeTemplate(name, node_type_id, tenant_id, flavor_id)
-    DB.session.add(nt)
-    for process_name in values:
-        process = NodeProcess.query.filter_by(name=process_name).first()
-        conf = values.get(process_name)
-        for prop in process.node_process_properties:
-            val = conf.get(prop.name, None)
-            if not val and prop.required:
-                if not prop.default:
-                    raise RuntimeError('Template \'%s\', value missed '
-                                       'for required param: %s %s'
-                                       % (name, process.name, prop.name))
-                val = prop.default
-            DB.session.add(NodeTemplateConfig(nt.id, prop.id, val))
-    DB.session.commit()
-
-    return get_node_template(id=nt.id)
-
-
 def _cluster(cluster):
     if not cluster:
         abort_and_log(404, 'Cluster not found')
@@ -173,94 +220,3 @@ def _cluster(cluster):
         d['service_urls'][service.name] = service.url
 
     return Resource('cluster', d)
-
-
-def get_cluster(**args):
-    return _cluster(Cluster.query.filter_by(**args).first())
-
-
-def get_clusters(**args):
-    return [_cluster(cluster) for cluster in
-            Cluster.query.filter_by(**args).all()]
-
-
-def create_cluster(values):
-    values = values.pop('cluster')
-
-    name = values.pop('name')
-    base_image_id = values.pop('base_image_id')
-    # todo(slukjanov): take tenant_id from headers
-    tenant_id = "tenant-01"
-    templates = values.pop('node_templates')
-
-    # todo(slukjanov): check that we can create objects in the specified tenant
-
-    cluster = Cluster(name, base_image_id, tenant_id)
-    DB.session.add(cluster)
-    for template in templates:
-        count = templates.get(template)
-        template_id = _template_id_by_name(template)
-        cnc = ClusterNodeCount(cluster.id, template_id, int(count))
-        DB.session.add(cnc)
-    DB.session.commit()
-
-    eventlet.spawn(_cluster_creation_job, request.headers, cluster.id)
-
-    return get_cluster(id=cluster.id)
-
-
-def _cluster_creation_job(headers, cluster_id):
-    cluster = Cluster.query.filter_by(id=cluster_id).first()
-    LOG.debug("Starting cluster '%s' creation: %s", cluster_id,
-              _cluster(cluster).dict)
-
-    if CONF.allow_cluster_ops:
-        cluster_ops.launch_cluster(headers, cluster)
-    else:
-        LOG.info("Cluster ops are disabled, use --allow-cluster-ops flag")
-
-    # update cluster status
-    cluster = Cluster.query.filter_by(id=cluster.id).first()
-    cluster.status = 'Active'
-    DB.session.add(cluster)
-    DB.session.commit()
-
-
-def terminate_cluster(**args):
-    # update cluster status
-    cluster = Cluster.query.filter_by(**args).first()
-    cluster.status = 'Stoping'
-    DB.session.add(cluster)
-    DB.session.commit()
-
-    eventlet.spawn(_cluster_termination_job, request.headers, cluster.id)
-
-
-def _cluster_termination_job(headers, cluster_id):
-    cluster = Cluster.query.filter_by(id=cluster_id).first()
-    LOG.debug("Stoping cluster '%s' creation: %s", cluster_id,
-              _cluster(cluster).dict)
-
-    if CONF.allow_cluster_ops:
-        cluster_ops.stop_cluster(headers, cluster)
-    else:
-        LOG.info("Cluster ops are disabled, use --allow-cluster-ops flag")
-
-    DB.session.delete(cluster)
-    DB.session.commit()
-
-
-def terminate_node_template(**args):
-    template = NodeTemplate.query.filter_by(**args).first()
-    if template:
-        if len(template.nodes):
-            abort_and_log(500, "There are active nodes created using "
-                               "template '%s' you trying to terminate"
-                               % args)
-        else:
-            DB.session.delete(template)
-            DB.session.commit()
-
-        return True
-    else:
-        return False

savanna/service/cluster_ops.py

@@ -20,7 +20,7 @@ from jinja2 import PackageLoader
 from paramiko import SSHClient, AutoAddPolicy
 from oslo.config import cfg
 from savanna.storage.models import Node, ServiceUrl
-from savanna.storage.storage import DB
+from savanna.storage.db import DB
 from savanna.utils.openstack.nova import novaclient
 from savanna.openstack.common import log as logging
 

savanna/storage/db.py

@@ -0,0 +1,40 @@
+# Copyright (c) 2013 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from flask.ext.sqlalchemy import SQLAlchemy
+from oslo.config import cfg
+
+DB = SQLAlchemy()
+
+opts = [
+    cfg.StrOpt('database_uri',
+               default='sqlite:////tmp/savanna.db',
+               help='URL for sqlalchemy database'),
+    cfg.BoolOpt('echo',
+                default=False,
+                help='Sqlalchemy echo')
+]
+
+CONF = cfg.CONF
+CONF.register_opts(opts, group='sqlalchemy')
+
+
+def setup_storage(app):
+    app.config['SQLALCHEMY_DATABASE_URI'] = CONF.sqlalchemy.database_uri
+    app.config['SQLALCHEMY_ECHO'] = CONF.sqlalchemy.echo
+
+    DB.app = app
+    DB.init_app(app)
+    DB.create_all()

savanna/storage/defaults.py

@@ -13,96 +13,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from savanna.storage.models import NodeProcess, NodeProcessProperty, \
+from savanna.storage.storage import create_node_type, \
-    NodeType, NodeTemplate, NodeTemplateConfig, Cluster, ClusterNodeCount
+    create_node_template, create_node_process
-from savanna.storage.storage import DB
 from savanna.openstack.common import log as logging
 
 LOG = logging.getLogger(__name__)
 
 
-def create_node_process(name, properties):
-    """
-    Creates new node process and node process properties
-    :param name: process name
-    :param properties: array of triples (name, required, default)
-    :return: created node process
-    """
-    process = NodeProcess(name)
-    DB.session.add(process)
-    DB.session.commit()
-    for p in properties:
-        prop = NodeProcessProperty(process.id, p[0], p[1], p[2])
-        DB.session.add(prop)
-    DB.session.commit()
-    return process
-
-
-def create_node_type(name, processes):
-    """
-    Creates new node type using specified list of processes
-    :param name:
-    :param processes:
-    :return:
-    """
-    node_type = NodeType(name)
-    node_type.processes = processes
-    DB.session.add(node_type)
-    DB.session.commit()
-    return node_type
-
-
-def create_node_template(name, node_type_id, tenant_id, flavor_id, configs):
-    """
-    Creates new node templates
-    :param name: template name
-    :param node_type_id: node type
-    :param tenant_id: tenant
-    :param flavor_id: flavor
-    :param configs: dict of process->property->value
-    :return: created node template
-    """
-    node_template = NodeTemplate(name, node_type_id, tenant_id, flavor_id)
-    DB.session.add(node_template)
-    for process_name in configs:
-        process = NodeProcess.query.filter_by(name=process_name).first()
-        conf = configs.get(process_name)
-        for prop in process.node_process_properties:
-            val = conf.get(prop.name, None)
-            if not val and prop.required:
-                if not prop.default:
-                    raise RuntimeError('Template \'%s\', value missed '
-                                       'for required param: %s %s'
-                                       % (name, process.name, prop.name))
-                val = prop.default
-            DB.session.add(NodeTemplateConfig(node_template.id, prop.id, val))
-    DB.session.commit()
-
-    return node_template
-
-
-def create_cluster(name, base_image_id, tenant_id, templates):
-    """
-    Creates new cluster
-    :param name: cluster name
-    :param base_image_id: base image
-    :param tenant_id: tenant
-    :param templates: dict of template->count
-    :return: created cluster
-    """
-    cluster = Cluster(name, base_image_id, tenant_id)
-    DB.session.add(cluster)
-    for template in templates:
-        count = templates.get(template)
-        cnc = ClusterNodeCount(cluster.id,
-                               NodeTemplate.query.filter_by(name=template)
-                               .first().id, int(count))
-        DB.session.add(cnc)
-    DB.session.commit()
-
-    return cluster
-
-
 def setup_defaults(reset_db=False, gen_templates=False):
     nt_jt_nn = None
     nt_jt = None

savanna/storage/models.py

@@ -15,7 +15,7 @@
 
 from uuid import uuid4
 
-from savanna.storage.storage import DB
+from savanna.storage.db import DB
 
 
 class NodeTemplate(DB.Model):

savanna/storage/storage.py

@@ -13,28 +13,150 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from flask.ext.sqlalchemy import SQLAlchemy
+from savanna.storage.db import DB
-from oslo.config import cfg
 
-DB = SQLAlchemy()
+from savanna.storage.models import NodeTemplate, NodeProcess, Cluster, \
-
+    ClusterNodeCount, NodeTemplateConfig, NodeType, NodeProcessProperty
-opts = [
+from savanna.utils.api import abort_and_log
-    cfg.StrOpt('database_uri',
-               default='sqlite:////tmp/savanna.db',
-               help='URL for sqlalchemy database'),
-    cfg.BoolOpt('echo',
-                default=False,
-                help='Sqlalchemy echo')
-]
-
-CONF = cfg.CONF
-CONF.register_opts(opts, group='sqlalchemy')
 
 
-def setup_storage(app):
+## Node Template ops:
-    app.config['SQLALCHEMY_DATABASE_URI'] = CONF.sqlalchemy.database_uri
-    app.config['SQLALCHEMY_ECHO'] = CONF.sqlalchemy.echo
 
-    DB.app = app
+def get_node_template(**args):
-    DB.init_app(app)
+    return NodeTemplate.query.filter_by(**args).first()
-    DB.create_all()
+
+
+def get_node_templates(**args):
+    return NodeTemplate.query.filter_by(**args).all()
+
+
+def create_node_template(name, node_type_id, tenant_id, flavor_id, configs):
+    """
+    Creates new node templates
+    :param name: template name
+    :param node_type_id: node type
+    :param tenant_id: tenant
+    :param flavor_id: flavor
+    :param configs: dict of process->property->value
+    :return: created node template
+    """
+    node_template = NodeTemplate(name, node_type_id, tenant_id, flavor_id)
+    DB.session.add(node_template)
+    for process_name in configs:
+        process = NodeProcess.query.filter_by(name=process_name).first()
+        conf = configs.get(process_name)
+        for prop in process.node_process_properties:
+            val = conf.get(prop.name, None)
+            if not val and prop.required:
+                if not prop.default:
+                    raise RuntimeError('Template \'%s\', value missed '
+                                       'for required param: %s %s'
+                                       % (name, process.name, prop.name))
+                val = prop.default
+            DB.session.add(NodeTemplateConfig(node_template.id, prop.id, val))
+    DB.session.commit()
+
+    return node_template
+
+
+def terminate_node_template(**args):
+    template = NodeTemplate.query.filter_by(**args).first()
+    if template:
+        if len(template.nodes):
+            abort_and_log(500, "There are active nodes created using "
+                               "template '%s' you trying to terminate"
+                               % args)
+        else:
+            DB.session.delete(template)
+            DB.session.commit()
+
+        return True
+    else:
+        return False
+
+
+## Cluster ops:
+
+def get_cluster(**args):
+    return Cluster.query.filter_by(**args).first()
+
+
+def get_clusters(**args):
+    return Cluster.query.filter_by(**args).all()
+
+
+def create_cluster(name, base_image_id, tenant_id, templates):
+    """
+    Creates new cluster
+    :param name: cluster name
+    :param base_image_id: base image
+    :param tenant_id: tenant
+    :param templates: dict of template->count
+    :return: created cluster
+    """
+    cluster = Cluster(name, base_image_id, tenant_id)
+    DB.session.add(cluster)
+    for template in templates:
+        count = templates.get(template)
+        template_id = get_node_template(name=template).id
+        cnc = ClusterNodeCount(cluster.id, template_id, int(count))
+        DB.session.add(cnc)
+    DB.session.commit()
+
+    return cluster
+
+
+def terminate_cluster(**args):
+    cluster = get_cluster(**args)
+    DB.session.delete(cluster)
+    DB.session.commit()
+
+
+def update_cluster_status(new_status, **args):
+    cluster = Cluster.query.filter_by(**args).first()
+    cluster.status = new_status
+    DB.session.add(cluster)
+    DB.session.commit()
+
+    return cluster
+
+
+## Node Process ops:
+
+def create_node_process(name, properties):
+    """
+    Creates new node process and node process properties
+    :param name: process name
+    :param properties: array of triples (name, required, default)
+    :return: created node process
+    """
+    process = NodeProcess(name)
+    DB.session.add(process)
+    DB.session.commit()
+    for p in properties:
+        prop = NodeProcessProperty(process.id, p[0], p[1], p[2])
+        DB.session.add(prop)
+    DB.session.commit()
+
+    return process
+
+
+## Node Type ops:
+
+def get_node_type(**args):
+    return NodeType.query.filter_by(**args).first()
+
+
+def create_node_type(name, processes):
+    """
+    Creates new node type using specified list of processes
+    :param name:
+    :param processes:
+    :return:
+    """
+    node_type = NodeType(name)
+    node_type.processes = processes
+    DB.session.add(node_type)
+    DB.session.commit()
+
+    return node_type

savanna/tests/unit/test_api_v02.py

@@ -26,7 +26,7 @@ from savanna.main import make_app
 from savanna.service import api
 from savanna.storage.defaults import setup_defaults
 from savanna.storage.models import Node, NodeTemplate
-from savanna.storage.storage import DB
+from savanna.storage.db import DB
 import savanna.main
 from savanna.utils import scheduler
 from savanna.openstack.common import log as logging
@@ -85,8 +85,8 @@ def _stub_auth_valid(*args, **kwargs):
 CONF = cfg.CONF
 CONF.import_opt('debug', 'savanna.openstack.common.log')
 CONF.import_opt('allow_cluster_ops', 'savanna.config')
-CONF.import_opt('database_uri', 'savanna.storage.storage', group='sqlalchemy')
+CONF.import_opt('database_uri', 'savanna.storage.db', group='sqlalchemy')
-CONF.import_opt('echo', 'savanna.storage.storage', group='sqlalchemy')
+CONF.import_opt('echo', 'savanna.storage.db', group='sqlalchemy')
 
 
 class TestApi(unittest.TestCase):